Improved docs

Author: dakrauth

Justfile

@@ -26,6 +26,7 @@ update:
     {{PIP}} install -U \
         build \
         pytest \
+        docutils \
         pytest-sugar \
         pytest-clarity \
         freezegun \
@@ -132,3 +133,8 @@ strftime:
         f"Format Specifiers:\n{pt.get_string()}\n\n"
         "Notes:\n* - Locale-dependent\n+ - C99 extension\n! - when extension"
     )
+
+# Preview README.rst
+docs:
+    rst2html5 --output README.html README.rst
+    open README.html

README.rst

@@ -1,15 +1,13 @@
 #!/usr/bin/env python
 import argparse
-import logging
 import sys
+import logging
 from pathlib import Path
 
 from . import __version__, core, db, utils, lunar, exceptions
 from . import timezones
 from .config import Settings, __doc__ as FORMAT_HELP
 
-logger = logging.getLogger(__name__)
-
 
 def get_parser(settings):
     class DBSizeAction(argparse.Action):
@@ -227,11 +225,14 @@ def log_config(verbosity, settings):
             logging.getLogger("asyncio").setLevel(logging.WARNING)
 
     logging.basicConfig(level=log_level, format=log_format, force=True)
+    logger = utils.logger()
     logger.debug(
         "Configuration files read: %s",
         ", ".join(str(s) for s in settings.read_from) if settings.read_from else "None",
     )
 
+    return logger
+
 
 def main(sys_args, when=None, settings=None):
     if "--pdb" in sys_args:  # pragma: no cover
@@ -258,7 +259,7 @@ def main(sys_args, when=None, settings=None):
     parser = get_parser(settings)
     args = parser.parse_args(sys_args)
 
-    log_config(args.verbosity, settings)
+    logger = log_config(args.verbosity, settings)
     logger.debug(args)
     if args.help:
         parser.print_help()

src/when/cli.py

@@ -39,7 +39,7 @@
 +------+--------------------------------------------------------------------------+--------------------------+------+
 |  %M  | Minute (00-59)                                                           | 55                       |      |
 +------+--------------------------------------------------------------------------+--------------------------+------+
-|  %n  | New-line character ('\n')                                                |                          |  +   |
+|  %n  | New-line character                                                       | '\\n'                     |  +   |
 +------+--------------------------------------------------------------------------+--------------------------+------+
 |  %p  | AM or PM designation                                                     | PM                       |      |
 +------+--------------------------------------------------------------------------+--------------------------+------+
@@ -49,7 +49,7 @@
 +------+--------------------------------------------------------------------------+--------------------------+------+
 |  %S  | Second (00-61)                                                           | 02                       |      |
 +------+--------------------------------------------------------------------------+--------------------------+------+
-|  %t  | Horizontal-tab character ('\t')                                          |                          |  +   |
+|  %t  | Horizontal-tab character                                                 | '\\t'                     |  +   |
 +------+--------------------------------------------------------------------------+--------------------------+------+
 |  %T  | ISO 8601 time format (HH:MM:SS), equivalent to %H:%M:%S                  | 14:55:02                 |  +   |
 +------+--------------------------------------------------------------------------+--------------------------+------+

src/when/config.py

@@ -13,7 +13,7 @@
 from .db import client
 from .lunar import lunar_phase
 
-logger = logging.getLogger(__name__)
+logger = utils.logger()
 
 
 def holidays(settings, co="US", ts=None):

src/when/core.py

@@ -8,7 +8,7 @@
 from .. import utils
 from ..exceptions import DBError
 
-logger = logging.getLogger(__name__)
+logger = utils.logger()
 
 DB_FILENAME = Path(__file__).parent / "when.db"
 DB_SCHEMA = """
@@ -41,7 +41,7 @@
 SELECT c.id, c.name, c.ascii, c.sub, c.co, c.tz
 FROM city c
 WHERE
-    (c.id = :value OR c.name = :value OR c.ascii = :value)
+    (c.id = :value OR UPPER(c.name) = :value OR UPPER(c.ascii) = :value)
 """
 
 ALIASES_LISTING_QUERY = """
@@ -165,6 +165,10 @@ def add_alias(self, name, gid):
                 [(val.strip(), gid) for val in name.split(",")],
             )
 
+    @property
+    def size(self):
+        return self.filename.stat().st_size if self.filename.exists() else 0
+
     @utils.timer
     def create_db(self, data, remove_existing=True):
         if self.filename.exists():
@@ -179,7 +183,7 @@ def create_db(self, data, remove_existing=True):
             cur.executemany("INSERT INTO city VALUES (?, ?, ?, ?, ?, ?, ?)", data)
             nrows = cur.rowcount
 
-        print(f"Inserted {nrows} rows")
+        logger.info(f"Inserted {nrows:,} rows ({self.size:,} bytes)")
 
     def _execute(self, con, sql, params):
         return con.execute(sql, params).fetchall()
@@ -192,29 +196,32 @@ def _search(self, sql, value, params):
         return City.from_results(results)
 
     def parse_search(self, value):
-        bits = [a.strip() for a in value.split(",")]
+        bits = [a.strip().upper() for a in value.split(",")]
         nbits = len(bits)
         if nbits > 3:
             raise DBError(f"Invalid city search expression: {value}")
 
         match nbits:
             case 1:
-                return [value, None, None]
+                return [bits[0], None, None]
             case 2:
-                return [bits[0], bits[1], None]
+                return [bits[0], None, bits[1]]
             case 3:
                 return bits
 
     def search(self, value, exact=False):
-        value, co, sub = self.parse_search(value)
+        value, sub, co = self.parse_search(value)
         if exact:
+            data = {"value": value}
             sql = XSEARCH_QUERY
             if co:
-                sql = f"{sql} AND c.co = :co AND c.sub = :sub" if sub else f"{sql} AND c.co = :co"
+                data["co"] = co
+                sql = f"{sql} AND UPPER(c.co) = :co"
+                if sub:
+                    data["sub"] = sub
+                    sql = f"{sql} AND UPPER(c.sub) = :sub"
 
-            return self._search(
-                sql, value, {"value": value, "co": co.upper() if co else co, "sub": sub}
-            )
+            return self._search(sql, value, data)
 
         like_exprs = ["c.name LIKE :like", "c.ascii LIKE :like"]
         if co:
@@ -230,7 +237,7 @@ def search(self, value, exact=False):
             {
                 "like": f"%{value}%",
                 "value": value,
-                "co": co.upper() if co else co,
-                "sub": sub.upper() if sub else sub,
+                "co": co,
+                "sub": sub,
             },
         )

src/when/db/client.py

@@ -1,12 +1,35 @@
 import io
+import time
 import logging
 import zipfile
-from collections import defaultdict
+from collections import defaultdict, namedtuple
 from pathlib import Path
 
 from .. import utils
 
-logger = logging.getLogger(__name__)
+logger = utils.logger()
+GeoCity = namedtuple(
+    "GeoCity",
+    "gid,"
+    "name,"
+    "aname,"
+    "alt,"
+    "lat,"
+    "lng,"
+    "fclass,"
+    "fcode,"
+    "co,"
+    "cc2,"
+    "a1,"
+    "a2,"
+    "a3,"
+    "a4,"
+    "pop,"
+    "el,"
+    "dem,"
+    "tz,"
+    "mod"
+)
 
 DB_DIR = Path(__file__).parent
 GEONAMES_CITIES_URL_FMT = "https://download.geonames.org/export/dump/cities{}.zip"
@@ -27,7 +50,12 @@ def fetch_cities(size, dirname=DB_DIR):
     if txt_filename.exists():
         return txt_filename
 
-    zip_bytes = utils.fetch(GEONAMES_CITIES_URL_FMT.format(size))
+    url = GEONAMES_CITIES_URL_FMT.format(size)
+    logger.info(f"Beginning download from {url}")
+    start = time.time()
+    zip_bytes = utils.fetch(url)
+    end = time.time()
+    logger.info(f"Received {len(zip_bytes):,} bytes in {end-start:,}s")
     zip_filename = io.BytesIO(zip_bytes)
     with zipfile.ZipFile(zip_filename) as z:
         z.extract(txt_filename.name, txt_filename.parent)
@@ -77,49 +105,28 @@ def process_geonames_txt(fobj, minimum_population=15_000, admin_1=None):
     skip_if = {"PPL", "PPLL", "PPLS", "PPLF", "PPLR"}
     admin_1 = admin_1 or {}
     data = []
-    i = 0
-    for line in fobj:
+    for i, line in enumerate(fobj, 1):
         i += 1
-        (
-            gid,
-            name,
-            aname,
-            alt,
-            lat,
-            lng,
-            fclass,
-            fcode,
-            co,
-            cc2,
-            a1,
-            a2,
-            a3,
-            a4,
-            pop,
-            el,
-            dem,
-            tz,
-            mod,
-        ) = line.rstrip().split("\t")
-
-        pop = int(pop) if pop else 0
+        ct = GeoCity(*line.rstrip().split("\t"))
+
+        pop = int(ct.pop) if ct.pop else 0
         if (
-            (fcode in skip)
-            or (fcode in skip_if and (pop < minimum_population))
-            or (fcode == "PPLA5" and name.startswith("Marseille") and name[-1].isdigit())
+            (ct.fcode in skip)
+            or (ct.fcode in skip_if and (pop < minimum_population))
+            or (ct.fcode == "PPLA5" and ct.name.startswith("Marseille") and ct.name[-1].isdigit())
         ):
-            skipped[fcode] += 1
+            skipped[ct.fcode] += 1
             continue
 
-        fcodes[fcode] += 1
-        sub = admin_1.get(f"{co}.{a1}", a1)
-        data.append([int(gid), name, aname, co, sub, tz, int(pop)])
+        fcodes[ct.fcode] += 1
+        sub = admin_1.get(f"{ct.co}.{ct.a1}", ct.a1)
+        data.append([int(ct.gid), ct.name, ct.aname, ct.co, sub, ct.tz, pop])
 
     for title, dct in [["KEPT", fcodes], ["SKIP", skipped]]:
         for k, v in sorted(dct.items(), key=lambda kv: kv[1], reverse=True):
             logger.debug(f"{title} {k:5}: {v}")
 
-    logger.debug(f"Processed {i} lines, kept {len(data)}")
+    logger.info(f"Processed {i:,} lines, kept {len(data):,}")
     return data
 
 
@@ -139,5 +146,6 @@ def fetch_admin_1(dirname=DB_DIR):
     else:
         txt = utils.fetch(GEONAMES_ADMIN1_URL).decode()
         filename.write_text(txt)
+        logger.info(f"Downloaded {len(txt):,} bytes from {GEONAMES_ADMIN1_URL}")
 
     return load_admin1(txt)

src/when/db/make.py

@@ -3,6 +3,8 @@
 import re
 import sys
 import time
+import logging
+from functools import cache
 from datetime import datetime, timedelta
 from pathlib import Path
 
@@ -15,6 +17,11 @@
 from .exceptions import WhenError
 
 
+@cache
+def logger():
+    return logging.getLogger("when")
+
+
 def gettz(name=None):
     tz = _gettz(name)
     if name is None:

src/when/utils.py

@@ -217,9 +217,9 @@ def test_fetch_admin_1(self, loader):
         expect.unlink(True)
 
     def test_parse_search(self, db):
-        assert db.parse_search("a") == ["a", None, None]
-        assert db.parse_search("a, b") == ["a", "b", None]
-        assert db.parse_search("a, b,c") == ["a", "b", "c"]
+        assert db.parse_search("a") == ["A", None, None]
+        assert db.parse_search("a, b") == ["A", None, "B"]
+        assert db.parse_search("a, b,c") == ["A", "B", "C"]
         with pytest.raises(exceptions.DBError, match="Invalid city search expression: a,b,c,d"):
             db.parse_search("a,b,c,d")