The Principia Machina/TPC-PY/gen biblio unicode table.py: Difference between revisions

Latest revision as of 04:32, 28 September 2026

"""gen_biblio_unicode_table 0.1.0: the key form's character classes (RD-9),
generated from Python's unicodedata at the pinned Unicode version and
written into both residences of tpc_biblio between their BEGIN and END
UNICODE TABLE markers, after the pattern of the falcon module's
gen_unicode_table.py.

Classes: M Mn (nonspacing mark), P every punctuation category (Pc Pd Ps
Pe Pi Pf Po), S every symbol category (Sm Sc Sk So), F Cf, C Cc, Z Zs Zl
Zp; every other code point has none. The table is a string of ranges
"START-END" or "CP" in upper-case hexadecimal, each followed by its class
letter, joined by ";". Both residences check its SHA-256 when they first
read it, so the classification is identical whatever unicodedata or ICU
version the interpreter carries. Run with the Python whose unicodedata
carries the pinned version (Python 3.13: Unicode 15.1.0)."""
import hashlib
import os
import unicodedata

PINNED = "15.1.0"
H = os.path.dirname(os.path.abspath(__file__))
assert unicodedata.unidata_version == PINNED, unicodedata.unidata_version
MAP = {"Mn": "M", "Cf": "F", "Cc": "C"}


def cls(cp):
    k = unicodedata.category(chr(cp))
    if k[0] in "PSZ":
        return k[0]
    return MAP.get(k, "")


runs = []
prev = ""
start = 0
for cp in range(0x110000):
    c = cls(cp)
    if c != prev:
        if prev:
            runs.append((start, cp - 1, prev))
        prev, start = c, cp
if prev:
    runs.append((start, 0x10FFFF, prev))
table = ";".join(("%X-%X%s" % r) if r[0] != r[1] else ("%X%s" % (r[0], r[2]))
                 for r in runs)
digest = hashlib.sha256(table.encode("ascii")).hexdigest()
chunks = [table[i:i + 68] for i in range(0, len(table), 68)]

py = ["# BEGIN UNICODE TABLE (gen_biblio_unicode_table.py; do not edit)",
      'UNICODE_VERSION = "%s"' % PINNED,
      'UNICODE_TABLE_SHA256 = (\n    "%s")' % digest,
      "_UTABLE = ("] + ['    "%s"' % c for c in chunks] + [")",
                                                             "# END UNICODE TABLE"]
php = ["// BEGIN UNICODE TABLE (gen_biblio_unicode_table.py; do not edit)",
       "const TPC_BIBLIO_UNICODE_VERSION = '%s';" % PINNED,
       "const TPC_BIBLIO_UNICODE_TABLE_SHA256 = '%s';" % digest,
       "const TPC_BIBLIO_UTABLE = ''"] + ["    . '%s'" % c for c in chunks] + \
      ["    ;", "// END UNICODE TABLE"]


def put(path, begin, end, lines):
    s = open(path, encoding="utf-8", newline="").read()
    a = s.index(begin)
    b = s.index(end) + len(end)
    open(path, "w", encoding="utf-8", newline="").write(
        s[:a] + "\n".join(lines) + s[b:])


put(os.path.join(H, "tpc_biblio.py"), "# BEGIN UNICODE TABLE",
    "# END UNICODE TABLE", py)
put(os.path.join(H, "..", "tpc_php", "tpc_biblio.php"),
    "// BEGIN UNICODE TABLE", "// END UNICODE TABLE", php)
print("Unicode %s: %d ranges, %d characters, SHA-256 %s"
      % (PINNED, len(runs), len(table), digest))