The Principia Machina/TPC-PY/gen unicode table.py: Difference between revisions

Latest revision as of 04:32, 28 September 2026

"""gen_unicode_table 0.1.0: the TOKEN RULE's character classes, generated
from Python's unicodedata at the pinned Unicode version and written into
both residences between their BEGIN and END UNICODE TABLE markers.

Classes: L letter (Lu Ll Lt Lm Lo), M mark (Mn Mc Me), D Nd, N Nl or No,
P Pd, S Sm, C Sc, Z Zs, F Cf; every other code point has none. The table
is a string of ranges "START-END" or "CP" in upper-case hexadecimal, each
followed by its class letter, joined by ";". Both residences check its
SHA-256 when they first read it. Run with the Python whose unicodedata
carries the pinned version (Python 3.13: Unicode 15.1.0)."""
import hashlib
import os
import unicodedata

PINNED = "15.1.0"
H = os.path.dirname(os.path.abspath(__file__))
assert unicodedata.unidata_version == PINNED, unicodedata.unidata_version
MAP = {"Pd": "P", "Sm": "S", "Sc": "C", "Zs": "Z", "Cf": "F", "Nd": "D",
       "Nl": "N", "No": "N"}


def cls(cp):
    k = unicodedata.category(chr(cp))
    if k[0] in "LM":
        return k[0]
    return MAP.get(k, "")


runs = []
prev = ""
start = 0
for cp in range(0x110000):
    c = cls(cp)
    if c != prev:
        if prev:
            runs.append((start, cp - 1, prev))
        prev, start = c, cp
if prev:
    runs.append((start, 0x10FFFF, prev))
table = ";".join(("%X-%X%s" % r) if r[0] != r[1] else ("%X%s" % (r[0], r[2]))
                 for r in runs)
digest = hashlib.sha256(table.encode("ascii")).hexdigest()
chunks = [table[i:i + 68] for i in range(0, len(table), 68)]

py = ["# BEGIN UNICODE TABLE (gen_unicode_table.py; do not edit)",
      'UNICODE_VERSION = "%s"' % PINNED,
      'UNICODE_TABLE_SHA256 = (\n    "%s")' % digest,
      "_UTABLE = ("] + ['    "%s"' % c for c in chunks] + [")",
                                                             "# END UNICODE TABLE"]
php = ["// BEGIN UNICODE TABLE (gen_unicode_table.py; do not edit)",
       "const TPC_FALCON_UNICODE_VERSION = '%s';" % PINNED,
       "const TPC_FALCON_UNICODE_TABLE_SHA256 = '%s';" % digest,
       "const TPC_FALCON_UTABLE = ''"] + ["    . '%s'" % c for c in chunks] + \
      ["    ;", "// END UNICODE TABLE"]


def put(path, begin, end, lines):
    s = open(path, encoding="utf-8", newline="").read()
    a = s.index(begin)
    b = s.index(end) + len(end)
    open(path, "w", encoding="utf-8", newline="").write(
        s[:a] + "\n".join(lines) + s[b:])


put(os.path.join(H, "tpc_falcon.py"), "# BEGIN UNICODE TABLE",
    "# END UNICODE TABLE", py)
put(os.path.join(H, "..", "tpc_php", "tpc_falcon.php"),
    "// BEGIN UNICODE TABLE", "// END UNICODE TABLE", php)
print("Unicode %s: %d ranges, %d characters, SHA-256 %s"
      % (PINNED, len(runs), len(table), digest))