# -*- coding: utf-8 -*-
"""gen_dis_corpus 0.2.0: writes vectors/dis_corpus.json, the vectors of
tpc_digest's functions for the TPS digest and the instruction set digest
(D203.1, taken 2026-09-28: tpm_post_tpcpy_pause 0.1.0, D203 Option 1 (a)
and (b); tpm_tps_digest_spec 0.1.0) and of the per-page list (D216.1,
taken 2026-09-29: tpm_post_tpcpy_wave4_pause 0.2.0, D216 Option 1 (c)):
pre_texts(wikitext) the <pre> contents of one page, in order
d_tps(version, pages) D(TPS) over the pages of the TPS tree
d_is(version, d_tps, members) D(IS) over D(TPS) and the other members
of the index's instruction-artifact class
tps_pages(pages) the ordered list [title, h] of the kept
pages: the very list d_tps digests
0.2.0 (agent TQ4, increment 13; tpc_digest 0.3.0, proposed): the
tps_pages section is appended after d_is's; the 101 cases of 0.1.0 stand
as they were, in place. Its expectations are written apart from
tpc_digest: for every d_tps case answered OK, the list read back from that
case's own hand-written preimage (parse_fields, the netstring's inverse),
over the same pages; for every d_tps refusal but "version", the same
refusal over the same pages; and families of its own (blocks joined by one
LF, no normalization, the empty list, the order), each list from literal
titles and literal block texts. The generator also checks, over every
d_tps case, that tpc_digest's d_tps equals the digest of its tps_pages
list, built by this file's own netstring.
A corpus of its own beside digest_corpus.json, which is written by hand
and stays byte-identical at its tpc_digest 0.1.0 label (sr3_scope's
precedent). BASE form: {"id", "fn", "input", "expect"}, the result being
canon() of the function's answer; digest cases also carry "preimage", the
exact bytes hashed (ASCII here), so a reader can check any of them with
printf %s <preimage> | sha256sum.
VECTORS FIRST. Every expectation below is HAND-DERIVED: refusals written
literally; each digest from a field list written literally below, the
page hashes and the netstring built by this file's own code (sha256_hex,
netstring), apart from tpc_digest. The generator refuses to write the
corpus if tpc_digest disagrees with any expectation. The page sets are
synthetic: the real TPS 0.3.0 page set is a fixture outside tpc/ (ruling
S97; dis_fixtures/ at the tree root, not published with TPC-PY).
canon() renders a string as itself, so a pre_texts answer whose blocks
hold "," or "]" would read ambiguously; the pre_texts vectors keep such
characters out of their blocks, and the d_tps vectors pin what the
blocks hash to. Deterministic; standard library only; Python 3.8+.
Run: cd tpc && python3 gen_dis_corpus.py (Windows: py -3 gen_dis_corpus.py).
"""
import hashlib
import json
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
sys.dont_write_bytecode = True
import tpc_strike as S # noqa: E402 canon()
import tpc_digest as G # noqa: E402
GEN_VERSION = "0.2.0"
R = "The Principia Machina/TPS" # the tree's root title (D34.1)
HEX_A = "a" * 64
HEX_B = "0123456789abcdef" * 4
def sha256_hex(text):
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def netstring(fields):
"""D97's encoding, built here apart from tpc_digest: each field its
UTF-8 byte length in decimal, ":", its bytes; an int in base 10."""
out = []
for f in fields:
s = str(f) if type(f) is int else f
out.append("%d:%s" % (len(s.encode("utf-8")), s))
return "".join(out)
cases = []
def case(cid, fn, inp, expect, preimage=None):
c = {"id": cid, "fn": fn, "input": inp}
if preimage is not None:
c["preimage"] = preimage
c["expect"] = expect
cases.append(c)
def ok_digest(preimage):
return "[OK,%s]" % sha256_hex(preimage)
# ------------------------------------------------------------ pre_texts
P = "pre_texts"
REF_PRE = "[REFUSED,pre]"
case("pre/one-block", P,
{"wikitext": "''banner''\n\n<pre>\nA\nB\n</pre>\n\n[[Category:X]]\n"},
"[OK,[A\nB\n]]")
case("pre/two-blocks", P,
{"wikitext": "<pre>\nX\n</pre>\nbetween\n<pre>\nY\n</pre>\n"},
"[OK,[X\n,Y\n]]")
case("pre/no-block", P, {"wikitext": "just text\n"}, "[OK,[]]")
case("pre/empty-text", P, {"wikitext": ""}, "[OK,[]]")
case("pre/empty-block (canon writes [] as for no block; d_tps pins the difference)", P,
{"wikitext": "<pre>\n</pre>"}, "[OK,[]]")
case("pre/text-after-closing-ignored", P,
{"wikitext": "<pre>\nA\n</pre>tail <b>x</b>\n"}, "[OK,[A\n]]")
case("pre/prefix-inside-is-text (TPS 0.3.0 writes <prefix>-n)", P,
{"wikitext": "<pre>\n<prefix>-n and </prefix> x\n</pre>\n"},
"[OK,[<prefix>-n and </prefix> x\n]]")
case("pre/pretty-and-preamble-outside-are-text", P,
{"wikitext": "<pretty> </preamble>\n<pre>\nA\n</pre>\n<presence/>\n"}, "[OK,[A\n]]")
case("pre/non-ascii", P,
{"wikitext": "<pre>\ncaf\u00e9 \u2014 \u00b5 \U0001f600\n</pre>\n"},
"[OK,[caf\u00e9 \u2014 \u00b5 \U0001f600\n]]")
case("pre/crlf-inside-kept-as-is", P,
{"wikitext": "<pre>\nA\r\nB\n</pre>"}, "[OK,[A\r\nB\n]]")
case("pre/opening-without-lf", P, {"wikitext": "<pre>A\n</pre>"}, REF_PRE)
case("pre/opening-then-cr", P, {"wikitext": "<pre>\r\nA\n</pre>"}, REF_PRE)
case("pre/unclosed", P, {"wikitext": "<pre>\nA\n"}, REF_PRE)
case("pre/closing-before-opening", P, {"wikitext": "</pre>\n<pre>\nA\n</pre>"}, REF_PRE)
case("pre/second-opening-inside", P, {"wikitext": "<pre>\nA\n<pre>\nB\n</pre>"}, REF_PRE)
case("pre/upper-case-tags", P, {"wikitext": "<PRE>\nA\n</PRE>"}, REF_PRE)
case("pre/mixed-case-opening", P, {"wikitext": "<Pre>\nA\n</pre>"}, REF_PRE)
case("pre/attribute", P, {"wikitext": "<pre class=\"x\">\nA\n</pre>"}, REF_PRE)
case("pre/self-closing", P, {"wikitext": "<pre />\n"}, REF_PRE)
case("pre/self-closing-no-space", P, {"wikitext": "<pre/>\n"}, REF_PRE)
case("pre/tab-in-opening", P, {"wikitext": "<pre\t>\nA\n</pre>"}, REF_PRE)
case("pre/space-in-closing", P, {"wikitext": "<pre>\nA\n</pre >"}, REF_PRE)
case("pre/upper-case-closing", P, {"wikitext": "<pre>\nA\n</PRE>"}, REF_PRE)
case("pre/tag-name-at-end-of-text", P, {"wikitext": "x <pre"}, REF_PRE)
case("pre/closing-name-at-end-of-text", P, {"wikitext": "<pre>\nA\n</pre"}, REF_PRE)
case("pre/tag-in-comment-is-read-as-tag (named limit)", P,
{"wikitext": "<!-- <pre> -->\nx\n"}, REF_PRE)
case("pre/not-a-string", P, {"wikitext": 5}, REF_PRE)
case("pre/null", P, {"wikitext": None}, REF_PRE)
# ------------------------------------------------------------ d_tps
T = "d_tps"
ROOT_TEXT = "''root banner''\n<pre>\nROOT ONE\n</pre>\nmiddle\n<pre>\nROOT TWO\n</pre>\n"
ROOT_TEXT_B = "''another banner, [[a link]]''\n\n<pre>\nROOT ONE\n</pre>\n<pre>\nROOT TWO\n</pre>"
S01 = R + "/0.1 Front"
S59 = R + "/5.9 Nine"
S510 = R + "/5.10 Ten"
CT = R + "/Contents"
P_ROOT = [R, ROOT_TEXT]
P_01 = [S01, "<pre>\nFRONT\n</pre>"]
P_59 = [S59, "''b''\n<pre>\nSECTION 5.9\n</pre>\n"]
P_510 = [S510, "''b''\n<pre>\nSECTION 5.10\n</pre>\n"]
H_ROOT = sha256_hex("ROOT ONE\n" + "\n" + "ROOT TWO\n") # blocks joined by one LF
H_01 = sha256_hex("FRONT\n")
H_59 = sha256_hex("SECTION 5.9\n")
H_510 = sha256_hex("SECTION 5.10\n")
BASE_PAGES = [P_510, P_ROOT, P_59, P_01]
BASE_FIELDS = ["TPC-TPS", "1", "0.3.0", 4, R, H_ROOT, S01, H_01, S59, H_59, S510, H_510]
BASE_PRE = netstring(BASE_FIELDS)
BASE_EXPECT = ok_digest(BASE_PRE)
D_TPS_BASE = sha256_hex(BASE_PRE)
case("tps/root-and-sections-out-of-order: 0.1 < 5.9 < 5.10 as integers", T,
{"version": "0.3.0", "pages": BASE_PAGES}, BASE_EXPECT, BASE_PRE)
case("tps/input-order-reversed: equal", T,
{"version": "0.3.0", "pages": list(reversed(BASE_PAGES))}, BASE_EXPECT, BASE_PRE)
case("tps/banners-changed: equal", T,
{"version": "0.3.0", "pages": [[S510, "<pre>\nSECTION 5.10\n</pre>"], [R, ROOT_TEXT_B],
[S59, "new banner\n<pre>\nSECTION 5.9\n</pre>\n\n[[Category:Y]]"],
[S01, "x\n<pre>\nFRONT\n</pre>\ny"]]},
BASE_EXPECT, BASE_PRE)
case("tps/contents-without-block: left out, equal", T,
{"version": "0.3.0", "pages": BASE_PAGES + [[CT, "== Parts ==\n* a list\n"]]},
BASE_EXPECT, BASE_PRE)
case("tps/contents-with-a-block: left out by title, equal", T,
{"version": "0.3.0", "pages": BASE_PAGES + [[CT, "<pre>\nNOT RULE TEXT\n</pre>\n"]]},
BASE_EXPECT, BASE_PRE)
case("tps/section-without-block: left out, equal", T,
{"version": "0.3.0", "pages": BASE_PAGES + [[R + "/6.1 Empty", "no block here\n"]]},
BASE_EXPECT, BASE_PRE)
f = ["TPC-TPS", "1", "0.3.0", 4, R, H_ROOT, S01, H_01, S59, sha256_hex("SECTION 5.8\n"), S510, H_510]
case("tps/one-character-inside-a-block-changed: moves", T,
{"version": "0.3.0", "pages": [P_510, P_ROOT, [S59, "''b''\n<pre>\nSECTION 5.8\n</pre>\n"], P_01]},
ok_digest(netstring(f)), netstring(f))
f = ["TPC-TPS", "1", "0.3.0", 4, R, H_ROOT, S01, H_01, S59, H_510, S510, H_59]
case("tps/two-pages-contents-swapped-between-titles: moves", T,
{"version": "0.3.0", "pages": [[S510, P_59[1]], P_ROOT, [S59, P_510[1]], P_01]},
ok_digest(netstring(f)), netstring(f))
f = ["TPC-TPS", "1", "0.3.0", 5, R, H_ROOT, S01, H_01, S59, H_59, S510, H_510,
R + "/6.1 Empty", sha256_hex("")]
case("tps/section-with-an-empty-block: kept, n = 5", T,
{"version": "0.3.0", "pages": BASE_PAGES + [[R + "/6.1 Empty", "<pre>\n</pre>\n"]]},
ok_digest(netstring(f)), netstring(f))
f = ["TPC-TPS", "1", "0.3.1", 4, R, H_ROOT, S01, H_01, S59, H_59, S510, H_510]
case("tps/version-inside: 0.3.1 over the same pages moves", T,
{"version": "0.3.1", "pages": BASE_PAGES}, ok_digest(netstring(f)), netstring(f))
f = ["TPC-TPS", "1", "10.20.30", 4, R, H_ROOT, S01, H_01, S59, H_59, S510, H_510]
case("tps/version-with-several-digits", T,
{"version": "10.20.30", "pages": BASE_PAGES}, ok_digest(netstring(f)), netstring(f))
A51, A512, A52 = R + "/5.1 A", R + "/5.1.2 B", R + "/5.2 C"
f = ["TPC-TPS", "1", "0.4.0", 4, R, H_ROOT, A51, sha256_hex("a\n"), A512, sha256_hex("b\n"),
A52, sha256_hex("c\n")]
case("tps/three-part-numbers: 5.1 < 5.1.2 < 5.2", T,
{"version": "0.4.0", "pages": [[A52, "<pre>\nc\n</pre>"], [A512, "<pre>\nb\n</pre>"], P_ROOT,
[A51, "<pre>\na\n</pre>"]]},
ok_digest(netstring(f)), netstring(f))
f = ["TPC-TPS", "1", "0.3.0", 1, R, H_ROOT]
case("tps/root-only: n = 1", T, {"version": "0.3.0", "pages": [P_ROOT]},
ok_digest(netstring(f)), netstring(f))
f = ["TPC-TPS", "1", "0.3.0", 1, S59, H_59]
case("tps/root-without-block: left out, n = 1", T,
{"version": "0.3.0", "pages": [[R, "no block\n"], P_59]}, ok_digest(netstring(f)), netstring(f))
f = ["TPC-TPS", "1", "0.3.0", 2, R, H_ROOT, R + "/9999.9999 Last", sha256_hex("z\n")]
case("tps/four-digit-parts", T,
{"version": "0.3.0", "pages": [[R + "/9999.9999 Last", "<pre>\nz\n</pre>"], P_ROOT]},
ok_digest(netstring(f)), netstring(f))
f = ["TPC-TPS", "1", "0.3.0", 2, R, H_ROOT, R + "/0.0 Zero", sha256_hex("0\n")]
case("tps/zero-parts", T,
{"version": "0.3.0", "pages": [P_ROOT, [R + "/0.0 Zero", "<pre>\n0\n</pre>"]]},
ok_digest(netstring(f)), netstring(f))
NU = R + "/2.1 Caf\u00e9 and more"
f = ["TPC-TPS", "1", "0.3.0", 2, R, H_ROOT, NU, sha256_hex("\u00e9\n")]
case("tps/non-ascii-title-and-text", T,
{"version": "0.3.0", "pages": [[NU, "<pre>\n\u00e9\n</pre>"], P_ROOT]},
ok_digest(netstring(f)), netstring(f))
# refusals, in the reasons' order: version, pages, title, root, pre
for cid, v in (("tps/version-two-parts", "0.3"), ("tps/version-prefixed", "v0.3.0"),
("tps/version-leading-zero", "0.03.0"), ("tps/version-suffix", "0.3.0-rc1"),
("tps/version-trailing-lf", "0.3.0\n"), ("tps/version-empty", "")):
case(cid, T, {"version": v, "pages": BASE_PAGES}, "[REFUSED,version]")
case("tps/version-not-a-string", T, {"version": 3, "pages": BASE_PAGES}, "[REFUSED,version]")
case("tps/version-checked-before-pages", T, {"version": "x", "pages": {}}, "[REFUSED,version]")
case("tps/pages-an-object", T, {"version": "0.3.0", "pages": {}}, "[REFUSED,pages]")
case("tps/pages-a-string", T, {"version": "0.3.0", "pages": "x"}, "[REFUSED,pages]")
case("tps/page-not-a-pair", T, {"version": "0.3.0", "pages": [[R]]}, "[REFUSED,pages]")
case("tps/page-of-three", T, {"version": "0.3.0", "pages": [[R, "", ""]]}, "[REFUSED,pages]")
case("tps/page-text-not-a-string", T, {"version": "0.3.0", "pages": [[R, 5]]}, "[REFUSED,pages]")
case("tps/page-as-an-object", T,
{"version": "0.3.0", "pages": [{"title": R, "text": ROOT_TEXT}]}, "[REFUSED,pages]")
case("tps/duplicate-title", T, {"version": "0.3.0", "pages": BASE_PAGES + [P_59]}, "[REFUSED,title]")
case("tps/duplicate-section-number", T,
{"version": "0.3.0", "pages": BASE_PAGES + [[R + "/5.9 Other", "<pre>\nq\n</pre>"]]},
"[REFUSED,title]")
case("tps/duplicate-root", T, {"version": "0.3.0", "pages": [P_ROOT, [R, "<pre>\nq\n</pre>"]]},
"[REFUSED,title]")
for cid, t in (("tps/title-outside-the-tree", "Main Page"),
("tps/title-with-underscores", "The_Principia_Machina/TPS"),
("tps/section-with-underscore", R + "/5.9_Nine"),
("tps/section-leading-zero", R + "/05.9 Nine"),
("tps/section-one-part", R + "/5 Nine"),
("tps/section-without-name", R + "/5.9"),
("tps/section-five-digit-part", R + "/5.10000 Big"),
("tps/section-subpage", R + "/5.9 Nine/sub"),
("tps/section-two-spaces", R + "/5.9 Nine"),
("tps/section-trailing-space", R + "/5.9 Nine "),
("tps/section-trailing-dot", R + "/5.9. Nine"),
("tps/contents-in-lower-case", R + "/contents"),
("tps/root-with-trailing-slash", R + "/"),
("tps/title-with-lf", R + "\n"),
("tps/section-unicode-digit", R + "/5.\u0669 Nine"),
("tps/title-empty", "")):
case(cid, T, {"version": "0.3.0", "pages": BASE_PAGES + [[t, "<pre>\nq\n</pre>"]]},
"[REFUSED,title]")
case("tps/root-absent", T, {"version": "0.3.0", "pages": [P_59, P_510]}, "[REFUSED,root]")
case("tps/no-page", T, {"version": "0.3.0", "pages": []}, "[REFUSED,root]")
case("tps/root-absent-contents-present", T, {"version": "0.3.0", "pages": [[CT, "x"], P_59]},
"[REFUSED,root]")
case("tps/a-page-pre-refused", T,
{"version": "0.3.0", "pages": BASE_PAGES + [[R + "/6.1 Bad", "<pre>\nunclosed\n"]]},
"[REFUSED,pre]")
case("tps/contents-pre-refused-too", T,
{"version": "0.3.0", "pages": BASE_PAGES + [[CT, "<PRE>x</PRE>"]]}, "[REFUSED,pre]")
case("tps/title-reason-before-pre, bad title last", T,
{"version": "0.3.0", "pages": [[R + "/6.1 Bad", "<pre>\nx\n"], P_ROOT, ["Main Page", "x"]]},
"[REFUSED,title]")
case("tps/title-reason-before-pre, bad title first", T,
{"version": "0.3.0", "pages": [["Main Page", "x"], P_ROOT, [R + "/6.1 Bad", "<pre>\nx\n"]]},
"[REFUSED,title]")
case("tps/root-reason-before-pre", T,
{"version": "0.3.0", "pages": [[R + "/6.1 Bad", "<pre>\nx\n"]]}, "[REFUSED,root]")
# ------------------------------------------------------------ d_is
I = "d_is"
f = ["TPC-DIS", "1", "0.3.0", D_TPS_BASE, 0]
case("dis/m-0 over tps/root-and-sections-out-of-order", I,
{"version": "0.3.0", "d_tps": D_TPS_BASE, "members": []}, ok_digest(netstring(f)), netstring(f))
f = ["TPC-DIS", "1", "0.3.0", D_TPS_BASE, 1, "The Principia Machina/Member A", HEX_A]
case("dis/m-1", I, {"version": "0.3.0", "d_tps": D_TPS_BASE,
"members": [["The Principia Machina/Member A", HEX_A]]},
ok_digest(netstring(f)), netstring(f))
f = ["TPC-DIS", "1", "0.3.0", D_TPS_BASE, 2, "The Principia Machina/Member A", HEX_A,
"The Principia Machina/Member B", HEX_B]
case("dis/m-2 in index order", I, {"version": "0.3.0", "d_tps": D_TPS_BASE,
"members": [["The Principia Machina/Member A", HEX_A],
["The Principia Machina/Member B", HEX_B]]},
ok_digest(netstring(f)), netstring(f))
f = ["TPC-DIS", "1", "0.3.0", D_TPS_BASE, 2, "The Principia Machina/Member B", HEX_B,
"The Principia Machina/Member A", HEX_A]
case("dis/m-2 the other order: moves (the caller's order is kept)", I,
{"version": "0.3.0", "d_tps": D_TPS_BASE,
"members": [["The Principia Machina/Member B", HEX_B],
["The Principia Machina/Member A", HEX_A]]},
ok_digest(netstring(f)), netstring(f))
f = ["TPC-DIS", "1", "0.4.0", D_TPS_BASE, 0]
case("dis/version-inside", I, {"version": "0.4.0", "d_tps": D_TPS_BASE, "members": []},
ok_digest(netstring(f)), netstring(f))
case("dis/version-bad", I, {"version": "0.4", "d_tps": D_TPS_BASE, "members": []},
"[REFUSED,version]")
case("dis/d_tps-upper-case", I, {"version": "0.3.0", "d_tps": D_TPS_BASE.upper(), "members": []},
"[REFUSED,digest]")
case("dis/d_tps-63-characters", I, {"version": "0.3.0", "d_tps": D_TPS_BASE[:63], "members": []},
"[REFUSED,digest]")
case("dis/d_tps-not-a-string", I, {"version": "0.3.0", "d_tps": 0, "members": []},
"[REFUSED,digest]")
case("dis/members-an-object", I, {"version": "0.3.0", "d_tps": D_TPS_BASE, "members": {}},
"[REFUSED,members]")
case("dis/member-not-a-pair", I, {"version": "0.3.0", "d_tps": D_TPS_BASE, "members": [[HEX_A]]},
"[REFUSED,members]")
case("dis/member-hash-upper-case", I,
{"version": "0.3.0", "d_tps": D_TPS_BASE, "members": [["M", HEX_A.upper()]]},
"[REFUSED,members]")
case("dis/member-title-empty", I, {"version": "0.3.0", "d_tps": D_TPS_BASE, "members": [["", HEX_A]]},
"[REFUSED,members]")
case("dis/member-title-not-a-string", I,
{"version": "0.3.0", "d_tps": D_TPS_BASE, "members": [[7, HEX_A]]}, "[REFUSED,members]")
case("dis/member-titles-duplicate", I,
{"version": "0.3.0", "d_tps": D_TPS_BASE, "members": [["M", HEX_A], ["M", HEX_B]]},
"[REFUSED,members]")
# ------------------------------------------------------------ tps_pages
# 0.2.0 (D216.1): tpc_digest 0.3.0's tps_pages(pages) -> ["OK", [[title, h],
# ...]] or d_tps's refusal less "version": the ordered list of the kept
# pages, the very list d_tps digests (tps_pages_definition 0.1.0).
TP = "tps_pages"
def parse_fields(pre):
"""The field list of a netstring preimage (the inverse of netstring):
each length is a byte length, so the preimage is walked as UTF-8."""
b, out, i = pre.encode("utf-8"), [], 0
while i < len(b):
j = b.index(b":", i)
n = int(b[i:j].decode("ascii"))
out.append(b[j + 1:j + 1 + n].decode("utf-8"))
i = j + 1 + n
return out
def list_expect(pairs):
"""canon() of ["OK", [[title, h], ...]], written out by hand. canon
writes a string as itself, so no title here holds "," "[" or "]"
(tpm_tps_digest_spec 0.1.0, 7.5)."""
for title, _ in pairs:
assert "," not in title and "[" not in title and "]" not in title, title
return "[OK,[" + ",".join("[%s,%s]" % (title, h) for title, h in pairs) + "]]"
BASE_LIST = [[R, H_ROOT], [S01, H_01], [S59, H_59], [S510, H_510]]
# (a) the identity with d_tps, case by case: the list of every OK case read
# back from its preimage; every refusal but "version" the same refusal
for c in [c for c in cases if c["fn"] == T]:
pages = c["input"]["pages"]
if c["expect"].startswith("[OK,"):
fields = parse_fields(c["preimage"])
assert fields[:2] == ["TPC-TPS", "1"] and c["expect"] == ok_digest(c["preimage"])
n = int(fields[3])
assert len(fields) == 4 + 2 * n
pairs = [[fields[4 + 2 * k], fields[5 + 2 * k]] for k in range(n)]
case("pages/as " + c["id"], TP, {"pages": pages}, list_expect(pairs))
elif c["expect"] != "[REFUSED,version]":
case("pages/as " + c["id"], TP, {"pages": pages}, c["expect"])
elif pages == BASE_PAGES:
case("pages/as " + c["id"] + " (tps_pages takes no version)", TP, {"pages": pages},
list_expect(BASE_LIST))
else:
assert pages == {}, c["id"]
case("pages/as " + c["id"] + " (tps_pages takes no version)", TP, {"pages": pages},
"[REFUSED,pages]")
assert list_expect(BASE_LIST) == "[OK,[[%s,%s],[%s,%s],[%s,%s],[%s,%s]]]" % (
R, H_ROOT, S01, H_01, S59, H_59, S510, H_510)
# (b) h: the blocks joined by one LF, as UTF-8, nothing normalized
PR = [R, "<pre>\nR\n</pre>\n"]
H_PR = sha256_hex("R\n")
case("pages/three-blocks-joined-by-one-lf", TP,
{"pages": [[R, "<pre>\nA\n</pre>\n<pre>\nB\n</pre>x<pre>\nC\n</pre>"]]},
list_expect([[R, sha256_hex("A\n" + "\n" + "B\n" + "\n" + "C\n")]]))
case("pages/block-without-final-lf: none added", TP,
{"pages": [[R, "<pre>\nA</pre>"]]}, list_expect([[R, sha256_hex("A")]]))
case("pages/first-block-without-final-lf: one lf between", TP,
{"pages": [[R, "<pre>\nA</pre><pre>\nB\n</pre>"]]},
list_expect([[R, sha256_hex("A" + "\n" + "B\n")]]))
case("pages/cr-lf-inside-a-block-kept: no line-end conversion", TP,
{"pages": [[R, "<pre>\nA\r\nB\n</pre>\r\n"]]}, list_expect([[R, sha256_hex("A\r\nB\n")]]))
case("pages/lone-cr-inside-a-block-kept", TP,
{"pages": [[R, "<pre>\nA\rB\n</pre>"]]}, list_expect([[R, sha256_hex("A\rB\n")]]))
D11 = R + "/1.1 Decomposed"
case("pages/decomposed-text-kept: no Unicode normalization", TP,
{"pages": [PR, [D11, "<pre>\ne\u0301\n</pre>"]]},
list_expect([[R, H_PR], [D11, sha256_hex("e\u0301\n")]]))
case("pages/precomposed-text: another h", TP,
{"pages": [PR, [D11, "<pre>\n\u00e9\n</pre>"]]},
list_expect([[R, H_PR], [D11, sha256_hex("\u00e9\n")]]))
case("pages/trailing-spaces-and-tab-kept", TP,
{"pages": [[R, "<pre>\nA \t\n \n</pre>"]]}, list_expect([[R, sha256_hex("A \t\n \n")]]))
case("pages/astral-and-non-ascii-text", TP,
{"pages": [[R, "<pre>\n\u00b5 \u2014 \U0001f600\n</pre>"]]},
list_expect([[R, sha256_hex("\u00b5 \u2014 \U0001f600\n")]]))
case("pages/text-outside-the-blocks-not-read", TP,
{"pages": [[R, "== x ==\n<pre>\nR\n</pre>\n[[Category:TPS]] {{DISPLAYTITLE:y}}\n"]]},
list_expect([[R, H_PR]]))
case("pages/empty-block-on-the-root: kept, h of the empty text", TP,
{"pages": [[R, "<pre>\n</pre>"]]}, list_expect([[R, sha256_hex("")]]))
case("pages/two-empty-blocks: h of one lf", TP,
{"pages": [[R, "<pre>\n</pre><pre>\n</pre>"]]}, list_expect([[R, sha256_hex("\n")]]))
# (c) the kept pages and the empty list
case("pages/root-only-without-block: the empty list", TP,
{"pages": [[R, "no block\n"]]}, "[OK,[]]")
case("pages/root-without-block, contents with a block: the empty list", TP,
{"pages": [[CT, "<pre>\nq\n</pre>"], [R, ""]]}, "[OK,[]]")
case("pages/root-without-block, a section kept", TP,
{"pages": [[R + "/7.1 Kept", "<pre>\nk\n</pre>"], [R, "none"], [R + "/7.2 Left", "none"]]},
list_expect([[R + "/7.1 Kept", sha256_hex("k\n")]]))
# (d) the order: the root, then by number as integers part by part
ORD_GIVEN = [("10.1", "Ten one"), ("2.1", "Two one"), ("1.10", "One ten"), ("0.1", "Front"),
("1.2", "One two"), ("1.1", "One one"), ("1.1.1", "Deep")]
ORD_WANT = [("0.1", "Front"), ("1.1", "One one"), ("1.1.1", "Deep"), ("1.2", "One two"),
("1.10", "One ten"), ("2.1", "Two one"), ("10.1", "Ten one")]
case("pages/order: 0.1 1.1 1.1.1 1.2 1.10 2.1 10.1, given scrambled, root last", TP,
{"pages": [[R + "/" + num + " " + name, "<pre>\n" + num + "\n</pre>"] for num, name in ORD_GIVEN]
+ [PR]},
list_expect([[R, H_PR]] + [[R + "/" + num + " " + name, sha256_hex(num + "\n")]
for num, name in ORD_WANT]))
# (e) refusals of its own: "pages" before "title"
for cid, pg in (("pages/pages-an-object", {}), ("pages/pages-a-string", "x"),
("pages/pages-null", None), ("pages/pages-a-number", 5),
("pages/page-not-a-pair", [[R]]), ("pages/page-of-three", [[R, "", ""]]),
("pages/page-text-not-a-string", [[R, 5]]), ("pages/page-text-null", [[R, None]]),
("pages/page-title-not-a-string", [[5, "x"]]),
("pages/page-as-an-object", [{"title": R, "text": "x"}]),
("pages/pages-reason-before-title: a bad title then a bad pair", [["Main Page", "x"], [R]])):
case(cid, TP, {"pages": pg}, "[REFUSED,pages]")
case("pages/root-pre-refused: an opening inside a block", TP,
{"pages": [[R, "<pre>\nA\n<pre>\nB\n</pre>"]]}, "[REFUSED,pre]")
case("pages/pre-refused on a page given before the root", TP,
{"pages": [[R + "/6.2 Open", "<pre>\nx"], PR]}, "[REFUSED,pre]")
case("pages/pre-refused on a page with no block: a tag of another case", TP,
{"pages": [PR, [R + "/6.3 Tag", "text <PRE> text"]]}, "[REFUSED,pre]")
# spot check of the hand netstring against a preimage written out in full
assert netstring(["TPC-DIS", "1", "0.3.0", HEX_A, 0]) == (
"7:TPC-DIS1:15:0.3.064:" + HEX_A + "1:0")
def dispatch(fn, inp):
if fn == "pre_texts":
return G.pre_texts(inp["wikitext"])
if fn == "d_tps":
return G.d_tps(inp["version"], inp["pages"])
if fn == "d_is":
return G.d_is(inp["version"], inp["d_tps"], inp["members"])
if fn == "tps_pages":
return G.tps_pages(inp["pages"])
raise KeyError(fn)
def main():
bad = 0
ids = [c["id"] for c in cases]
assert len(set(ids)) == len(ids), "duplicate ids"
for c in cases:
got = S.canon(dispatch(c["fn"], c["input"]))
if got != c["expect"]:
bad += 1
print("MISMATCH %s\n expect %r\n got %r" % (c["id"], c["expect"], got))
# the identity, over every d_tps case: d_tps equals the digest of the
# tps_pages list, the digest built here by this file's netstring
for c in cases:
if c["fn"] != T:
continue
d = G.d_tps(c["input"]["version"], c["input"]["pages"])
p = G.tps_pages(c["input"]["pages"])
if d[0] == "OK":
f = ["TPC-TPS", "1", c["input"]["version"], len(p[1])] + [x for pr in p[1] for x in pr]
same = p[0] == "OK" and d[1] == sha256_hex(netstring(f))
else:
same = d == ["REFUSED", "version"] or d == p
if not same:
bad += 1
print("IDENTITY %s\n d_tps %r\n tps_pages %r" % (c["id"], d, p))
if bad:
print("refusing to write: %d of %d cases disagree with their hand-derived expectations"
% (bad, len(cases)))
return 1
corpus = {
"meta": {
"name": "tpc_dis_corpus", "version": "0.2.0",
"status": "PROPOSED 2026-09-30 (D216.1, taken 2026-09-29; tpm_post_tpcpy_wave4_pause 0.2.0 D216 "
"Option 1 (c)): the tps_pages cases; the cases of 0.1.0, declared with TPC-PY 0.2.0 on "
"2026-09-30, unchanged; not declared",
"conforms_tps": "0.5.0 (proposed; revision 8 states D216.1)",
"module": "tpc_digest 0.3.0 (proposed with TPC-PY 0.3.0)",
"spec": "tpm_tps_digest_spec 0.1.0; tps_pages_definition 0.1.0 (D216.1)",
"note": ("Every expectation is hand-derived: refusals written literally, each digest from a field "
"list written in gen_dis_corpus.py and hashed apart from tpc_digest; 'preimage' is the "
"exact bytes hashed; each tps_pages list read back from a d_tps case's preimage or "
"written from literal titles and block texts. The page sets are synthetic; the real "
"TPS 0.3.0 and 0.4.0 page sets are fixtures outside tpc/ (S97). Both residences MUST "
"reproduce every expect.")
},
"cases": cases,
}
with open(os.path.join(HERE, "vectors", "dis_corpus.json"), "w", encoding="utf-8",
newline="\n") as fh:
fh.write(json.dumps(corpus, indent=1, ensure_ascii=True) + "\n")
print("wrote", len(cases), "dis cases; all match hand-derived expectations")
return 0
if __name__ == "__main__":
sys.exit(main())
# END OF ARTIFACT --- gen_dis_corpus 0.2.0
# If this line is absent, the payload arrived truncated.
