From The Invariance Itself
"""gen_hunk_wiki_corpus 0.1.0: builds vectors/hunk_wiki_corpus.json, the
wiki-base hunk vectors for tpc_hunk 0.1.0 (HUNK-VEC; record 0.18.0 section
7: "the 2026-09-25 batch's 48 sets, from the wiki's revision texts").
Input: hunk_inputs/wiki_batch_2026-09-25.json beside this file, the 48 hunk
sets the wiki congruence batch of 2026-09-25 applied (revisions 3221-3269,
less 3267, the one page it created): for each page, the base revision's
text as the wiki served it, the base SHA-1 the batch asserted, the hunks as
applied, the revision saved and its SHA-1 read back.
Each case is one set: args {base: the revision text, hunks: [{old, new}]},
expected: tpc_hunk's render of hunk_apply. The expectation is MEASURED, not
hand-derived: the generator refuses to write the corpus unless, for every
set, (1) the base's SHA-1 equals the SHA-1 the batch asserted for the base
revision; (2) hunk_apply answers OK; (3) the SHA-1 of the result, with the
trailing white space the wiki's save removes (PHP rtrim: space, tab, LF,
CR, NUL, vertical tab) removed, equals the SHA-1 read back from the landed
revision. The render then pins the result byte for byte in both
residences. The corpus is ASCII (every non-ASCII character escaped).
Run: cd tpc && python3 gen_hunk_wiki_corpus.py (Windows: py ...)."""
import hashlib
import json
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from tpc_hunk import hunk_apply, run_case # noqa: E402
SRC = os.path.join(HERE, "hunk_inputs", "wiki_batch_2026-09-25.json")
SAVE_TRIM = " \t\n\r\x00\x0b"
def sha1(text):
return hashlib.sha1(text.encode("utf-8")).hexdigest()
doc = json.load(open(SRC, encoding="utf-8"))
out, bad = [], 0
for i, s in enumerate(doc["sets"], 1):
base = s["base"]
hunks = [{"old": o, "new": n} for o, n in s["hunks"]]
ok = sha1(base) == s["base_sha1"]
r = hunk_apply(base, hunks)
ok = ok and r["status"] == "OK" and sha1(r["text"].rstrip(SAVE_TRIM)) == s["readback_sha1"]
if not ok:
bad += 1
print("MISMATCH", s["title"], r.get("status"), r.get("reason"))
continue
args = {"base": base, "hunks": hunks}
name = ("W%02d %s: base revision %d (SHA-1 %s), %d hunk%s; landed as revision %d, "
"read-back SHA-1 %s (MEASURED)" % (
i, s["title"], s["base_revid"], s["base_sha1"][:8], len(hunks),
"" if len(hunks) == 1 else "s", s["landed_revid"], s["readback_sha1"][:8]))
out.append({"name": name, "fn": "hunk", "args": args, "hand_checked": False,
"measured": {"title": s["title"], "base_revid": s["base_revid"],
"base_sha1": s["base_sha1"], "landed_revid": s["landed_revid"],
"readback_sha1": s["readback_sha1"]},
"expected": run_case("hunk", args)})
if bad or len(out) != 48:
sys.exit("refusing to write: %d mismatches, %d sets" % (bad, len(out)))
corpus = {"corpus": "hunk_wiki", "version": "0.1.0",
"note": ("The 48 wiki-base hunk sets of the 2026-09-25 batch (HUNK-VEC), answered by "
"tpc_hunk's run_case; each expectation MEASURED against the wiki's base and "
"landed revisions (gen_hunk_wiki_corpus.py)."),
"cases": out}
with open(os.path.join(HERE, "vectors", "hunk_wiki_corpus.json"), "w", encoding="utf-8",
newline="\n") as f:
f.write(json.dumps(corpus, indent=1, sort_keys=True, ensure_ascii=True) + "\n")
print("wrote", len(out), "wiki-base hunk cases; every base and result SHA-1 matches the wiki")
