The Principia Machina/TPC-PY/gen identity corpus.py: Difference between revisions

Latest revision as of 02:11, 30 September 2026

"""gen_identity_corpus 0.1.0: builds identity_corpus.json for tpc_identity
0.1.0. Expectations are computed by tpc_identity.py and pinned; hand_
checked cases were worked by hand (D01's digest is also recomputed here
with hashlib over the D97 netstring, independently of the module)."""
import hashlib
import json
import os
import sys

here = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, here)
from tpc_identity import run_case  # noqa: E402

cases = []


def add(name, fn, args, hand=False):
    cases.append({"name": name, "fn": fn, "args": args, "hand_checked": hand,
                  "expected": run_case(fn, args)})


# ---- canonical_tools -------------------------------------------------------
TOOLS = '[{"name": "read_page", "description": "Read a wiki page", "parameters": {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}}, {"name": "push", "parameters": {"type": "object", "properties": {"records": {"type": "array", "items": {"type": "object"}}}}}]'
add("T01 keys sorted, whitespace removed", "tools", {"json": '{ "b" : 1 , "a" : 2 }'}, True)
add("T02 nested objects and arrays", "tools", {"json": '{"z": {"y": [3, {"x": null}]}, "a": []}'}, True)
add("T03 NFC: a decomposed string equals its composed form", "tools", {"json": '["e\\u0301", "\\u00e9"]'}, True)
add("T04 decimals: 1.0, 1, 1e0, 1.50, 0.001e-2, -0, -0.0, 12.340E+3", "tools", {"json": '[1.0, 1, 1e0, 1.50, 0.001e-2, -0, -0.0, 12.340E+3]'}, True)
add("T05 a realistic tool definition list", "tools", {"json": TOOLS}, False)
add("T06 the same list with other whitespace and key order canonicalizes alike", "tools", {"json": '[{"parameters":{"required":["title"],"properties":{"title":{"type":"string"}},"type":"object"},"description":"Read a wiki page","name":"read_page"},{"parameters":{"type":"object","properties":{"records":{"items":{"type":"object"},"type":"array"}}},"name":"push"}]'}, False)
add("T07 a trailing comma is refused", "tools", {"json": '[1, 2,]'}, True)
add("T08 a duplicate key is refused", "tools", {"json": '{"a": 1, "a": 2}'}, True)
add("T09 a duplicate key after NFC is refused", "tools", {"json": '{"\\u00e9": 1, "e\\u0301": 2}'}, True)
add("T10 NaN is refused", "tools", {"json": '[NaN]'}, True)
add("T11 a lone surrogate escape is refused", "tools", {"json": '"\\ud800"'}, True)
add("T12 a surrogate pair escape gives the astral character raw", "tools", {"json": '"\\ud83d\\ude00"'}, True)
add("T13 a raw control character in a string is refused", "tools", {"json": '"a\tb"'}, True)
add("T14 escapes: \\b \\f \\n \\r \\t \\/ \\\" \\\\ \\u0001 \\u007f", "tools", {"json": '"\\b\\f\\n\\r\\t\\/\\"\\\\\\u0001\\u007f"'}, True)
add("T15 an exponent of five digits is refused", "tools", {"json": '1e12345'}, True)
add("T16 an exponent of four digits is kept", "tools", {"json": '1e1234'}, True)
add("T17 depth 65 is refused", "tools", {"json": "[" * 65 + "1" + "]" * 65}, True)
add("T18 depth 64 is kept", "tools", {"json": "[" * 64 + "1" + "]" * 64}, True)
add("T19 the four whitespace characters around tokens", "tools", {"json": ' \t\r\n{ \t"a"\r\n:\t[ 1 ,\n2 ] }\n'}, True)
add("T20 empty input is refused", "tools", {"json": ''}, True)
add("T21 trailing text is refused", "tools", {"json": '[1] x'}, True)
add("T22 a leading zero is refused", "tools", {"json": '01'}, True)
add("T23 top-level string, number and literals", "tools", {"json": '["x", -0.5e1, true, false, null]'}, True)
add("T24 key order by code point: Z, a, e-acute, CJK, and a key with an escape", "tools", {"json": '{"\\u4e2d": 1, "\\u00e9": 2, "a": 3, "Z": 4, "a\\nb": 5, "": 6}'}, True)
add("T25 not a string is refused", "tools", {"json": 12}, True)
add("T26 a number with a fraction but no digits after the point is refused", "tools", {"json": '1.'}, True)
add("T27 an unescaped DEL and non-ASCII raw text are kept", "tools", {"json": '"\u007f\u6a21\u578b"'}, True)
add("T28 a large integer stays exact (no float rounding)", "tools", {"json": '[123456789012345678901234567890, 0.10000000000000000555]'}, True)

# ---- prompt_digest ---------------------------------------------------------
SALT = "0123456789abcdef0123456789abcdef"
FAM = "6:CLOSED16:Anthropic/Claude"
TMPL = "You are a certified agent of The Principia Machina.\nWork the register."
args = {"family": FAM, "salt": SALT, "template": TMPL, "tools": TOOLS}
add("D01 a digest; recomputed independently below", "digest", args, True)
canon_tools = run_case("tools", {"json": TOOLS})
fields = ["TPM-PROMPT", "1", FAM, SALT, TMPL, canon_tools]
enc = b"".join(b"%d:%s" % (len(f.encode("utf-8")), f.encode("utf-8")) for f in fields)
assert hashlib.sha256(enc).hexdigest() == cases[-1]["expected"]
add("D02 the template in NFD gives the same digest as in NFC", "digest", {"family": FAM, "salt": SALT, "template": "Re\u0301sume\u0301 the work.", "tools": "[]"}, True)
add("D02b the same template in NFC", "digest", {"family": FAM, "salt": SALT, "template": "R\u00e9sum\u00e9 the work.", "tools": "[]"}, True)
add("D03 tool key order and whitespace do not change the digest", "digest", {"family": FAM, "salt": SALT, "template": TMPL, "tools": '[{"parameters":{"required":["title"],"properties":{"title":{"type":"string"}},"type":"object"},"description":"Read a wiki page","name":"read_page"},{"parameters":{"type":"object","properties":{"records":{"items":{"type":"object"},"type":"array"}}},"name":"push"}]'}, True)
add("D04 one changed character in the template changes the digest", "digest", {"family": FAM, "salt": SALT, "template": TMPL + ".", "tools": TOOLS}, True)
add("D05 one changed character in the tools changes the digest", "digest", {"family": FAM, "salt": SALT, "template": TMPL, "tools": TOOLS.replace("push", "Push")}, True)
add("D06 a salt in upper case is refused", "digest", {"family": FAM, "salt": SALT.upper(), "template": TMPL, "tools": TOOLS}, True)
add("D07 a salt of 31 characters is refused", "digest", {"family": FAM, "salt": SALT[:31], "template": TMPL, "tools": TOOLS}, True)
add("D08 a salt of 33 characters is refused", "digest", {"family": FAM, "salt": SALT + "0", "template": TMPL, "tools": TOOLS}, True)
add("D09 a salt with a non-hex character is refused", "digest", {"family": FAM, "salt": SALT[:-1] + "g", "template": TMPL, "tools": TOOLS}, True)
add("D10 a salt that is not a string is refused", "digest", {"family": FAM, "salt": 123, "template": TMPL, "tools": TOOLS}, True)
add("D11 an empty family is refused", "digest", {"family": "", "salt": SALT, "template": TMPL, "tools": TOOLS}, True)
add("D12 invalid tools JSON is refused", "digest", {"family": FAM, "salt": SALT, "template": TMPL, "tools": "[1,"}, True)
add("D13 an empty template is a digest", "digest", {"family": FAM, "salt": SALT, "template": "", "tools": "[]"}, True)
add("D14 a whitespace difference in the template changes the digest (no collapsing)", "digest", {"family": FAM, "salt": SALT, "template": TMPL.replace("\n", "  \n"), "tools": TOOLS}, True)
add("D15 a template that is not a string is refused", "digest", {"family": FAM, "salt": SALT, "template": None, "tools": TOOLS}, True)
add("D16 another salt gives another digest", "digest", {"family": FAM, "salt": "f" * 32, "template": TMPL, "tools": TOOLS}, True)
add("D17 a CJK family and template", "digest", {"family": "6:CLOSED12:\u6a21\u578b/\u4e59", "salt": SALT, "template": "\u4f60\u597d", "tools": "{}"}, True)
add("D18 another family gives another digest", "digest", {"family": "6:CLOSED13:Google/Gemini", "salt": SALT, "template": TMPL, "tools": TOOLS}, True)

# ---- family_of -------------------------------------------------------------
def row(name, family, open_weights=False, server=None, quant=None):
    return {"name": name, "family": family, "open": open_weights, "server": server, "quantization": quant}


MAP = [
    row("claude-fable-5-1-20260601", "Anthropic/Claude"),
    row("claude-fable-5-1", "Anthropic/Claude"),
    row("anthropic.claude-fable-5-1-v1:0", "Anthropic/Claude"),
    row("claude-opus-4-20250514", "Anthropic/Claude"),
    row("gemini-3-pro", "Google/Gemini"),
    row("together/llama-4-70b-fp8", "Meta/Llama-4", True, "together", "fp8"),
    row("fireworks/llama-4-70b-bf16", "Meta/Llama-4", True, "fireworks", "bf16"),
    row("mod\u00e8le-\u00e9", "Maison/Mod\u00e8le"),
]
add("F01 a dated snapshot", "family", {"model": "claude-fable-5-1-20260601", "map": MAP}, True)
add("F02 a dateless alias", "family", {"model": "claude-fable-5-1", "map": MAP}, True)
add("F03 a server-prefixed spelling of a closed model stays in the family (D156.1 (c))", "family", {"model": "anthropic.claude-fable-5-1-v1:0", "map": MAP}, True)
add("F04 open weights with a pinned server and quantization: the key carries the triple", "family", {"model": "together/llama-4-70b-fp8", "map": MAP}, True)
add("F05 the same lineage through another server is another key (D156.1 (b))", "family", {"model": "fireworks/llama-4-70b-bf16", "map": MAP}, True)
add("F06 open weights without a pinned server: no row, UNMAPPED", "family", {"model": "llama-4-70b", "map": MAP}, True)
add("F07 an unknown model is UNMAPPED", "family", {"model": "gpt-9", "map": MAP}, True)
add("F08 a case difference is UNMAPPED (exact match)", "family", {"model": "Claude-Fable-5-1", "map": MAP}, True)
add("F09 the model string in NFD matches its NFC row", "family", {"model": "mode\u0300le-e\u0301", "map": MAP}, True)
add("F10 an open row without a server is refused", "family", {"model": "x", "map": MAP + [row("x", "Y", True, None, "fp8")]}, True)
add("F11 a closed row with a server is refused", "family", {"model": "x", "map": MAP + [row("x", "Y", False, "s", None)]}, True)
add("F12 a duplicate name is refused", "family", {"model": "x", "map": MAP + [row("gemini-3-pro", "Google/Gemini")]}, True)
add("F13 a duplicate name after NFC is refused", "family", {"model": "x", "map": MAP + [row("mode\u0300le-e\u0301", "Maison/Mod\u00e8le")]}, True)
add("F14 an empty name is refused", "family", {"model": "x", "map": MAP + [row("", "Y")]}, True)
add("F15 open not a boolean is refused", "family", {"model": "x", "map": MAP + [{"name": "x", "family": "Y", "open": 1, "server": None, "quantization": None}]}, True)
add("F16 an empty model string is refused", "family", {"model": "", "map": MAP}, True)
add("F17 a model that is not a string is refused", "family", {"model": 5, "map": MAP}, True)
add("F18 a map that is not a list is refused", "family", {"model": "x", "map": {"name": "x"}}, True)
add("F19 an extra key in a row is refused", "family", {"model": "x", "map": [{"name": "x", "family": "Y", "open": False, "server": None, "quantization": None, "note": ""}]}, True)
add("F20 an empty map: UNMAPPED", "family", {"model": "x", "map": []}, True)
add("F21 an empty family is refused", "family", {"model": "x", "map": [row("x", "")]}, True)

# ---- change_class ----------------------------------------------------------
CL = "6:CLOSED16:Anthropic/Claude"
GO = "6:CLOSED13:Google/Gemini"
OP1 = "4:OPEN12:Meta/Llama-48:together3:fp8"
OP2 = "4:OPEN12:Meta/Llama-49:fireworks4:bf16"
add("CL01 the same model reported: SAME, the cluster not read", "class", {"certified": CL, "last": "claude-fable-5-1", "reported": "claude-fable-5-1", "lookup": {"family": CL, "cluster": None}}, True)
add("CL02 a changed model inside the family, IN: RECORDED CHANGE", "class", {"certified": CL, "last": "claude-opus-4-20250514", "reported": "claude-fable-5-1", "lookup": {"family": CL, "cluster": "IN"}}, True)
add("CL03 a changed model inside the family, CUSP: CUSP PATH", "class", {"certified": CL, "last": "claude-opus-4-20250514", "reported": "claude-fable-5-1", "lookup": {"family": CL, "cluster": "CUSP"}}, True)
add("CL04 a changed model, OUT: REFUSED MODEL", "class", {"certified": CL, "last": "claude-opus-4-20250514", "reported": "claude-fable-5-1", "lookup": {"family": CL, "cluster": "OUT"}}, True)
add("CL05 a changed model, FADING through SUCCESSOR or CARRIED: REFUSED MODEL", "class", {"certified": CL, "last": "claude-opus-4-20250514", "reported": "claude-fable-5-1", "lookup": {"family": CL, "cluster": "FADING"}}, True)
add("CL06 a changed model, UNLISTED: REFUSED MODEL", "class", {"certified": CL, "last": "claude-opus-4-20250514", "reported": "claude-fable-5-1", "lookup": {"family": CL, "cluster": "UNLISTED"}}, True)
add("CL07 a changed model, excluded: REFUSED MODEL", "class", {"certified": CL, "last": "claude-opus-4-20250514", "reported": "claude-fable-5-1", "lookup": {"family": CL, "cluster": "EXCLUDED"}}, True)
add("CL08 another family, even IN: REFUSED FAMILY", "class", {"certified": CL, "last": "claude-fable-5-1", "reported": "gemini-3-pro", "lookup": {"family": GO, "cluster": "IN"}}, True)
add("CL09 an UNMAPPED model: REFUSED MODEL, detail UNMAPPED", "class", {"certified": CL, "last": "claude-fable-5-1", "reported": "claude-fable-6", "lookup": {"family": "UNMAPPED", "cluster": None}}, True)
add("CL10 a changed model with no cluster answer is refused", "class", {"certified": CL, "last": "claude-opus-4-20250514", "reported": "claude-fable-5-1", "lookup": {"family": CL, "cluster": None}}, True)
add("CL11 the same string but a family that no longer matches: REFUSED FAMILY (the family check first)", "class", {"certified": CL, "last": "gemini-3-pro", "reported": "gemini-3-pro", "lookup": {"family": GO, "cluster": "IN"}}, True)
add("CL12 an unknown cluster value is refused", "class", {"certified": CL, "last": "a", "reported": "b", "lookup": {"family": CL, "cluster": "MAYBE"}}, True)
add("CL13 an empty reported model is refused", "class", {"certified": CL, "last": "a", "reported": "", "lookup": {"family": CL, "cluster": "IN"}}, True)
add("CL14 a lookup missing its cluster key is refused", "class", {"certified": CL, "last": "a", "reported": "b", "lookup": {"family": CL}}, True)
add("CL15 the reported model in NFD equals the last in NFC: SAME", "class", {"certified": CL, "last": "mod\u00e8le", "reported": "mode\u0300le", "lookup": {"family": CL, "cluster": None}}, True)
add("CL16 open weights: a change of server is a change of family key: REFUSED FAMILY", "class", {"certified": OP1, "last": "together/llama-4-70b-fp8", "reported": "fireworks/llama-4-70b-bf16", "lookup": {"family": OP2, "cluster": "IN"}}, True)
add("CL17 SAME with a cluster answer supplied: still SAME", "class", {"certified": CL, "last": "claude-fable-5-1", "reported": "claude-fable-5-1", "lookup": {"family": CL, "cluster": "OUT"}}, True)
add("CL18 an empty certified family is refused", "class", {"certified": "", "last": "a", "reported": "b", "lookup": {"family": CL, "cluster": "IN"}}, True)

# ---- identity_compare ------------------------------------------------------
add("CP01 all four agree", "compare", {"declared": "Anthropic", "stated": "Anthropic", "relayed": "Anthropic", "inferred": "Anthropic"}, True)
add("CP02 the inferred vendor absent: three pairs ABSENT, AGREE", "compare", {"declared": "Anthropic", "stated": "Anthropic", "relayed": "Anthropic", "inferred": None}, True)
add("CP03 the relayed vendor differs: DIFFER on its three pairs", "compare", {"declared": "Anthropic", "stated": "Anthropic", "relayed": "Google", "inferred": "Anthropic"}, True)
add("CP04 all absent: ABSENT", "compare", {"declared": None, "stated": None, "relayed": None, "inferred": None}, True)
add("CP05 only the declared vendor: ABSENT", "compare", {"declared": "Anthropic", "stated": None, "relayed": None, "inferred": None}, True)
add("CP06 declared and stated only, agreeing: AGREE", "compare", {"declared": "Anthropic", "stated": "Anthropic", "relayed": None, "inferred": None}, True)
add("CP07 the stated vendor differs", "compare", {"declared": "Anthropic", "stated": "OpenAI", "relayed": "Anthropic", "inferred": "Anthropic"}, True)
add("CP08 a case difference is a difference", "compare", {"declared": "Anthropic", "stated": "anthropic", "relayed": "Anthropic", "inferred": "Anthropic"}, True)
add("CP09 NFD against NFC agrees", "compare", {"declared": "Soci\u00e9t\u00e9", "stated": "Socie\u0301te\u0301", "relayed": None, "inferred": None}, True)
add("CP10 an empty declared value is refused", "compare", {"declared": "", "stated": "Anthropic", "relayed": None, "inferred": None}, True)
add("CP11 a value that is not a string is refused", "compare", {"declared": "Anthropic", "stated": 3, "relayed": None, "inferred": None}, True)
add("CP12 the inferred vendor differs from the other three", "compare", {"declared": "Anthropic", "stated": "Anthropic", "relayed": "Anthropic", "inferred": "Google"}, True)
add("CP13 two absent, the remaining pair differs", "compare", {"declared": None, "stated": "Anthropic", "relayed": "Google", "inferred": None}, True)
add("CP14 four distinct vendors: six differences", "compare", {"declared": "A", "stated": "B", "relayed": "C", "inferred": "D"}, True)
add("CP15 the declared vendor absent, the other three agree", "compare", {"declared": None, "stated": "Anthropic", "relayed": "Anthropic", "inferred": "Anthropic"}, True)
add("CP16 whitespace is not collapsed: a trailing space differs", "compare", {"declared": "Anthropic", "stated": "Anthropic ", "relayed": None, "inferred": None}, True)

json.dump({"corpus": "identity", "version": "0.1.0", "cases": cases},
          open(os.path.join(here, "identity_corpus.json"), "w", encoding="utf-8", newline="\n"), indent=1,
          sort_keys=True, ensure_ascii=False)
print("cases", len(cases))