The Principia Machina/TPC-PY/diff lint.py: Difference between revisions

Latest revision as of 04:32, 28 September 2026

"""diff_lint 0.1.0: the cross-residence differential for tpc_lint 0.1.0.
Random zones built from the tables' own template expansions (random case,
white space of every kind, negations before them, stop characters, hidden
characters inside, letters glued to their edges), credential shapes of
every PL2 pattern with class runs around their bounds, hidden and
White_Space code points, combining marks, Hangul jamo (NFC composes),
CRLF and CR; random tool lists (MCP, Anthropic and OpenAI shapes; known
and unknown member names; credential-shaped and hidden keys; poisoned
descriptions; open-ended names in snake, camel and server-prefixed form;
every JSON escape) and their mutations to invalid JSON or a wrong shape;
invalid UTF-8 given as hex; caps around the zone's length and invalid
caps; counts, counts_of, norm and tokens. Both residences on the same
inputs; 0 differences is the pass. Every code point drawn is assigned
(or unassigned) alike from Unicode 13.0 to 15.1, so the run is valid
under Python 3.10 to 3.13 against PHP's ICU.
Usage: python3 diff_lint.py [seed] [cases]."""
import json
import os
import random
import subprocess
import sys
import tempfile

here = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, here)
from tpc_lint import run_case, LINT_TABLES_JSON  # noqa: E402

seed = int(sys.argv[1]) if len(sys.argv) > 1 else 1
count = int(sys.argv[2]) if len(sys.argv) > 2 else 20000
random.seed(seed)
TB = json.loads(LINT_TABLES_JSON)
CLS = TB["classes"]

HIDDEN = [0x00, 0x07, 0x1B, 0x7F, 0x84, 0x9F, 0xAD, 0x200B, 0x200C, 0x200D,
          0x200E, 0x202A, 0x202E, 0x2066, 0x2069, 0x2028, 0x2029, 0xFEFF, 0xE000,
          0xF8FF, 0x0378, 0xFFFE, 0xFDD0, 0xE0001, 0xE0041, 0xE007F]
WSP = [0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x20, 0x85, 0xA0, 0x1680, 0x2000, 0x2005,
       0x200A, 0x202F, 0x205F, 0x3000]
LETTERS = ([0xE9, 0xC9, 0xDF, 0x130, 0x131, 0x3B1, 0x391, 0x430, 0x410, 0x2019,
            0x1F600, 0xFF41, 0xFF21, 0x4E2D, 0x6587, 0x5C0F, 0xAC00]
           + list(range(0x61, 0x7B)) + list(range(0x41, 0x5B)) + list(range(0x30, 0x3A)))
MARKS = [0x300, 0x301, 0x308, 0x327, 0x345]
PUNCT = list(".!?;:,'\"-_/()[]<>|~@=+*&%$#")
NEGS = ["do not ", "Do Not ", "never ", "NEVER ", "don't ", "don\u2019t ", "no ",
        "cannot ", "not ", "won't "]
FILL = ["please", "the", "agent", "and", "then", "work", "register", "a", "x1",
        "Kernel", "TPM", "result", "of", "in"]


def rchr(pool):
    return chr(random.choice(pool))


def recase(s):
    r = random.random()
    if r < 0.4:
        return s
    if r < 0.6:
        return s.upper()
    return "".join(c.upper() if random.random() < 0.5 else c for c in s)


def rspace():
    r = random.random()
    if r < 0.6:
        return " "
    return "".join(rchr(WSP) for _ in range(random.randint(1, 3)))


def expansion(parts):
    out = "".join(random.choice(p) for p in parts)
    if random.random() < 0.05 and len(out) > 2:
        i = random.randrange(1, len(out))
        out = out[:i] + random.choice(["\u200b", "", "x", " "]) + out[i:]
    return out


def phrase():
    code = random.choice(["PL3", "PL4", "PL5", "PL9", "PL6"])
    row = random.choice(TB[code])
    parts = random.choice(row[1])
    s = recase(expansion(parts)).replace(" ", "\x00SP\x00")
    s = s.replace("\x00SP\x00", "\x01")
    s = "".join(rspace() if c == "\x01" else c for c in s)
    r = random.random()
    if r < 0.2:
        s = random.choice(NEGS) + " ".join(random.sample(FILL, random.randint(0, 4))) + " " + s
    elif r < 0.3:
        s = random.choice(NEGS) + random.choice([". ", "; ", ": ", "! ", ", "]) + s
    elif r < 0.37:
        s = random.choice(["x", "9", "_", "-", ".", "\u00e9"]) + s
    if random.random() < 0.1:
        s += random.choice(["x", "s", "d", "9", "_", "."])
    return s


def pl7_template():
    row = random.choice(TB["PL7"])
    s = recase(expansion(random.choice(row[2])))
    return s


def cred():
    row = random.choice(TB["PL2"])
    pat = random.choice(row[1])
    out = []
    for el in pat:
        if el[0] == "L":
            out.append(recase(el[1]))
        else:
            chars = CLS[el[1]]
            lo, hi = el[2], el[3]
            r = random.random()
            if r < 0.6:
                n = random.randint(max(0, lo - 1), min(hi + 1, lo + 3))
            elif r < 0.9:
                n = random.randint(max(0, lo), min(hi, lo + 40))
            else:
                n = random.randint(max(0, hi - 1), hi + 1) if hi <= 300 else lo
            s = "".join(random.choice(chars) for _ in range(n))
            out.append(recase(s) if el[1] not in ("S", "E", "Q") else s)
    s = "".join(out)
    if random.random() < 0.1:
        s = random.choice(["a", "_", "Z", "."]) + s
    if random.random() < 0.1:
        s += random.choice(["a", "_", "Z", ".", "'", "-"])
    return s


def noise():
    r = random.random()
    if r < 0.3:
        return " ".join(random.choice(FILL) for _ in range(random.randint(1, 6)))
    if r < 0.45:
        return "".join(rchr(HIDDEN) for _ in range(random.randint(1, 3)))
    if r < 0.6:
        return "".join(rchr(LETTERS) + (rchr(MARKS) if random.random() < 0.3 else "")
                       for _ in range(random.randint(1, 8)))
    if r < 0.7:
        return "\u1100\u1161" * random.randint(1, 3) + "e\u0301"
    if r < 0.8:
        return random.choice(["\r\n", "\r", "\n", "\t"]) * random.randint(1, 3)
    if r < 0.9:
        return "".join(random.choice(PUNCT) for _ in range(random.randint(1, 5)))
    return random.choice(["TPS VERSION: 0.2.0", "| class: instruction artifact",
                          "END OF ARTIFACT --- TPS 0.2.0", "PART 6  AGENT CONTRACT",
                          "AKIAIOSFODNN7EXAMPLE", "password=hunter2hunter2"])


def zone():
    frags = []
    for _ in range(random.randint(0, 8)):
        r = random.random()
        frags.append(phrase() if r < 0.35 else cred() if r < 0.55 else noise())
    seps = ["", " ", "\n", ". ", "\r\n", "; ", rchr(WSP)]
    z = ""
    for f in frags:
        z += random.choice(seps) + f
    if random.random() < 0.02:
        z = z * random.randint(50, 400)
    return z


NAME_TOKENS = ["run", "shell", "command", "bash", "sh", "fetch", "url", "http", "request",
               "get", "send", "email", "message", "post", "read", "page", "push", "web",
               "code", "interpreter", "wget", "Browser", "upload", "tool", "refresh", "exec",
               "list", "mcp", "search", "memory", "2", "x"]


def tool_name():
    toks = random.sample(NAME_TOKENS, random.randint(1, 3))
    r = random.random()
    if r < 0.4:
        return "_".join(toks)
    if r < 0.6:
        return toks[0] + "".join(t[:1].upper() + t[1:] for t in toks[1:])
    if r < 0.7:
        return "mcp__" + "__".join(toks)
    if r < 0.8:
        return "".join(t.upper() for t in toks)
    if r < 0.9:
        return random.choice(toks) + rchr(LETTERS)
    return recase("-".join(toks))


KEYS = ["name", "title", "description", "type", "properties", "items", "default",
        "enum", "path", "url", "query", "token", "$schema", "annotations", "readOnlyHint",
        "p\u200b", "AKIAIOSFODNN7EXAMPLE", "\u00e9", "", "Name", "0", "cache_control"]


def desc():
    r = random.random()
    if r < 0.35:
        return phrase()
    if r < 0.55:
        return pl7_template() + " " + noise()
    if r < 0.7:
        return cred()
    return noise()


def schema(depth):
    props = {}
    for _ in range(random.randint(0, 4)):
        k = random.choice(KEYS)
        v = {"type": random.choice(["string", "number", "object"])}
        if random.random() < 0.6:
            v["description"] = desc()
        if random.random() < 0.3:
            v["default"] = desc()
        if random.random() < 0.2:
            v["title"] = desc()
        if depth < 2 and random.random() < 0.2:
            v["properties"] = schema(depth + 1)["properties"]
        if random.random() < 0.1:
            v["enum"] = [desc(), 1, None, True]
        props[k] = v
    s = {"type": "object", "properties": props}
    if props and random.random() < 0.5:
        s["required"] = random.sample(list(props), random.randint(1, len(props)))
    return s


def tool():
    shape = random.random()
    t = {}
    if random.random() < 0.9:
        t["name"] = tool_name() if random.random() < 0.95 else random.choice([5, None, ["x"]])
    if random.random() < 0.8:
        t["description"] = desc()
    if random.random() < 0.2:
        t["title"] = desc()
    if shape < 0.4:
        t["inputSchema"] = schema(0)
    elif shape < 0.7:
        t["input_schema"] = schema(0)
    elif shape < 0.85:
        return {"type": "function", "function": dict(t, parameters=schema(0))}
    if random.random() < 0.1:
        t[random.choice(KEYS)] = desc()
    return t


def tools_text():
    lst = [tool() for _ in range(random.randint(0, 4))]
    r = random.random()
    if r < 0.03:
        lst = {"tools": lst}
    elif r < 0.05:
        lst.append(random.choice([1, "x", [], None]))
    s = json.dumps(lst, ensure_ascii=random.random() < 0.5,
                   indent=random.choice([None, None, 1, 2]),
                   separators=random.choice([None, (",", ":"), (", ", ": ")]))
    if random.random() < 0.05 and s:
        i = random.randrange(len(s))
        s = s[:i] + random.choice(["", ",", "]", "}", "\"", "NaN", "\\ud800", "\t", "\\x"]) + s[i + 1:]
    if random.random() < 0.03:
        s = s.replace('"}', '", "a": 1, "a": 2}', 1)
    return s


def cap_for(z):
    r = random.random()
    n = len(z)
    if r < 0.4:
        return 8000
    if r < 0.8:
        return max(1, n + random.randint(-2, 2))
    if r < 0.93:
        return random.randint(1, 1048576)
    return random.choice([0, -1, 1048577, 8000.0, True, "8000", None])


BAD_HEX = ["ff", "c0af", "eda080", "e280", "f4908080", "80", "fe", "c3"]

cases = []
for _ in range(count):
    fn = random.choices(["lint", "counts", "counts_of", "norm", "tokens"],
                        weights=[80, 8, 4, 4, 4])[0]
    if fn in ("lint", "counts"):
        z = zone()
        a = {"tools": tools_text(), "cap": cap_for(z)}
        r = random.random()
        if r < 0.02:
            a["zone_hex"] = (z.encode("utf-8") + bytes.fromhex(random.choice(BAD_HEX))).hex()
        elif r < 0.03:
            a["zone"] = random.choice([None, 3, ["x"]])
        else:
            a["zone"] = z
        if random.random() < 0.02:
            del a["tools"]
            a["tools_hex"] = bytes.fromhex(random.choice(BAD_HEX)).hex()
        cases.append({"fn": fn, "args": a})
    elif fn == "counts_of":
        fl = []
        for _ in range(random.randint(0, 5)):
            f = {"code": random.choice(["PL%d" % i for i in range(10)]),
                 "item": "x", "locator": "-"}
            r = random.random()
            if r < 0.05:
                f["code"] = "PL10"
            elif r < 0.08:
                del f["locator"]
            elif r < 0.1:
                f["extra"] = "y"
            elif r < 0.12:
                f["item"] = 1
            fl.append(f)
        cases.append({"fn": fn, "args": {"findings": fl if random.random() < 0.95 else {"a": 1}}})
    elif fn == "norm":
        cases.append({"fn": fn, "args": {"text": zone() if random.random() < 0.95 else 7}})
    else:
        cases.append({"fn": fn, "args": {"name": tool_name() if random.random() < 0.95 else None}})

CASES_PATH = os.environ.get("TPC_DIFF_CASES")  # named: kept, for crosspy.py
if not CASES_PATH:
    fd, CASES_PATH = tempfile.mkstemp(prefix="tpc_diff_", suffix=".json")  # outside the tree
    os.close(fd)
with open(CASES_PATH, "w", encoding="utf-8") as f:
    json.dump(cases, f, ensure_ascii=False)
py = [run_case(c["fn"], c["args"]) for c in cases]
res = subprocess.run(["php", "-d", "memory_limit=1G", os.path.join(here, "diff_lint.php"),
                      CASES_PATH],
                     capture_output=True, text=True)
if not os.environ.get("TPC_DIFF_CASES"):
    os.remove(CASES_PATH)
if res.returncode != 0:
    print(res.stderr[:2000])
    sys.exit(2)
php = json.loads(res.stdout)
diffs = sum(1 for a, b in zip(py, php) if a != b)
by = {}
codes = {}
for c, x in zip(cases, py):
    bad = x.startswith("ERROR") or '"code":"PL0"' in x or '"PL0":1' in x
    k = c["fn"] + ("/ERROR" if bad else "")
    by[k] = by.get(k, 0) + 1
    if c["fn"] == "lint":
        for i in range(10):
            codes["PL%d" % i] = codes.get("PL%d" % i, 0) + x.count('"code":"PL%d"' % i)
print("seed", seed, "cases", len(cases), "differences", diffs + abs(len(py) - len(php)),
      json.dumps(by, sort_keys=True))
print("findings by code in lint results:", json.dumps(codes, sort_keys=True))
for c, a, b in zip(cases, py, php):
    if a != b:
        print(json.dumps(c, ensure_ascii=True)[:800])
        print(a[:400])
        print("---")
        print(b[:400])
        break