"""diff_falcon 0.2.0: random differential run of tpc_falcon, Python against
PHP (0.1.1: SERVED entries in the source lists, SM10; 0.2.0: the
output-schema digest, schema_canon and schema_digest, C7, drawn after
the others so the earlier functions' streams are unchanged). Usage:
python3 diff_falcon.py <seed> [count]. Writes its cases to a
temporary file outside the tree (kept where TPC_DIFF_CASES names it)
(count cases per function: canon, draw, reconcile, stage, qmatch,
qcheck, source, hop, stop, schema_canon, schema_digest), runs
diff_falcon.php over it and compares every render byte for byte."""
import json
import os
import random
import subprocess
import sys
import tempfile
from tpc_falcon import (run_case, SETS, TYPES, falcon_canon, _netstring,
falcon_stop)
seed = int(sys.argv[1]) if len(sys.argv) > 1 else 1
count = int(sys.argv[2]) if len(sys.argv) > 2 else 20000
rnd = random.Random(seed)
T0 = 1789948800
DAY = 86400
TEXT = ["a", "b", "A", " ", " ", "\t", "\n", "\r", "e\u0301", "\u00e9",
"\u212b", "\u00c5", "\u1100\u1161", "\uac00", "\u00a0", "|", "\\",
"\u000b", "x", "\ufb01", "\u0301", "Opus", "\u00e7", "c\u0327"]
SYMS = ["m", "g", "s", "A", "K", "mol", "cd", "rad", "sr", "Hz", "N", "Pa",
"J", "W", "C", "V", "F", "Ohm", "S", "Wb", "T", "H", "lm", "lx", "Bq",
"Gy", "Sv", "kat", "min", "h", "d", "au", "deg", "arcmin", "arcsec",
"ha", "L", "t", "eV", "degC", "Da", "Np", "B", "dB", "ft", "x", "da"]
PREF = ["", "", "", "k", "m", "u", "da", "d", "c", "M", "G", "T", "h", "q",
"Q", "kd"]
SEGS = ["a", "B", ".", "..", "%2E", "%2e%2E", "%7e", "%2f", "", "c%41",
"x y", "\u00e9", "%zz", "@", ":", "[1]"]
def pick(xs):
return xs[rnd.randrange(len(xs))]
def r_text():
if rnd.random() < 0.04:
return pick([5, None, True, 1.5, ["a"]])
return "".join(pick(TEXT) for _ in range(rnd.randint(0, 6)))
def r_digits(lo=0, hi=3):
return "".join(pick("0012345679") for _ in range(rnd.randint(lo, hi)))
def r_decimal():
if rnd.random() < 0.04:
return pick([1.5, 2, None, "1,5", " 1", "\u0663", "1e", "e5"])
s = pick(["", "", "", "-", "+"]) + r_digits(0, 4)
if rnd.random() < 0.4:
s += "." + r_digits(0, 3)
if rnd.random() < 0.3:
s += pick("eE") + pick(["", "", "-", "+"]) + r_digits(0, 5)
return s
def r_unit():
if rnd.random() < 0.05:
return pick(["1", "", " m", "m ", "m/s", "kg.m"])
toks = []
for _ in range(rnd.randint(1, 3)):
t = pick(PREF) + pick(SYMS)
r = rnd.random()
if r < 0.3:
t += str(rnd.randint(-3, 3))
elif r < 0.35:
t += str(rnd.randint(-9999, 9999))
elif r < 0.37:
t += "0" + str(rnd.randint(0, 99))
toks.append(t)
return pick([" ", " ", " ", " "]).join(toks)
def r_unitvalue():
if rnd.random() < 0.04:
return pick([5, None, "5", "5 m"])
return r_decimal() if rnd.random() < 0.03 else (
str(r_decimal()) + " " + r_unit())
def r_date():
if rnd.random() < 0.04:
return pick([20260925, None, "2026-9-25", "2026-09-25T00:00Z"])
y, m, d = rnd.randint(0, 9999), rnd.randint(0, 13), rnd.randint(0, 32)
if rnd.random() < 0.5:
m, d = rnd.randint(1, 12), rnd.randint(1, 28)
if rnd.random() < 0.1:
m, d = 2, 29
return ("%04d-%02d-%02d" if rnd.random() < 0.6 else "%04d%02d%02d") % (
y, m, d)
def r_url():
if rnd.random() < 0.03:
return pick([None, 5, "", "//a/b", "http://a b/"])
s = pick(["http", "HTTP", "https", "HtTpS", "urn", "mailto", "1x", "a+b"])
s += ":"
if rnd.random() < 0.7:
s += "//"
if rnd.random() < 0.2:
s += pick(["user", "User:pw", "u%41", "a@b"]) + "@"
if rnd.random() < 0.15:
s += pick(["[FE80::1]", "[::a]", "[bad"])
else:
s += "".join(pick(["a", "B", "%41", "%c3%a9", "-", ".", "x",
"%zz", "\u00e9"])
for _ in range(rnd.randint(0, 4)))
if rnd.random() < 0.3:
s += pick([":80", ":", ":8x", ":0443"])
if rnd.random() < 0.8:
s += pick(["/", ""]) + "/".join(pick(SEGS)
for _ in range(rnd.randint(0, 4)))
if rnd.random() < 0.3:
s += "?" + pick(["b=1&a=2", "", "Q", "%7E"])
if rnd.random() < 0.2:
s += "#" + pick(["F", "", "%2f"])
return s
def r_value(spec):
t = spec["type"]
if t == "text":
return r_text()
if t == "decimal":
return r_decimal()
if t == "unitvalue":
return r_unitvalue()
if t == "date":
return r_date()
if t == "url":
return r_url()
if t == "enum":
return pick(spec["values"] + ["Open", 1, None])
return pick([True, False, True, False, "true", 1, None])
def r_spec(name="x"):
t = pick(TYPES)
s = {"name": name, "type": t, "class": "MECHANICAL"}
if t == "enum":
s["values"] = pick([["open", "closed"], ["OK", "STALE", "WRONG"]])
return s
def gen_canon():
spec = r_spec()
if rnd.random() < 0.02:
spec = pick([{"name": "x", "type": "number", "class": "MECHANICAL"},
{"name": "IV", "type": "text", "class": "MECHANICAL"},
{"name": "x", "type": "text", "class": "OTHER"}])
return {"value": r_value(spec), "spec": spec}
# Small value pools so that outputs often agree after canonical form.
POOL = {"text": ["x", " x", "x ", "e\u0301", "\u00e9", "y", "a\u00a0b",
"a b"],
"decimal": ["1.5", "1.50", "15e-1", "2", "2.0", "0.2"],
"unitvalue": ["9.81 m s-2", "9.810 s-2 m", "5 cm", "0.05 m"],
"date": ["2026-09-25", "20260925", "2026-09-26"],
"url": ["http://a/b", "HTTP://A/b", "http://a/./b", "http://a/B",
"http://a/%7e", "http://a/~"],
"boolean": [True, False]}
BAD = {"text": [5], "decimal": ["x", 1.5], "unitvalue": ["1 kdB"],
"date": ["2026-02-30"], "url": ["http://a b"], "boolean": ["true"],
"enum": ["bad"]}
KEYS = {"text": [["g", " g"], ["h", "h "], ["m"]],
"decimal": [["1", "1.0"], ["2", "2.00"]],
"date": [["20260925", "2026-09-25"], ["2026-09-24"]]}
def pool_value(spec):
if rnd.random() < 0.02:
return pick(BAD[spec["type"]])
if spec["type"] == "enum":
return pick(spec["values"])
return pick(POOL[spec["type"]])
def r_decl(s):
"""QUOTE AFFIX and QUOTE SCALE on a spec, mostly where allowed."""
if s.get("quote") and s["type"] in ("decimal", "unitvalue"):
if rnd.random() < 0.3:
s["affix"] = pick(["%", "$", "per cent", "pct", chr(0xb0) + "C"])
if rnd.random() < 0.3:
s["scale"] = pick(["million", "k", "bn", "M"])
if rnd.random() < 0.01:
s[pick(["affix", "scale"])] = pick(["%", "million", "ab", None, 5,
"lots"])
def gen_reconcile():
nf = rnd.randint(1, 3)
fields = []
for i in range(nf):
if rnd.random() < 0.3:
cols = [{"name": "k", "type": pick(["text", "decimal", "date"]),
"class": "MECHANICAL"}]
for j in range(rnd.randint(1, 2)):
s = r_spec("c%d" % j)
if rnd.random() < 0.4:
s["quote"] = True
r_decl(s)
if rnd.random() < 0.2:
s["class"] = "INSPECTION"
cols.append(s)
fields.append({"name": "r%d" % i, "type": "rows", "key": ["k"],
"class": "MECHANICAL", "columns": cols})
else:
s = r_spec("f%d" % i)
if rnd.random() < 0.4:
s["quote"] = True
r_decl(s)
if rnd.random() < 0.15:
s["class"] = "INSPECTION"
fields.append(s)
schema = {"class": pick(["FINDINGS", "PATCH", "SURVEY"]),
"narrow": rnd.random() < 0.3, "fields": fields}
if rnd.random() < 0.25:
schema["confidential"] = rnd.random() < 0.8
if rnd.random() < 0.005:
schema["confidential"] = pick(["yes", 1, None])
if rnd.random() < 0.02:
schema["class"] = "OTHER"
nout = pick([2] * 6 + [3] * 5 + [4]) if rnd.random() > 0.01 else 1
outputs = []
for i in range(nout):
acct = "a%d" % i if rnd.random() > 0.003 else "a0"
status = "OVERDUE" if rnd.random() < 0.02 else "RETURNED"
if status == "OVERDUE" or rnd.random() < 0.02:
outputs.append({"account": acct, "status": status,
"result": None})
continue
res = {}
for f in fields:
if f["type"] == "rows":
groups = KEYS[f["columns"][0]["type"]]
keys = [pick(g) for g in rnd.sample(
groups, rnd.randint(0, len(groups)))]
if keys and rnd.random() < 0.03:
keys.append(keys[0])
rows = []
for kv in keys:
row = {"k": kv}
for cspec in f["columns"][1:]:
row[cspec["name"]] = pool_value(cspec)
rows.append(row)
res[f["name"]] = rows
else:
res[f["name"]] = pool_value(f)
r = rnd.random()
if r < 0.01:
res["extra"] = 1
elif r < 0.02 and res:
del res[pick(sorted(res))]
elif r < 0.15:
res["IV"] = pick([["x"], [], ["a", "b"], ["x"], ["\u202e"]])
elif r < 0.16:
res["IV"] = pick(["x", [1]])
elif r < 0.17:
res = pick([[], "x", 5])
outputs.append({"account": acct, "status": status, "result": res})
ids = []
for f in fields:
if f["type"] != "rows":
ids.append(f["name"])
continue
kspec = f["columns"][0]
for kv in ["g", "h", "m", "1", "2", "2026-09-25", "2026-09-24"]:
try:
k = _netstring([falcon_canon(kv, kspec)]).decode("utf-8")
except ValueError:
continue
for cspec in f["columns"][1:]:
ids.append("%s[%s].%s" % (f["name"], k, cspec["name"]))
quotes = []
for i in range(nout):
q = {}
for iid in ids:
if rnd.random() < 0.5:
q[iid] = pick(["PASS", "FAIL", "UNKNOWN", "PASS"])
if rnd.random() < 0.01:
q["x"] = "MAYBE"
quotes.append(q)
if rnd.random() < 0.01:
quotes = quotes[:-1]
return {"outputs": outputs, "quotes": quotes, "schema": schema}
def gen_draw():
m = rnd.randint(2, 9) if rnd.random() < 0.9 else rnd.randint(0, 1)
cands = []
for i in range(m):
assigned = rnd.randint(0, 4)
cands.append({"account": "a%d" % i,
"sponsor": "S%d" % rnd.randint(1, 4),
"family": "V%d" % rnd.randint(1, 3),
"snapshot": "x%d" % rnd.randint(1, 4),
"tier": rnd.random() < 0.4,
"good_standing": rnd.random() < 0.95,
"sandbox": rnd.random() < 0.05,
"over_cap": rnd.random() < 0.05,
"window": rnd.randint(0, 3),
"ontime": rnd.randint(0, assigned),
"assigned": assigned})
accts = [c["account"] for c in cands]
nk = rnd.randint(0, min(2, m))
kept = rnd.sample(accts, nk)
prior = kept + [a for a in accts
if a not in kept and rnd.random() < 0.05]
n = rnd.randint(1, 3) if nk else pick([2, 2, 3])
wait = rnd.randint(0, 100)
drafted = rnd.randint(0, 1000)
now = drafted + pick([0, max(wait - 1, 0), wait, wait + 5])
r = "%032x" % rnd.getrandbits(128)
args = {"candidates": cands, "n": n,
"state": {"kept": kept, "prior": prior, "drafted": drafted,
"now": now, "falcon_wait": wait}, "r": r}
x = rnd.random()
if x < 0.01:
args["n"] = pick([0, 4, 1.0, True])
elif x < 0.02:
args["r"] = r.upper()
elif x < 0.03 and cands:
cands[0]["tier"] = 1
elif x < 0.04:
args["state"]["now"] = drafted - 1
return args
KINDS = ["LANDED"] * 6 + ["EVALUATED", "BLOCKED", "UNBLOCKED", "SUSPENDED",
"RESTORED", "VIOLATION"]
def gen_stage():
events = []
for _ in range(rnd.randint(0, 15)):
k = pick(KINDS)
t = T0 + rnd.randint(0, 40) * DAY + pick([0, 0, 60, 3600,
rnd.randint(0, DAY - 1)])
e = [k, t]
if k in ("LANDED", "SUSPENDED", "RESTORED", "VIOLATION"):
e.append("a%d" % rnd.randint(1, 3))
events.append(e)
as_of = T0 + rnd.randint(0, 200) * DAY + pick([0, 1, -1, 7200])
if rnd.random() < 0.5:
th = SETS[pick("ABC")]
else:
th = {"blocks": rnd.randint(1, 6), "dates": rnd.randint(1, 4),
"span_days": rnd.randint(0, 10),
"dormant_days": rnd.randint(1, 100)}
x = rnd.random()
if x < 0.01 and events:
events[0] = ["LANDED", events[0][1]]
elif x < 0.02:
events.append(["REGISTERED", T0])
elif x < 0.03:
th = {"blocks": 0, "dates": 1, "span_days": 0, "dormant_days": 1}
return {"events": events, "as_of": as_of, "thresholds": th}
# Quote match pieces for the TOKEN RULE: digits, signs (T1), separators
# and group spaces (T2), joins (T3), exponent forms (T4), affixes and
# per-cent words (T5), brackets and quotation marks (T7), scales (T8), Zs
# spaces, Cf characters, letters, whitespace and non-ASCII. NFC needs every
# character assigned by Unicode 13.0 except the two digits and letters of
# 15.0 and 15.1, which NFC leaves alone and the pinned table classifies.
QP = ["12", "123", "1", "2", "3", "5", "10", "234", "000", "72.4", "1,000",
".", ",", "-", "+", "<", "=", "~", "^", "/", ":", "'", "_", " ", " ",
"\t", "(", ")", "[", '"', "%", "$", "e", "E", " x ", "*", "cm", "a",
"x", "Opus", "opus", chr(0x2212), chr(0x2013), chr(0xb1), chr(0x2248),
chr(0x2264), chr(0x2265), chr(0x2019), chr(0xa0), chr(0x2009),
chr(0x202f), chr(0xd7), chr(0xb7), chr(0x207b), chr(0x2075), chr(0xb2),
chr(0x2030), chr(0x20ac), chr(0xb0), chr(0x2032), chr(0xe9),
"e" + chr(0x301), chr(0x301), chr(0x6771) + chr(0x4eac), chr(0x3001),
chr(0x663), chr(0x2007), chr(0x2002), chr(0x3000), chr(0x1680),
chr(0x200b), chr(0xad), chr(0x2060), chr(0xfeff), chr(0x200e),
chr(0x2044), chr(0x2215), chr(0x2236), chr(0x1e4f1), chr(0x2ebf0),
" million", "million", "Millions", " M", "k", " k", " K", " bn", "Bn",
" B", " m", "m", " thousand", " lakh", "s", " E", " e ", "**", " ** ",
" times 10^", "times", " per cent", "per", " cent", "percent", " pct",
"PCT", " / ", " - ", " = ", " models", "km", chr(0x2014), chr(0x2010),
chr(0x2e3a), chr(0x301c), chr(0xff0d), chr(0xff5e), chr(0xf7),
chr(0x29f8), chr(0x2053), "~", chr(0xbd), chr(0x2153), chr(0x2189),
" 1/2", chr(0xff05), chr(0xfe6a), chr(0x66a), chr(0x609), chr(0x60a),
" bps", "bp", " basis points", " basis", " point", " per mille",
"mille", "permille", " pp", " ppm", "PPB", " ppt", " percentage",
" mln", " BLN", " tn", "trn", " Mio", "mrd", " MM", " mm", " dozen",
" thousandths", "tenth", " quadrillion", " to "]
AFFIX = [None, None, "%", "$", chr(0xb0) + "C", chr(0x2030), chr(0x20ac),
"percent", "per cent", "pct", chr(0xff05), chr(0x66a), "bps",
"basis point", "basis points", "per mille", "pp", "ppm",
"percentage points"]
SCALE = [None, None, None, "million", "k", "K", "M", "bn", "m", "B",
"thousand", "lakh", "MM", "mln", "mio", "dozen", "thousandth",
"tenth"]
def r_hay():
return "".join(pick(QP) for _ in range(rnd.randint(0, 8)))
def r_needle(h):
if h and rnd.random() < 0.5:
i = rnd.randint(0, len(h) - 1)
return h[i:rnd.randint(i + 1, len(h))]
return "".join(pick(QP) for _ in range(rnd.randint(1, 3)))
def r_flags():
aff = pick(AFFIX)
sc = pick(SCALE)
acc = rnd.random() < 0.2
if rnd.random() < 0.01:
aff = pick(["ab", "", 5, "%%", "Percent", "points", "Basis Point"])
if rnd.random() < 0.01:
sc = pick(["lots", "", 5, "Million", "millions", "Mio", "mm",
"thousandths"])
if rnd.random() < 0.005:
acc = pick(["yes", 1, None])
return aff, sc, acc
def gen_qmatch():
parts = [pick(QP) for _ in range(rnd.randint(0, 8))]
h = "".join(parts)
if parts and rnd.random() < 0.5:
i = rnd.randint(0, len(parts) - 1)
v = "".join(parts[i:rnd.randint(i + 1, len(parts))])
else:
v = r_needle(h)
if rnd.random() < 0.01:
v = pick([12, None, ["12"]])
aff, sc, acc = r_flags()
return {"value": v, "quote": h, "affix": aff, "scale": sc,
"accounting": acc}
def gen_qcheck():
parts = [pick(QP) for _ in range(rnd.randint(0, 8))]
page = "".join(parts)
if parts and rnd.random() < 0.6:
i = rnd.randint(0, len(parts) - 1)
qparts = parts[i:rnd.randint(i + 1, len(parts))]
quote = "".join(qparts)
j = rnd.randint(0, len(qparts) - 1)
value = ("".join(qparts[j:rnd.randint(j + 1, len(qparts))])
if rnd.random() < 0.7 else r_needle(quote))
else:
quote = r_needle(page)
value = r_needle(quote)
if rnd.random() < 0.05:
page = None
if rnd.random() < 0.01:
value = 5
aff, sc, acc = r_flags()
return {"page": page, "quote": quote, "value": value, "affix": aff,
"scale": sc, "accounting": acc}
HOSTS = ["example.org", "Example.ORG", "docs.example.com", "api.example.net",
"evil.example.org", "example.org.", "127.0.0.1", "[::1]",
"exa mple.org", "%65xample.org", "example.org.evil.com",
"github.com", "GitHub.com"]
PATHS = ["", "/", "/org", "/org/", "/org2", "/models", "/modelsx", "/a", "/..",
"/.", "/%2E%2E", "/%2F", "/%7Euser", "/v1/list", "/bench.csv",
"/bench%2ecsv", "/p", "/..;", "/%252E%252E", "/;x", "/a;b", "/%5C",
"/%3B", "/%252F", "/%25252E", "/...", "/v1.2", "/repo", "/tree",
"/main", "/blob", "/issues", "/%C3%A9", "/%" + "25" * 6 + "41",
"/%" + "25" * 7 + "41", "/refs", "/REFS", "/%C0%AE", "/%ED%A0%80",
"/%E2%82", "/%F0%9F%98%80", "/%80", "/%25C0%25AE", "/%F4%90%80%80"]
def r_cite():
if rnd.random() < 0.02:
return pick([5, None, "", "https:/org/x"])
s = pick(["https", "https", "https", "HTTPS", "http", "ftp"]) + "://"
if rnd.random() < 0.05:
s += pick(["user", "u:p"]) + "@"
s += pick(HOSTS) + pick(["", "", "", ":443", ":8443", ":", ":444"])
s += "".join(pick(PATHS) for _ in range(rnd.randint(0, 3)))
if rnd.random() < 0.3:
s += "?" + pick(["format=csv", "format=json", "utm=x", "", "p=../x"])
if rnd.random() < 0.2:
s += "#" + pick(["top", ""])
return s
GH = ["CODE HOST", "github.com", "/*/*/tree/*/", "/*/*/blob/*/",
"/*/*/commit/*/"]
LISTS = [
[["EXACT", "https://data.example.org/bench.csv"],
["PREFIX", "https://example.org/org/"],
["PREFIX", "https://docs.example.com/models"],
["EXACT", "https://api.example.net:8443/v1/list?format=csv"]],
[["PREFIX", "https://example.org/a/"], ["PREFIX", "https://example.org/a/org/"]],
[["PREFIX", "https://example.org/org/"],
["EXACT", "https://example.org/org/p"]],
[["EXACT", "https://example.org"]],
[GH, ["PREFIX", "https://github.com/org/repo/tree/main/"],
["EXACT", "https://data.example.org/bench.csv"],
["PREFIX", "https://example.org/org/"]],
[GH, ["PREFIX", "https://github.com/org/repo/blob/0123abcd/"],
["EXACT", "https://github.com/org/repo/issues/1"]],
[["CODE HOST", "github.com", "/*/*/tree/*/"],
["PREFIX", "https://github.com/org/repo/tree/main/docs/"],
["PREFIX", "https://github.com/org/repo/tree/dev/"]], []]
DECL = ["https://example.org/org", "https://example.org/org/",
"https://Example.ORG/a/%7eb", "https://api.example.net:8443/v1/list",
"https://api.example.net:8443/v1/list?format=csv",
"https://docs.example.com/models/",
"https://github.com/org/repo/tree/main/",
"https://github.com/org/repo/commit/0123abcd/",
"https://docs.example.com/", "http://example.org/", "https://u@a.org/",
"https://10.0.0.1/", "https://example.org/p#f", "https://a.org:/",
"https://github.com/org/", "https://github.com/org/repo",
"https://example.org/a/..;/", "https://example.org/a/%2E%2E/b/",
"https://example.org/a;b", "https://Example.ORG/a/./b",
"https://github.com/org/repo/tree/refs/", "https://example.org/%C0%AE/",
"https://github.com/org/repo/blob/Refs/pull/"]
GOOD = DECL[:8]
SERVED = ["https://api.crossref.org/v1/works/10.1016/s0140-6736(97)11096-0",
"https://api.crossref.org/v1/works/10.1002/(SICI)1097-0266(199708)18:7%3C509::AID-SMJ882%3E3.0.CO%3B2-Z",
"https://export.arxiv.org/api/query?id_list=2601.12345v2",
"https://doi.org/ra/10.1000/x", "https://api.datacite.org/dois/10.5281/zenodo.1234567",
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&retmode=json&id=9500320",
"https://example.org/org/x", "https://data.example.org/bench.csv", "https://doi.org/",
"https://github.com/org/repo/raw/main/x", "http://doi.org/ra/x", "https://doi.org/ra/x#f",
"https://u@doi.org/ra/x", "https://api.crossref.org/v1/works/10.1000/a%2Fb"]
MARKS = [GH, ["CODE HOST", "github.com"], ["CODE HOST", "github.com", "/*/"],
["CODE HOST", "GitHub.com", "/*/*/tree/*/"],
["CODE HOST", "github.com", "/*/*/tree/*"],
["CODE HOST", "docs.example.com", "/*/"],
["CODE HOST", "github.com", "/*/*/*/"]]
def r_sources():
if rnd.random() < 0.5:
sources = [list(x) for x in pick(LISTS)]
else:
sources = [[pick(["EXACT", "PREFIX", "PREFIX"]), pick(GOOD)]
for _ in range(rnd.randint(0, 4))]
if rnd.random() < 0.3:
sources.insert(rnd.randint(0, len(sources)), pick(MARKS[:3]))
if rnd.random() < 0.5:
for _ in range(rnd.randint(1, 3)):
sources.insert(rnd.randint(0, len(sources)),
["SERVED", pick(SERVED[:10] if rnd.random() < 0.9 else SERVED)])
if rnd.random() < 0.05:
sources.append(["SERVED", pick(SERVED[:10]), "x"])
if rnd.random() < 0.05:
sources.append(pick([["LIKE", "https://a.org/"], "https://a.org/",
["EXACT", pick(DECL[9:])],
["PREFIX", pick(DECL[8:])], pick(MARKS)]))
return sources
def r_near(sources):
es = [x for x in sources if type(x) is list and len(x) == 2
and type(x[1]) is str]
if es and rnd.random() < 0.6:
cited = pick(es)[1] + pick(["", "", "", "", "/", "x", "/x", "/x/y", "/../x",
"?q=1", "#f", "/%2E%2E/x", "/%2Fx",
"/..;/x", "/%252E%252E/x", "/a;b",
"/%5Cx", "data.csv", "../issues/1",
"/%C0%AE%C0%AE/x", "/%ED%A0%80",
"/refs/pull/1/", "v3"])
if rnd.random() < 0.2:
cited = cited.replace("https", pick(["HTTPS", "http"]), 1)
if rnd.random() < 0.1:
cited = cited.replace("%3B", "%3b").replace("(", "%28").replace("-", "%2D")
return cited
return r_cite()
def gen_source():
sources = r_sources()
return {"cited": r_near(sources), "sources": sources}
def gen_hop():
sources = r_sources()
x = rnd.random()
valid = [i for i, e in enumerate(sources) if type(e) is list
and len(e) == 2 and e[0] in ("EXACT", "PREFIX")]
served = [i for i, e in enumerate(sources) if type(e) is list
and len(e) == 2 and e[0] == "SERVED"]
if x < 0.8 and valid:
matched = pick(valid)
elif x < 0.85 and served:
matched = pick(served)
elif x < 0.9 and sources:
matched = rnd.randrange(len(sources))
elif x < 0.97:
matched = rnd.randint(-1, len(sources) + 1)
else:
matched = pick(["0", None, 1.0, True])
return {"hop": r_near(sources), "matched": matched, "sources": sources}
def gen_stop():
lim = {"breaker": pick(["0", "0.1", "0.25", "0.3", "0.5", "1", "0.3333"]),
"window": pick([1, 2, 3, 5, 20])}
if rnd.random() < 0.01:
lim = pick([{"breaker": "0.30001", "window": 20},
{"breaker": 0.3, "window": 20},
{"breaker": "0.3", "window": 0}, {"breaker": "0.3"}])
counts = []
flying = []
n = 0
spoil = rnd.random() < 0.05
for _ in range(rnd.randint(0, 40)):
st = falcon_stop(counts, {"breaker": "0.3", "window": 20}
if type(lim.get("breaker")) is not str
or "window" not in lim or lim["window"] == 0
or len(lim["breaker"]) > 6 else lim)["status"]
x = rnd.random()
if spoil and x < 0.1:
counts.append(pick([["ISSUED", "f1"], ["FINISHED", "zz", 1, 0, 0],
["STOPPED"], ["RESUMED", 1]]))
break
if st == "STOPPED" and (not flying or x < 0.3):
counts.append(["RESUMED"])
elif st == "ISSUING" and (not flying or x < 0.5):
n += 1
counts.append(["ISSUED", "f%d" % n])
flying.append("f%d" % n)
elif flying:
fid = flying.pop(rnd.randrange(len(flying)))
items = rnd.randint(0, 10)
dec = rnd.randint(0, items)
unr = rnd.randint(0, items - dec)
if spoil and rnd.random() < 0.1:
counts.append(["FINISHED", fid, items, dec, items + 1])
break
counts.append(["FINISHED", fid, items, dec, unr])
else:
counts.append(["RESUMED"])
return {"counts": counts, "limits": lim}
# Strings for the schema digest: line ends, NFD and NFC, the escapes, the
# controls, DEL, U+2028, a character beyond the BMP (never a lone
# surrogate: TPC-PHP cannot decode one).
STXT = TEXT + ["\r\n", '"', "\\", "\u0000", "\u001f", "\u007f", "\u2028",
"\U0001f600", "\u00e9", "e\u0301"]
def r_stext():
return "".join(pick(STXT) for _ in range(rnd.randint(0, 4)))
def shuffled(d):
"""The same object with its members in a random order."""
ks = list(d)
rnd.shuffle(ks)
return {k: d[k] for k in ks}
def r_sspec(name, allow_rows):
if allow_rows and rnd.random() < 0.25:
cols = [{"name": "k", "type": pick(["text", "decimal", "date"]),
"class": "MECHANICAL"}]
for j in range(rnd.randint(1, 3)):
cols.append(r_sspec("c%d" % j, False))
key = ["k"] + (["c0"] if rnd.random() < 0.2 and cols[1]["class"] == "MECHANICAL"
and cols[1]["type"] != "rows" else [])
return shuffled({"name": name, "type": "rows", "key": key,
"class": "MECHANICAL", "columns": cols})
s = r_spec(name)
if s["type"] == "enum" and rnd.random() < 0.6:
vals = []
for _ in range(rnd.randint(1, 4)):
v = r_stext()
if v not in vals:
vals.append(v)
s["values"] = vals
x = rnd.random()
if x < 0.3:
s["quote"] = True
elif x < 0.55:
s["quote"] = False
r_decl(s)
if rnd.random() < 0.3:
s["class"] = "INSPECTION"
return shuffled(s)
def gen_schema():
fields = [r_sspec("f%d" % i, True) for i in range(rnd.randint(1, 3))]
schema = {"class": pick(["FINDINGS", "PATCH", "SURVEY"]),
"narrow": rnd.random() < 0.4, "fields": fields}
x = rnd.random()
if x < 0.3:
schema["confidential"] = True
elif x < 0.6:
schema["confidential"] = False
if rnd.random() < 0.06:
bad = rnd.randrange(10)
if bad == 0:
return {"schema": pick([None, ["class"], "FINDINGS", 5, True, {}])}
if bad == 1:
del schema["narrow"]
elif bad == 2:
schema["narrow"] = pick(["false", 0, None])
elif bad == 3:
schema["fields"] = pick([{}, [], [[]], [None]])
elif bad == 4:
fields[0]["name"] = pick(["IV", "a-b", "", 5, "1a"])
elif bad == 5:
schema[pick(["version", "Narrow", "quote"])] = pick([True, "1"])
elif bad == 6:
schema["class"] = pick(["REPORT", "findings", None])
elif bad == 7:
fields.append(dict(fields[0]))
elif bad == 8:
fields[0]["quote"] = pick(["false", 0, None])
else:
schema["confidential"] = pick(["no", 1, None])
return {"schema": shuffled(schema)}
GEN = [("canon", gen_canon), ("draw", gen_draw),
("reconcile", gen_reconcile), ("stage", gen_stage),
("qmatch", gen_qmatch), ("qcheck", gen_qcheck),
("source", gen_source), ("hop", gen_hop), ("stop", gen_stop),
("schema_canon", gen_schema), ("schema_digest", gen_schema)]
cases = []
for fn, g in GEN:
for _ in range(count):
cases.append({"fn": fn, "args": g()})
CASES_PATH = os.environ.get("TPC_DIFF_CASES") # named: kept, for crosspy.py
if not CASES_PATH:
fd, CASES_PATH = tempfile.mkstemp(prefix="tpc_diff_", suffix=".json") # outside the tree
os.close(fd)
with open(CASES_PATH, "w", encoding="utf-8") as f:
json.dump(cases, f)
py = [run_case(c["fn"], c["args"]) for c in cases]
here = os.path.dirname(os.path.abspath(__file__))
php = json.loads(subprocess.run(["php", os.path.join(here, "diff_falcon.php"), CASES_PATH],
capture_output=True, text=True,
check=True).stdout)
if not os.environ.get("TPC_DIFF_CASES"):
os.remove(CASES_PATH)
assert len(php) == len(py)
first = None
for fn, _ in GEN:
idx = [i for i, c in enumerate(cases) if c["fn"] == fn]
diffs = [i for i in idx if py[i] != php[i]]
cover = {}
for i in idx:
out = py[i]
if out.startswith("ERROR"):
keys = ["ERROR"]
elif fn in ("schema_canon", "schema_digest"):
keys = ["OK"]
elif fn == "canon":
keys = ["OK:" + cases[i]["args"]["spec"]["type"]]
elif fn == "draw":
keys = [out.split("\n")[0][5:]]
elif fn == "stage":
keys = [out.split("\n")[0][6:] +
("+DORMANT" if "dormant|true" in out else "")]
elif fn in ("qmatch", "qcheck"):
keys = [out[6:]]
elif fn in ("source", "hop"):
keys = [":".join(out.split("\n")[0].split("|")[1:4:2]
if out.startswith("source|ENTRY")
else out.split("\n")[0].split("|")[1:3])]
if fn == "hop":
keys = [keys[0] + ":" + out.split("\n")[-1][4:]]
elif fn == "stop":
ls = out.split("\n")
keys = [ls[0][5:] + ("+ALONE" if ls[2] != "alone|-" else "")]
else:
keys = [out.split("\n")[-1].split("|")[1]]
keys += sorted({ln.split("|")[2] for ln in out.split("\n")
if ln.startswith("item|")})
keys += sorted({"MALFORMED:" + ln.split("|")[4]
for ln in out.split("\n")
if "|MALFORMED|" in ln})
for k in keys:
cover[k] = cover.get(k, 0) + 1
print("seed %d %-9s cases %d differences %d" % (seed, fn, len(idx),
len(diffs)))
print(" " + ", ".join("%s %d" % (k, cover[k]) for k in sorted(cover)))
if diffs and first is None:
first = diffs[0]
if first is not None:
print(json.dumps(cases[first], ensure_ascii=True))
print(py[first])
print("---")
print(php[first])
