The Principia Machina/TPC-PY/diff biblio.py: Difference between revisions
m 1 revision imported |
TPC-PY source diff_biblio.py (0.1.0), SHA-256 58976011ebffb18c; conformed TPS 0.4.0; PROPOSED, not declared (TPC-PY 0.2.0) |
||
| Line 8: | Line 8: | ||
earlier, so Python 3.10's unicodedata, Python 3.13's and ICU classify | earlier, so Python 3.10's unicodedata, Python 3.13's and ICU classify | ||
them alike (the key form's classes come from the pinned table in any | them alike (the key form's classes come from the pinned table in any | ||
case; NFC, NFD and the zone's text safety come from the interpreter).""" | case; NFC, NFD and the zone's text safety come from the interpreter). | ||
Increment 12 (tpc_biblio 0.2.0): view's statuses are drawn from the six | |||
status words, STRUCK and "" (now refused) replaced by WITHDRAWN and NOT | |||
GRADED.""" | |||
import json | import json | ||
import os | import os | ||
| Line 513: | Line 516: | ||
lit.append(lit[0]) | lit.append(lit[0]) | ||
recs.append({"id": pick(["RR-%d" % i, "RR-%d" % (i + 20), "RR-%d" % (i + 30)] * 4 + ["RR-1", "RR-10", "RR-2", "RR-7"]), | recs.append({"id": pick(["RR-%d" % i, "RR-%d" % (i + 20), "RR-%d" % (i + 30)] * 4 + ["RR-1", "RR-10", "RR-2", "RR-7"]), | ||
"status": pick(["PROVISIONAL"] * 6 + ["ESTABLISHED"] * 3 + [" | "status": pick(["PROVISIONAL"] * 6 + ["ESTABLISHED"] * 3 + ["WITHDRAWN", "HELD", "REFUTED", "NOT GRADED"]), | ||
"literature": lit}) | "literature": lit}) | ||
if rnd.random() < 0.015: | if rnd.random() < 0.015: | ||
Revision as of 02:10, 30 September 2026
"""diff_biblio 0.1.0: random differential run of tpc_biblio, Python against
PHP. Usage: python3 diff_biblio.py <seed> [count]. Writes its cases to a
temporary file outside the tree (kept where TPC_DIFF_CASES names it)
(count cases per function: doi, shape, arxiv, isbn, issn, pmid, pmcid,
key, dupe, check, canon, preimage, url, compare, withdrawn, view,
holdlist), runs diff_biblio.php over it and compares every render byte
for byte. The alphabet holds code points assigned at Unicode 13.0 or
earlier, so Python 3.10's unicodedata, Python 3.13's and ICU classify
them alike (the key form's classes come from the pinned table in any
case; NFC, NFD and the zone's text safety come from the interpreter).
Increment 12 (tpc_biblio 0.2.0): view's statuses are drawn from the six
status words, STRUCK and "" (now refused) replaced by WITHDRAWN and NOT
GRADED."""
import json
import os
import random
import subprocess
import sys
import tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from tpc_biblio import run_case, CSL_KEYS, CSL_TYPES, NAME_KEYS, RESULTS, DECL_KINDS
seed = int(sys.argv[1]) if len(sys.argv) > 1 else 1
count = int(sys.argv[2]) if len(sys.argv) > 2 else 3000
rnd = random.Random(seed)
TEXT = ["a", "b", "Z", "x", " ", " ", "\t", "\n", "\r\n", "e\u0301", "\u00e9",
"\u00c9", "\u212b", "\u00c5", "\u1100\u1161", "\uac00", "\u00a0",
"\u2003", "\u3000", "|", "\\", "\"", "'", "-", "\u2010", "\u2014",
"\u2019", ".", ",", ";", ":", "(", ")", "<", ">", "&", "#", "%",
"$", "+", "=", "^", "~", "_", "/", "\u00fc", "u\u0308", "\u00d8",
"\u00df", "\u03a9", "\u03c9", "\u0301", "\ufb01", "\u91cf", "\u0663",
"5", "0", "1", "\u200b", "\u00ad", "\u202e", "\u0001", "\u007f",
"\u2028", "\u00b0", "\u2030", "\u00d7", "\u00bd", "\u2044",
"<i>", "</i>", "<sup>", "&", "<", "’", "’",
" ", "�", "RETRACTED:", "Retracted: ", "WITHDRAWN ARTICLE:",
"withdrawn", "WITHDRAWN", "Withdrawn.", "unwithdrawn"]
SAFE = ["a", "b", "Z", "x", " ", "e\u0301", "\u00e9", "\u00c9", "\u212b",
"\u1100\u1161", "\u00fc", "\u00d8", "\u03a9", "\u91cf", "5", "-",
".", ",", "(", ")", "<", ">", "&", "%", "'", "\"", "\\", "|",
"\t", "\n", "<i>", "&", "’", "RETRACTED: "]
DOI_SUF = ["a", "B", "z", "0", "9", ".", "-", "_", "(", ")", "<", ">", ";",
":", "/", "%", "%3C", "%3E", "%zz", "%C0%AE", "%25", " ", "\t",
"\u00e9", "e\u0301", "\u200b", "\u202e", "\u00a0", "\u03a9", "~",
"#", "?", "&", "=", "+"]
def pick(xs):
return xs[rnd.randrange(len(xs))]
def r_text(alpha=TEXT, lo=0, hi=8, bad=0.04):
if rnd.random() < bad:
return pick([5, None, True, 1.5, ["a"], {"a": 1}])
return "".join(pick(alpha) for _ in range(rnd.randint(lo, hi)))
def r_safe(lo=1, hi=6):
return "".join(pick(SAFE) for _ in range(rnd.randint(lo, hi)))
def r_digits(lo, hi):
return "".join(pick("0123456789") for _ in range(rnd.randint(lo, hi)))
def r_yymm():
r = rnd.random()
if r < 0.5:
return pick(["0704", "0705", "1412", "1501", "2601", "9912", "0703",
"1500", "1413", "0700", "2613", "9108", "0000"])
return r_digits(4, 4)
def r_version():
return pick(["", "", "", "v1", "v2", "v10", "v0", "v01", "V1", "v", "v1v2"])
def r_arxiv_id():
r = rnd.random()
if r < 0.45:
return r_yymm() + "." + r_digits(3, 6) + r_version()
if r < 0.9:
arch = pick(["hep-th", "math", "cond-mat", "astro-ph", "cs", "q-bio",
"a", "HEP-TH", "hep_th", "physics", "x" * 16, "x" * 17])
sc = pick(["", "", ".GT", ".AG", ".mes-hall", ".G", ".GTX", ".gt"])
return arch + sc + "/" + r_yymm() + r_digits(2, 4) + r_version()
return r_text(lo=0, hi=12, bad=0)
def r_arxiv():
if rnd.random() < 0.03:
return r_text()
v = r_arxiv_id()
return pick(["", "", "arXiv:", "ARXIV:", "https://arxiv.org/abs/",
"http://ARXIV.org/abs/", "arxiv.org/abs/", "arXiv: "]) + v
def r_doi_core():
r = rnd.random()
if r < 0.15:
return "10.48550/" + pick(["arXiv.", "arxiv.", "ARXIV.", "other.", ""]) + r_arxiv_id()
pre = pick(["10.", "10.", "10.", "11.", "10", "1.", "10..", "10.1000.10"]) + \
(r_digits(0, 5) if rnd.random() < 0.8 else "")
if rnd.random() < 0.9:
pre += "/"
suf = "".join(pick(DOI_SUF) for _ in range(rnd.randint(0, 8)))
return pre + suf
def r_doi():
if rnd.random() < 0.05:
return r_text()
core = r_doi_core()
return pick(["", "", "", "doi:", "DOI:", "Doi:", "https://doi.org/",
"http://dx.doi.org/", "HTTPS://DOI.ORG/", "https://example.org/",
"doi: "]) + core
def isbn10_check(nine):
t = sum((10 - i) * int(nine[i]) for i in range(9))
c = (11 - t % 11) % 11
return "X" if c == 10 else str(c)
def isbn13_check(twelve):
t = sum((1 if i % 2 == 0 else 3) * int(twelve[i]) for i in range(12))
return str((10 - t % 10) % 10)
def hyphenate(s):
out = ""
for ch in s:
out += ch
if rnd.random() < 0.15:
out += pick(["-", " ", "\u2010", ""])
return out
def r_isbn():
if rnd.random() < 0.05:
return r_text()
r = rnd.random()
if r < 0.4:
nine = r_digits(9, 9)
c = isbn10_check(nine) if rnd.random() < 0.7 else pick("0123456789Xx")
if rnd.random() < 0.3 and c == "X":
c = "x"
return hyphenate(nine + c)
if r < 0.8:
tw = (pick(["978", "979", "977", "97"]) + r_digits(9, 10))[:12]
c = isbn13_check(tw) if len(tw) == 12 and rnd.random() < 0.7 else pick("0123456789X")
return hyphenate(tw + c)
return hyphenate(r_digits(8, 14))
def r_issn():
if rnd.random() < 0.05:
return r_text()
seven = r_digits(7, 7)
t = sum((8 - i) * int(seven[i]) for i in range(7))
r = t % 11
c = 0 if r == 0 else 11 - r
good = "X" if c == 10 else str(c)
c = good if rnd.random() < 0.7 else pick("0123456789Xx")
if rnd.random() < 0.3 and c == "X":
c = "x"
s = seven[:4] + pick(["-", "-", "-", "", " "]) + seven[4:] + c
return pick(["", "", "ISSN "]) + s
def r_pmid():
if rnd.random() < 0.05:
return r_text()
return pick(["", "", "0", "PMID:", "-", " "]) + r_digits(0, 12) + pick(["", "", " ", "\u200b"])
def r_pmcid():
if rnd.random() < 0.05:
return r_text()
return pick(["PMC", "PMC", "pmc", "PmC", "", "PMCID:"]) + r_digits(0, 9) + pick(["", "", "v1", " "])
GOOD_DOI = ["10.1016/S0140-6736(97)11096-0", "10.1016/s0140-6736(97)11096-0", "10.1000/x", "10.1000/X",
"10.1002/(SICI)1097-0266(199708)18:7<509::AID-SMJ882>3.0.CO;2-Z", "10.5281/zenodo.1234567",
"10.1103/FixRevD.113.012345", "10.1000/\u00e9", "10.1000.10/abc"]
GOOD_ARXIV = ["2601.12345", "2601.12345v2", "2601.12345v3", "hep-th/0701999", "math.GT/0309136v1", "0704.0001"]
GOOD_ISBN = ["0-306-40615-2", "9780306406157", "978-0-306-40615-7", "080442957X", "979-10-90636-07-1"]
GOOD_ISSN = ["0140-6736", "1474-547X", "0378-5955", "2470-0010", "1097-0266"]
GOOD_PMID = ["9500320", "1", "20137807"]
GOOD_PMCID = ["PMC1", "PMC8243230", "pmc1"]
def gv(good, wild, p=0.85):
return pick(good) if rnd.random() < p else wild()
def r_key():
return {"text": r_text(lo=0, hi=14), "kind": pick(["title", "title", "name", "TITLE", 5])}
def r_ids(valid=False):
o = {}
for k, gd, wl in (("DOI", GOOD_DOI, r_doi), ("ARXIV", GOOD_ARXIV, r_arxiv), ("ISBN", GOOD_ISBN, r_isbn),
("ISSN", GOOD_ISSN, r_issn), ("PMID", GOOD_PMID, r_pmid), ("PMCID", GOOD_PMCID, r_pmcid)):
if rnd.random() < 0.35:
o[k] = gv(gd, wl, 0.9 if valid else 0.5)
if rnd.random() < 0.05:
o[pick(["ORCID", "doi", "URL"])] = "x"
if rnd.random() < 0.02:
return pick([[], "x", None])
return o
def r_name():
if rnd.random() < 0.02:
return pick(["x", 5, {}, {"given": "A"}, {"family": ""}, {"family": "A", "comma-suffix": True}])
o = {"family": pick(["Wakefield", "Santos", "M\u00fcller", "Muller", "\u00d8st", "Editor", r_safe(1, 4)])}
if rnd.random() < 0.15:
del o["family"]
o["literal"] = pick(["Fixture Collaboration", r_safe(1, 4)])
for k in ("given", "dropping-particle", "non-dropping-particle", "suffix"):
if rnd.random() < (0.5 if k == "given" else 0.08):
o[k] = r_safe(1, 4) if rnd.random() < 0.98 else r_text(lo=0, hi=3)
return o
def r_date():
if rnd.random() < 0.02:
return pick([{"raw": "1998"}, {"date-parts": [["1998"]]}, {"date-parts": []}, "1998",
{"date-parts": [[1998]], "circa": "yes"}, {"date-parts": [[1998, 13]]},
{"date-parts": [[True]]}, {"date-parts": [[1998], [1999], [2000]]}])
parts = []
for _ in range(pick([1, 1, 1, 2])):
p = [pick([1998, 2026, 2022, 2020, 2026, 1998, 2019, 2001, 1998, 2026, 2022, 2020, -500, 0, 9999] * 6 + [10000, -10000])]
if rnd.random() < 0.6:
p.append(pick([1, 2, 12, 6, 8, 2, 1, 12, 3, 4, 5, 7, 9, 10, 11, 6, 8, 2] * 3 + [13, 0]))
if rnd.random() < 0.5:
p.append(pick([1, 28, 31, 26, 28, 1, 31, 26, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] * 3 + [32]))
parts.append(p)
o = {"date-parts": parts}
if rnd.random() < 0.2:
o["circa"] = pick([True, False])
return o
def r_item(good=0.7):
o = {}
for k in CSL_KEYS:
base = 0.97 if k in ("type", "title", "author", "issued") else 0.15
if rnd.random() >= base:
continue
if k == "type":
o[k] = pick(CSL_TYPES) if rnd.random() < 0.97 else pick(["preprint", 5, ""])
elif k in ("author", "editor", "translator", "container-author", "collection-editor"):
o[k] = [r_name() for _ in range(rnd.randint(1 if k == "author" else 0, 3))] if rnd.random() < 0.98 else {"family": "x"}
elif k in ("issued", "accessed", "original-date"):
o[k] = r_date()
elif k == "DOI":
o[k] = gv(GOOD_DOI + ["10.48550/arXiv.2601.12345", "https://doi.org/10.1000/x", "doi:10.1000/X"], r_doi)
elif k == "ISBN":
o[k] = gv(GOOD_ISBN, r_isbn)
elif k == "ISSN":
o[k] = gv(GOOD_ISSN, r_issn)
elif k == "PMID":
o[k] = gv(GOOD_PMID, r_pmid)
elif k == "PMCID":
o[k] = gv(GOOD_PMCID, r_pmcid)
elif k == "URL":
o[k] = "https://example.org/" + r_safe(0, 4)
elif k == "title":
o[k] = pick(["Ileal-lymphoid-nodular hyperplasia, non-specific colitis, and pervasive developmental disorder in children",
"Vacuum structure of a fixture field", "Fixture Handbook", "Dynamic capabilities: and strategic management",
"The E. coli genome & its structure", r_safe(1, 12), r_safe(1, 12)])
else:
o[k] = r_safe(1, 12) if rnd.random() < 0.97 else r_text(lo=0, hi=4)
if rnd.random() < 0.03:
o[pick(["abstract", "note", "custom", "keyword", "references"])] = "x"
if rnd.random() < 0.01:
return pick([[], "x", None])
return o
def r_caps():
if rnd.random() < 0.02:
return pick([{}, {"entry_max": 1}, {"entry_max": 16384, "title": 0, "text": 500, "name": 200, "url": 2000, "names": 100},
{"entry_max": 16384, "title": 1000, "text": 500, "name": 200, "url": 2000, "names": 100, "x": 1}])
return {"entry_max": pick([16384] * 30 + [300, 100]), "title": pick([1000] * 30 + [8, 3]),
"text": pick([500] * 30 + [8, 3]), "name": pick([200] * 30 + [4, 1]), "url": pick([2000] * 30 + [30]),
"names": pick([100] * 30 + [2, 1])}
def r_existing():
out = []
for _ in range(rnd.randint(0, 3)):
out.append([pick(["DOI", "ARXIV", "ISBN"]), pick(["10.1000/x", "10.1016/s0140-6736(97)11096-0",
"2601.12345", "9780306406157", r_safe()]),
pick(["RR-BIB1", "RR-BIB2", "RR-BIB3"])])
if rnd.random() < 0.015:
out.append(pick([["DOI", "x"], ["ISSN", "x", "RR-BIB1"], "x"]))
return out
def r_entry_ids(item):
"""identifiers likely to match the item's fields."""
if type(item) is not dict or rnd.random() < 0.15:
return r_ids(True)
o = {}
for k in ("DOI", "ISBN", "ISSN", "PMID", "PMCID"):
if k in item and rnd.random() < 0.9:
o[k] = item[k]
if k == "DOI" and type(o[k]) is str and rnd.random() < 0.3:
o[k] = o[k].lower()
if "DOI" in o and type(o["DOI"]) is str and o["DOI"][:8] == "10.48550":
del o["DOI"]
o["ARXIV"] = pick(["2601.12345", "2601.12345v2", "2601.12345v3", "2601.99999"])
if rnd.random() < 0.3:
o["ARXIV"] = gv(GOOD_ARXIV, r_arxiv)
if rnd.random() < 0.05:
o[pick(["DOI", "ISSN", "ISBN"])] = pick(["10.1000/x", "0140-6736", "0-306-40615-2"])
return o
def gen_check():
item = r_item()
sup = pick([None] * 30 + ["RR-BIB1", "RR-BIB3", "RR-BIB1", "RR-BIB3", "", 5])
e = {"item": item, "identifiers": r_entry_ids(item), "supersedes": sup}
if rnd.random() < 0.015:
e = pick([{"item": item, "identifiers": {}}, {"item": item, "identifiers": {}, "supersedes": None, "x": 1}, "x"])
return {"entry": e, "existing": r_existing(), "caps": r_caps()}
def gen_canon():
item = r_item()
if type(item) is dict and rnd.random() < 0.2:
item["id"] = pick(["RR-BIB1", "RR-BIB2"])
return {"bib_id": pick(["RR-BIB1"] * 20 + ["RR-BIB2"] * 10 + ["", 5, "x\u200b"]),
"proposed_by": pick(["DeltaS Zero"] * 20 + ["acct"] * 10 + ["", None]),
"landed_by": pick(["Prime AI"] * 20 + ["acct"] * 10 + [5]),
"landed": pick(["2026-09-26T12:00:00Z"] * 40 + ["now", "", 1]),
"supersedes": pick([None] * 30 + ["RR-BIB1"] * 10 + [5, ""]),
"identifiers": r_entry_ids(item), "item": item}
def gen_url():
tmpl = pick(["https://doi.org/ra/{ID}", "https://api.crossref.org/v1/works/{ID}",
"https://api.datacite.org/dois/{ID}", "https://export.arxiv.org/api/query?id_list={ID}",
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&retmode=json&id={ID}",
"https://pmc.ncbi.nlm.nih.gov/tools/idconv/api/v1/articles/?format=json&ids={ID}",
"http://doi.org/ra/{ID}", "https://doi.org/ra/", "https://doi.org/{ID}/{ID}",
"https://doi.org/ra/{ID}#x", "https://{ID}.doi.org/", "https://doi.org/{id}/{ID}",
"https://doi.org:8443/ra/{ID}", "https://doi.org/ra /{ID}", "https://doi.org{ID}", 5,
"https://d\u00f6i.org/ra/{ID}", "https://-doi.org/ra/{ID}"])
d = {"kind": pick(DECL_KINDS + ("OPENALEX", "DOI")), "template": tmpl}
if rnd.random() < 0.2:
d["name"] = pick(["lookup", "", 5])
if rnd.random() < 0.05:
d = pick([{"kind": "DOI-RA"}, {"template": tmpl}, dict(d, pacing=3), "x"])
k = d.get("kind") if type(d) is dict else None
ident = (r_doi() if k in ("DOI-RA", "DOI-CROSSREF", "DOI-DATACITE") else r_arxiv() if k == "ARXIV"
else r_pmid() if k == "PMID" else r_pmcid())
if rnd.random() < 0.15:
ident = pick([r_doi(), r_arxiv(), r_pmid(), r_pmcid()])
if rnd.random() < 0.4:
ident = pick(GOOD_DOI if k in ("DOI-RA", "DOI-CROSSREF", "DOI-DATACITE") else GOOD_ARXIV if k == "ARXIV"
else GOOD_PMID if k == "PMID" else GOOD_PMCID)
return {"declaration": d, "identifier": ident}
def r_fields(item, ids, kind):
f = {"year": None, "title": None, "subtitle": None, "family": None, "name_form": None, "doi": None,
"issn": [], "isbn": [], "arxiv_doi": None, "arxiv_id": None}
if type(item) is dict:
if rnd.random() < 0.8 and type(item.get("issued")) is dict:
try:
f["year"] = item["issued"]["date-parts"][0][0] + pick([0, 0, 0, 1])
except (KeyError, IndexError, TypeError):
f["year"] = 1998
if rnd.random() < 0.85 and type(item.get("title")) is str:
t = item["title"]
f["title"] = pick(["", "", "RETRACTED: ", "<i>"]) + t + pick(["", "", ".", " x"])
if rnd.random() < 0.2 and ":" in t:
a, b = t.split(":", 1)
f["title"], f["subtitle"] = a, b
names = item.get("author") or item.get("editor")
if rnd.random() < 0.85 and type(names) is list and names and type(names[0]) is dict:
nm = names[0]
fam = nm.get("family") or nm.get("literal") or ""
form = pick(["FAMILY", "GIVEN FAMILY", "FAMILY INITIALS"])
if kind == "ARXIV":
form = "GIVEN FAMILY"
if kind in ("PMID", "PMCID"):
form = "FAMILY INITIALS"
fam = fam if type(fam) is str else "x"
f["family"] = (fam if form == "FAMILY" else
pick(["Sara ", "", "x"]) + fam if form == "GIVEN FAMILY" else
fam + pick([" AJ", "", "x"]))
f["name_form"] = form
if rnd.random() < 0.7 and "DOI" in ids:
f["doi"] = ids["DOI"] if rnd.random() < 0.8 else r_doi()
if type(f["doi"]) is str and rnd.random() < 0.5:
f["doi"] = f["doi"].lower()
if rnd.random() < 0.3 and "ISSN" in ids:
f["issn"] = [ids["ISSN"]] + ([r_issn()] if rnd.random() < 0.5 else [])
elif rnd.random() < 0.2:
f["issn"] = [r_issn()]
if rnd.random() < 0.3 and "ISBN" in ids:
f["isbn"] = [ids["ISBN"]]
if kind == "ARXIV" and rnd.random() < 0.5:
v = ids.get("DOI") if "DOI" in ids and rnd.random() < 0.7 else r_doi()
f["arxiv_doi"] = (pick(["https://doi.org/", ""]) + v) if type(v) is str else v
if kind == "DOI" and rnd.random() < 0.4:
f["arxiv_id"] = ids.get("ARXIV", pick(GOOD_ARXIV)) if rnd.random() < 0.7 else r_arxiv()
for k in ("issn", "isbn"):
f[k] = [x for x in f[k] if type(x) is str and x != ""]
if rnd.random() < 0.02:
f[pick(["year", "title", "family", "name_form"])] = pick([5, "x", None, True])
if rnd.random() < 0.01:
del f[pick(list(f))]
return f
def r_notice():
if rnd.random() < 0.02:
return pick([{"type": "x"}, {"type": "", "identifier": None, "date": None, "source": "x"}, "x",
{"type": "retraction", "identifier": "a b", "date": None, "source": "x"},
{"type": "retraction", "identifier": "-", "date": None, "source": "x"},
{"type": "retraction", "identifier": None, "date": "2010-02-30", "source": "x"}])
return {"type": pick(["retraction", "correction", "expression_of_concern", "Reinstatement",
"Retracted Publication", "Retraction in", "withdrawn", "Partial-Retraction", r_safe()]),
"identifier": pick([None, "10.1016/s0140-6736(10)60175-4", "20137807", "10.1/X"]),
"date": pick([None, "2010-02-06", "2026-03-01", "2024-02-29"]),
"source": pick(["retraction-watch", "publisher", "pubmed", "arxiv"])}
def r_record(kind, item, ids):
ident = ids.get(kind, "x")
if rnd.random() < 0.05:
ident = pick([r_doi(), r_arxiv(), r_pmid(), r_pmcid(), ident.lower() if type(ident) is str else ident])
st = pick(["FOUND"] * 8 + ["NOT FOUND", "NOT FOUND", "FAILED", "FAILED", None, "OK"])
r = {"kind": kind, "identifier": ident}
if kind == "DOI":
ra = pick([{"status": "FOUND", "ra": "Crossref"}] * 6 + [{"status": "FOUND", "ra": "DataCite"}] * 3
+ [{"status": "FOUND", "ra": "mEDRA"}] * 2 + [{"status": "NOT FOUND", "ra": None}] * 2
+ [{"status": "FAILED", "ra": None}, {"status": "FOUND", "ra": None},
{"status": "NOT FOUND", "ra": "Crossref"}, "x"])
r["ra"] = ra
reg = None
if type(ra) is dict and ra["status"] == "FOUND":
reg = {"Crossref": "crossref", "DataCite": "datacite"}.get(ra["ra"], pick([None, "generic"]))
if rnd.random() < 0.02:
reg = pick(["crossref", "datacite", "generic", None, "arxiv"])
r["registry"] = reg
r["status"] = st if reg is not None else (None if rnd.random() < 0.9 else st)
else:
r["registry"] = ("arxiv" if kind == "ARXIV" else "pubmed") if rnd.random() < 0.98 else pick(["crossref", None])
r["status"] = st
if r["status"] == "FOUND":
r["fields"] = r_fields(item, ids, kind)
r["notices"] = [r_notice() for _ in range(rnd.randint(0, 3))]
else:
r["fields"] = None if rnd.random() < 0.95 else r_fields(item, ids, kind)
r["notices"] = [] if rnd.random() < 0.95 else [r_notice()]
if rnd.random() < 0.01:
del r[pick(list(r))]
return r
def gen_compare():
item = r_item()
ids = r_entry_ids(item)
recs = []
if type(ids) is dict:
for k in ("DOI", "ARXIV", "PMID", "PMCID"):
if k in ids and rnd.random() < 0.85:
recs.append(r_record(k, item, ids))
if rnd.random() < 0.05:
recs.append(r_record(pick(["DOI", "ARXIV", "PMID", "PMCID"]), item, ids))
e = {"item": item, "identifiers": ids}
if rnd.random() < 0.01:
e = pick([{"item": item, "identifiers": ids, "supersedes": None}, "x"])
if rnd.random() < 0.01:
recs = pick(["x", None])
return {"entry": e, "records": recs,
"date": pick(["2026-09-26"] * 40 + ["2024-02-29"] * 4 + ["2023-02-29", "2026-13-01", "x", 5])}
def gen_withdrawn():
return {"comment": pick([r_text(lo=0, hi=12), None, "withdrawn", "This paper has been Withdrawn",
"unwithdrawn", "withdrawn5", "\u00e9withdrawn"])}
BIBS = ["RR-BIB1", "RR-BIB10", "RR-BIB2", "RR-BIB3", "RR-BIB4", "RR-BIB5"]
def gen_view():
ents = []
for b in BIBS:
if rnd.random() < 0.8:
ents.append({"bib_id": b, "check": pick(RESULTS * 8 + ("OK",)) if rnd.random() < 0.995 else 5,
"notices": pick([0, 0, 0, 0, 1, 2, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 1] * 4 + [-1]), "primary": pick([True, True, False])})
if rnd.random() < 0.015:
ents.append(dict(ents[0]) if ents else {"bib_id": "RR-BIB1"})
finds = []
have = [e["bib_id"] for e in ents if type(e) is dict and "bib_id" in e] or ["RR-BIB1"]
for _ in range(rnd.randint(0, 4)):
finds.append({"bib_id": pick(have * 20 + ["RR-BIB9"]),
"result": pick((["SOURCE AGREES", "SOURCE DIFFERS"] + ["NOT FOUND"] * 12) * 3 + ["x"]),
"agreed": pick([True, True, False]) if rnd.random() < 0.99 else "yes"})
recs = []
for i in range(rnd.randint(0, 6)):
have = [e["bib_id"] for e in ents if type(e) is dict and "bib_id" in e]
lit = [b for b in (have if rnd.random() < 0.9 else BIBS) if rnd.random() < 0.4] + (["RR-BIB9"] if rnd.random() < 0.02 else [])
if rnd.random() < 0.015 and lit:
lit.append(lit[0])
recs.append({"id": pick(["RR-%d" % i, "RR-%d" % (i + 20), "RR-%d" % (i + 30)] * 4 + ["RR-1", "RR-10", "RR-2", "RR-7"]),
"status": pick(["PROVISIONAL"] * 6 + ["ESTABLISHED"] * 3 + ["WITHDRAWN", "HELD", "REFUTED", "NOT GRADED"]),
"literature": lit})
if rnd.random() < 0.015:
recs.append(pick([{"id": "RR-1", "literature": []}, "x"]))
args = {"records": recs, "entries": ents, "findings": finds}
if rnd.random() < 0.015:
args[pick(list(args))] = pick(["x", None, {}])
return args
def gen_holdlist():
ents = []
for b in BIBS:
if rnd.random() < 0.8:
ents.append({"bib_id": b, "proposed_by": pick(["acct", "other", "lead"]),
"landed_by": pick(["acct", "other", "lead"]), "block_id": pick(["B1", "B2", "B3", "B4"])})
if rnd.random() < 0.03:
ents.append(pick([dict(ents[0]) if ents else {"bib_id": "x"}, {"bib_id": "RR-BIB1", "proposed_by": "acct", "block_id": "B2"}, "x"]))
checks = {}
for b in BIBS:
if rnd.random() < 0.97:
checks[b] = pick(RESULTS * 12 + ("OK",)) if rnd.random() < 0.995 else 5
args = {"entries": ents, "account": pick(["acct"] * 30 + ["other", "other", "nobody", "", 5]),
"blocks": [b for b in ["B1", "B2", "B3", "B4"] if rnd.random() < 0.6], "checks": checks}
if rnd.random() < 0.03:
args[pick(list(args))] = pick(["x", None, [], {}])
return args
GEN = [("doi", lambda: {"text": r_doi()}), ("shape", lambda: {"text": r_doi_core() if rnd.random() < 0.9 else r_text()}),
("arxiv", lambda: {"text": gv(GOOD_ARXIV, r_arxiv, 0.3)}), ("isbn", lambda: {"text": r_isbn()}),
("issn", lambda: {"text": r_issn()}), ("pmid", lambda: {"text": r_pmid()}),
("pmcid", lambda: {"text": r_pmcid()}), ("key", r_key), ("dupe", lambda: {"identifiers": r_ids()}),
("check", gen_check), ("canon", gen_canon), ("preimage", gen_canon), ("url", gen_url),
("compare", gen_compare), ("withdrawn", gen_withdrawn), ("view", gen_view), ("holdlist", gen_holdlist)]
cases = []
for fn, g in GEN:
for _ in range(count):
cases.append({"fn": fn, "args": g()})
CASES_PATH = os.environ.get("TPC_DIFF_CASES") # named: kept, for crosspy.py
if not CASES_PATH:
fd, CASES_PATH = tempfile.mkstemp(prefix="tpc_diff_", suffix=".json") # outside the tree
os.close(fd)
with open(CASES_PATH, "w", encoding="utf-8") as f:
json.dump(cases, f)
py = [run_case(c["fn"], c["args"]) for c in cases]
php = json.loads(subprocess.run(["php", os.path.join(HERE, "diff_biblio.php"), CASES_PATH], capture_output=True, text=True,
check=True).stdout)
if not os.environ.get("TPC_DIFF_CASES"):
os.remove(CASES_PATH)
assert len(php) == len(py)
first = None
total = 0
for fn, _ in GEN:
idx = [i for i, c in enumerate(cases) if c["fn"] == fn]
diffs = [i for i in idx if py[i] != php[i]]
total += len(diffs)
cover = {}
for i in idx:
out = py[i]
if out.startswith("ERROR"):
keys = [out[6:].split(":")[0] if fn in ("check", "canon", "preimage", "dupe") else "ERROR"]
elif fn == "doi":
keys = [out.split("|")[1]]
elif fn in ("shape", "withdrawn"):
keys = [out.split("|")[1]]
elif fn == "arxiv":
keys = [out.split("|")[4] + ("+v" if out.split("|")[2] != "0" else "")]
elif fn == "isbn":
keys = ["given" + out.split("|")[2]]
elif fn == "dupe":
keys = [out.split("|")[1]]
elif fn == "check":
keys = ["OK:" + out.split("\n")[0].split("|")[2]]
elif fn == "compare":
keys = [out.split("\n")[0][8:]]
keys += sorted({ln.split("|")[2] for ln in out.split("\n")[1:] if ln[:1] in "DAPI"})
keys += ["NOTICES" if "\nnotice|" in out else "no-notice"]
elif fn == "view":
keys = ["n" + out.split("\n")[0][5:]]
elif fn == "holdlist":
keys = ["n" + out.split("\n")[0][9:]]
else:
keys = ["OK"]
for k in keys:
cover[k] = cover.get(k, 0) + 1
print("seed %d %-9s cases %d differences %d" % (seed, fn, len(idx), len(diffs)))
print(" " + ", ".join("%s %d" % (k, cover[k]) for k in sorted(cover)))
if diffs and first is None:
first = diffs[0]
print("seed %d total differences %d" % (seed, total))
if first is not None:
print(json.dumps(cases[first], ensure_ascii=True))
print(py[first])
print("---")
print(php[first])