The Principia Machina/TPC-PY tools/verify tpc py.py: Difference between revisions

Latest revision as of 02:11, 30 September 2026

# -*- coding: utf-8 -*-
"""verify_tpc_py 0.3.0: check a TPC-PY download against its digests (CO-8;
TPS 5.11.9; PMP-TPM TPC: "An agent verifies the digest before running it").

0.3.0 (agent TR, 2026-09-28; TPC-PY 0.2.0's release): with --fetch or
--index, --expect-tps V and --expect-version L also check the index
page's entry (TPS 5.11.9: version and conformed TPS version): its lines
"* Conformed TPS version: V" and "* TPC-PY version: L (...)" (the latter
written by build_tpc_py_pages 0.3.0 with --version-label; an index
without it fails --expect-version). The index's status line is printed.
Everything else is 0.2.0's.

Three ways to run it:
  python3 verify_tpc_py.py DIR [--manifest tpc_py_manifest.json |
                               --index INDEX_WIKITEXT] [--expect-tree HEX]
      checks the files already in DIR (a folder laid out as tpc/); with
      neither --manifest nor --index, every file found in DIR is listed and
      only the tree digest checks them (so give --expect-tree).
      [--expect-tps V] [--expect-version L] with --index or --fetch.
  python3 verify_tpc_py.py --sums SHA256SUMS DIR
      checks every file a sha256sum-form list names, relative to DIR (for
      a whole kit, tpc_php/ included); no tree digest.
  python3 verify_tpc_py.py --fetch INDEX_PHP_URL DIR [--expect-tree HEX]
      reads the index page "The Principia Machina/TPC-PY" and every source
      page it lists as raw text (INDEX_PHP_URL?title=...&action=raw), saves
      each under its path in DIR (which must be empty or absent), then
      checks them. INDEX_PHP_URL is the wiki's index.php, for example
      https://invarians.org/w/index.php (VERIFY the path on the host). A
      file the index lists under "Files kept in pages" is fetched from its
      numbered pages <title>/1, /2, ... and joined in order.

The expected SHA-256 of each file comes from the manifest JSON the build
wrote, or from the index page's manifest block (<pre>, sha256sum form).
The tree digest is computed over the files the manifest lists, under the
convention the index page states: SHA-256 over the netstring encoding
(TPS 2.5.10) of ["TPC-PY-TREE", "1", path_1, sha256_1, ...], paths
relative to tpc/ with "/" separators, in code-point order. A manifest read
from the same wiki proves only that the pages agree with each other; the
anchor is the tree digest H-DECL declares for the version (TPS 5.11.9):
pass it as --expect-tree, or compare the printed value with the
declaration by eye.

Exit 0 only when every listed file is present with its SHA-256, no file in
DIR is unlisted (__pycache__ and *.pyc ignored), and, when --expect-tree is
given, the tree digest equals it. Standard library only; Python 3.8+;
Windows and Linux (files are read and written as bytes, so no line-end
translation touches them)."""
import hashlib
import json
import os
import re
import sys
import urllib.parse
import urllib.request

VERSION = "0.3.0"
BASE = "The Principia Machina/TPC-PY"
TAG, FORM = "TPC-PY-TREE", "1"
LINE = re.compile(r"^([0-9a-f]{64})  (\S+)$")


def netstring_digest(fields):
    h = hashlib.sha256()
    for f in fields:
        b = f.encode("utf-8")
        h.update(str(len(b)).encode("ascii") + b":" + b)
    return h.hexdigest()


def tree_digest(entries):
    fields = [TAG, FORM]
    for path, sha in sorted(entries):
        fields += [path, sha]
    return netstring_digest(fields)


def manifest_from_index(text):
    """The <pre> block under "== Manifest ==": one "sha256  path" per line."""
    m = re.search(r"== Manifest ==\s*<pre>\n(.*?)\n</pre>", text, re.S)
    if not m:
        raise ValueError("the index text holds no manifest block")
    out = []
    for ln in m.group(1).split("\n"):
        mm = LINE.match(ln)
        if not mm:
            raise ValueError("not a manifest line: %r" % ln[:80])
        out.append((mm.group(2), mm.group(1)))
    return out


def pages_from_index(text):
    """The <pre> block under "== Files kept in pages ==", if any: one
    "n  path" per line (the file is the n numbered pages joined)."""
    m = re.search(r"== Files kept in pages ==.*?<pre>\n(.*?)\n</pre>", text, re.S)
    out = {}
    if m:
        for ln in m.group(1).split("\n"):
            mm = re.match(r"^([1-9][0-9]{0,3})  (\S+)$", ln)
            if not mm:
                raise ValueError("not a pages line: %r" % ln[:80])
            out[mm.group(2)] = int(mm.group(1))
    return out


def index_entry(text):
    """The index's status line, conformed TPS version and TPC-PY version."""
    st = re.search(r"^'{3}Status: (.*?)'{3}", text, re.M)
    tps = re.search(r"^\* Conformed TPS version: (\S+)$", text, re.M)
    ver = re.search(r"^\* TPC-PY version: (\S+) \((proposed|declared)\)$", text, re.M)
    return (st.group(1) if st else None, tps.group(1) if tps else None,
            (ver.group(1), ver.group(2)) if ver else None)


def entry_check(text, opt):
    """--expect-tps and --expect-version against the index; True if every
    expectation given holds."""
    st, tps, ver = index_entry(text)
    print("index: status %s; conformed TPS %s; TPC-PY version %s" % (
        st or "-", tps or "-", ("%s (%s)" % ver) if ver else "-"))
    ok = True
    if opt.get("--expect-tps") is not None:
        if tps == opt["--expect-tps"]:
            print("conformed TPS version equals the expected value")
        else:
            print("CONFORMED TPS VERSION DIFFERS from the expected %s" % opt["--expect-tps"])
            ok = False
    if opt.get("--expect-version") is not None:
        if ver and ver[0] == opt["--expect-version"]:
            print("TPC-PY version equals the expected value")
        else:
            print("TPC-PY VERSION DIFFERS from the expected %s" % opt["--expect-version"])
            ok = False
    return ok


def sums_list(path):
    out = []
    for ln in open(path, encoding="utf-8").read().splitlines():
        if not ln.strip():
            continue
        mm = re.match(r"^([0-9a-f]{64}) [ *](\S+)$", ln)
        if not mm:
            raise ValueError("not a sha256sum line: %r" % ln[:80])
        out.append((mm.group(2), mm.group(1)))
    return out


def safe_path(root, rel):
    """rel must be a relative path of plain segments: no "..", no absolute
    path, no backslash (the listing is untrusted input)."""
    if (not rel or rel.startswith("/") or "\\" in rel or ":" in rel
            or any(seg in ("", ".", "..") for seg in rel.split("/"))):
        raise ValueError("refused path %r" % rel)
    return os.path.join(root, *rel.split("/"))


def fetch(url, title):
    q = urllib.parse.urlencode({"title": title, "action": "raw"})
    req = urllib.request.Request(url + ("&" if "?" in url else "?") + q,
                                 headers={"User-Agent": "verify_tpc_py/%s (TPM; TPC-PY check)" % VERSION})
    with urllib.request.urlopen(req, timeout=60) as r:
        return r.read()


def check(root, listed, expect_tree, tree=True, ignore=()):
    ok = True
    names = set()
    found = []
    for path, sha in listed:
        p = safe_path(root, path)
        names.add(path)
        if not os.path.isfile(p):
            print("MISSING   %s" % path)
            ok = False
            continue
        got = hashlib.sha256(open(p, "rb").read()).hexdigest()
        found.append((path, got))
        if got != sha:
            print("DIFFERS   %s  expected %s  got %s" % (path, sha, got))
            ok = False
    for r, dirs, files in os.walk(root):
        dirs[:] = [d for d in dirs if d != "__pycache__"]
        for n in files:
            if n.endswith(".pyc"):
                continue
            rel = os.path.relpath(os.path.join(r, n), root).replace(os.sep, "/")
            if rel not in names and rel not in ignore:
                print("UNLISTED  %s" % rel)
                ok = False
    if len(found) < len(listed):
        print("files listed: %d, found: %d; tree digest not computed" % (len(listed), len(found)))
        print("RESULT: FAILURES")
        return 1
    if not tree:
        print("files listed: %d; every one found" % len(listed))
        print("RESULT:", "ALL MATCH" if ok else "FAILURES")
        return 0 if ok else 1
    td = tree_digest(found)       # over the files as found, not as listed
    print("files listed: %d; tree digest of the files found: %s" % (len(listed), td))
    if expect_tree:
        if td == expect_tree.lower():
            print("tree digest equals the expected value")
        else:
            print("TREE DIGEST DIFFERS from the expected %s" % expect_tree)
            ok = False
    print("RESULT:", "ALL MATCH" if ok else "FAILURES")
    return 0 if ok else 1


def main(argv):
    a = argv[1:]
    opt = {}
    pos = []
    i = 0
    while i < len(a):
        if a[i] in ("--manifest", "--index", "--expect-tree", "--fetch", "--sums",
                    "--expect-tps", "--expect-version"):
            if i + 1 >= len(a):
                sys.exit("missing value for %s" % a[i])
            opt[a[i]] = a[i + 1]
            i += 2
        else:
            pos.append(a[i])
            i += 1
    if len(pos) != 1:
        sys.exit(__doc__.split("Three ways to run it:")[1].split("The expected")[0])
    root = pos[0]
    entry_ok = True
    if ("--expect-tps" in opt or "--expect-version" in opt) and not (
            "--fetch" in opt or "--index" in opt):
        sys.exit("--expect-tps and --expect-version need --fetch or --index")
    if "--sums" in opt:
        rel = os.path.relpath(os.path.abspath(opt["--sums"]), os.path.abspath(root))
        return check(root, sums_list(opt["--sums"]), None, tree=False,
                     ignore=(rel.replace(os.sep, "/"),))
    if "--fetch" in opt:
        url = opt["--fetch"]
        if os.path.exists(root) and os.listdir(root):
            sys.exit("refused: %s is not empty" % root)
        index = fetch(url, BASE).decode("utf-8")
        entry_ok = entry_check(index, opt)
        listed = manifest_from_index(index)
        split = pages_from_index(index)
        n = 0
        for path, _ in listed:
            p = safe_path(root, path)
            os.makedirs(os.path.dirname(p) or ".", exist_ok=True)
            if path in split:
                data = b"".join(fetch(url, "%s/%s/%d" % (BASE, path, k))
                                for k in range(1, split[path] + 1))
                n += split[path]
            else:
                data = fetch(url, BASE + "/" + path)
                n += 1
            with open(p, "wb") as f:
                f.write(data)
        print("fetched the index and %d pages (%d files) into %s" % (n, len(listed), root))
    elif "--manifest" in opt:
        m = json.load(open(opt["--manifest"], encoding="utf-8"))
        listed = [(f["path"], f["sha256"]) for f in m["files"]]
        if m.get("tree_digest") and not opt.get("--expect-tree"):
            print("manifest's tree digest: %s (not the declared value unless it matches"
                  " the declaration)" % m["tree_digest"])
    elif "--index" in opt:
        index = open(opt["--index"], encoding="utf-8").read()
        entry_ok = entry_check(index, opt)
        listed = manifest_from_index(index)
    else:
        listed = []
        for r, dirs, files in os.walk(root):
            dirs[:] = [d for d in dirs if d != "__pycache__"]
            for n in files:
                if not n.endswith(".pyc"):
                    p = os.path.join(r, n)
                    listed.append((os.path.relpath(p, root).replace(os.sep, "/"),
                                   hashlib.sha256(open(p, "rb").read()).hexdigest()))
        print("no manifest given: every file of %s listed as found" % root)
    rc = check(root, listed, opt.get("--expect-tree"))
    if not entry_ok:
        print("RESULT: FAILURES (the index's entry)")
        return 1
    return rc


if __name__ == "__main__":
    sys.exit(main(sys.argv))

# END OF ARTIFACT --- verify_tpc_py 0.3.0
# If this line is absent, the payload arrived truncated.