#!/usr/bin/env python3 """ Root & Branch adoption validator — check a submission before sending it to anyone. Runs on any Python 3.8+ with nothing installed. It talks to no network and sends nothing anywhere: it reads your file, tells you whether it conforms, and tells you exactly what would and would not be published. Run it as many times as you like. python3 validate.py entries.csv --program program.json python3 validate.py submission.json WHY A VALIDATOR RATHER THAN A CONVERSATION A program should be able to adopt this without asking anyone's permission and without waiting on a reply. If this script says the file conforms, it conforms. Nobody at the register gets to decide otherwise after the fact. WHAT IT CHECKS, AND WHAT IT REFUSES TO GUESS Required fields, year ranges, URL shape, the consent vocabulary, and the exact wording of the program attestation. It will not infer a missing name, will not normalise a practice into an occupation code, and will not treat a blank consent field as permission. A row it cannot judge is an error, not a default. """ import argparse, csv, datetime, json, os, re, sys from collections import Counter FORMAT_VERSION = "0.1" STATEMENT = ( "I am authorized to make this statement for the program named above. Each entry " "in this file records a pairing the program administered or funded. The program " "does not attest that any practice was performed well, and makes no statement " "about any participant's skill. Every entry is drawn from material the program " "has already published, or from records the program holds and is free to share." ) CONSENT = { "published_by_program": ("publishes", "already public; the register mirrors and links back"), "participants_consented_to_register": ("publishes", "consent on file covering a third-party register"), "participant_declined": ("withheld", "never published; carried so your totals reconcile"), "unknown": ("counted only", "held as an aggregate count, never as a name"), } URL_RE = re.compile(r"^https?://[^\s/$.?#].[^\s]*$", re.I) JUR_RE = re.compile(r"^[A-Z]{2}$") ENTRY_FIELDS = { "local_id", "cycle_year", "master_name", "successor_name", "successor_count", "practice", "location", "source_url", "consent", "notes", } REQUIRED_ENTRY = {"cycle_year", "practice", "source_url", "consent"} class Problems: def __init__(self): self.errors, self.warnings = [], [] def err(self, where, msg, fix=""): self.errors.append((where, msg, fix)) def warn(self, where, msg, fix=""): self.warnings.append((where, msg, fix)) def check_program(p, pr): if not isinstance(p, dict): pr.err("program", "the program block is missing or is not an object") return for k in ("organization", "program_name", "public_url", "jurisdiction"): if not str(p.get(k, "")).strip(): pr.err("program." + k, "required and empty") if p.get("public_url") and not URL_RE.match(str(p["public_url"])): pr.err("program.public_url", f"not a URL: {p['public_url']!r}", "an https:// address for the program's own page") if p.get("jurisdiction") and not JUR_RE.match(str(p["jurisdiction"])): pr.err("program.jurisdiction", f"expected a two-letter code, got {p['jurisdiction']!r}", "MO, TX, AL, or a country code for programs outside the US") if "@" in str(p.get("contact_route", "")) and "." not in str(p.get("contact_route", "")): pr.warn("program.contact_route", "does not look like a working address") def check_attestation(a, pr): if not isinstance(a, dict): pr.err("attestation", "missing") return for k in ("by_name", "by_title", "date"): if not str(a.get(k, "")).strip(): pr.err("attestation." + k, "required and empty") d = str(a.get("date", "")) if d: try: datetime.date.fromisoformat(d) except ValueError: pr.err("attestation.date", f"not a date: {d!r}", "YYYY-MM-DD") got = " ".join(str(a.get("statement_accepted", "")).split()) if got != " ".join(STATEMENT.split()): pr.err("attestation.statement_accepted", "does not match the required wording exactly", "copy the statement from the kit without editing it; if the program " "cannot accept it as written, the format is not right for this program " "and nothing should be sent") def check_entry(e, n, pr): where = f"entry {n}" extra = set(e) - ENTRY_FIELDS if extra: pr.err(where, "unrecognised field(s): " + ", ".join(sorted(extra)), "remove them, or put them in notes") for k in REQUIRED_ENTRY: if not str(e.get(k, "")).strip(): pr.err(where, f"{k} is required and empty") y = str(e.get("cycle_year", "")).strip() if y: if not y.isdigit(): pr.err(where, f"cycle_year is not a number: {y!r}") elif not (1900 <= int(y) <= 2100): pr.err(where, f"cycle_year out of range: {y}") u = str(e.get("source_url", "")).strip() if u and not URL_RE.match(u): pr.err(where, f"source_url is not a URL: {u!r}", "the page where this pairing is published") c = str(e.get("consent", "")).strip() if c and c not in CONSENT: pr.err(where, f"consent is not one of the four values: {c!r}", "one of: " + ", ".join(CONSENT)) m, s = str(e.get("master_name", "")).strip(), str(e.get("successor_name", "")).strip() for label, v in (("master_name", m), ("successor_name", s)): if v and re.search(r"^(n/?a|none|unknown|tbd|not published|-+)$", v, re.I): pr.err(where, f"{label} contains a placeholder: {v!r}", "leave the field out entirely; a blank is a fact, a placeholder is noise") if not m and not s: pr.warn(where, "neither party is named", "the entry is still accepted and counted, but it cannot carry a Lineage Number") cnt = str(e.get("successor_count", "")).strip() if cnt and s: pr.err(where, "successor_count is for cohorts, and a successor is named", "use one or the other") if cnt and not cnt.isdigit(): pr.err(where, f"successor_count is not a number: {cnt!r}") def load(path, program_path): if path.lower().endswith(".json"): with open(path) as f: doc = json.load(f) return doc, None if not program_path: return None, ("A CSV carries only the entries, so the program block has to come " "from somewhere.\nAdd --program program.json (the kit has a template).") with open(program_path) as f: head = json.load(f) rows = [] with open(path, newline="", encoding="utf-8-sig") as f: for r in csv.DictReader(f): rows.append({k.strip(): (v or "").strip() for k, v in r.items() if k and k.strip()}) head["entries"] = rows head.setdefault("format_version", FORMAT_VERSION) return head, None def main(): ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("file", help="entries.csv or submission.json") ap.add_argument("--program", help="program.json, when the entries are a CSV") a = ap.parse_args() if not os.path.exists(a.file): print(f"No such file: {a.file}", file=sys.stderr) return 2 try: doc, problem = load(a.file, a.program) except json.JSONDecodeError as e: print(f"That file is not valid JSON — {e}", file=sys.stderr) return 2 if problem: print(problem, file=sys.stderr) return 2 pr = Problems() if str(doc.get("format_version", "")) != FORMAT_VERSION: pr.err("format_version", f"expected {FORMAT_VERSION!r}, got {doc.get('format_version')!r}") check_program(doc.get("program"), pr) check_attestation(doc.get("attestation"), pr) entries = doc.get("entries") or [] if not entries: pr.err("entries", "no entries in the file") for i, e in enumerate(entries, start=1): check_entry(e, i, pr) tally = Counter(str(e.get("consent", "")).strip() for e in entries) named = sum(1 for e in entries if str(e.get("master_name", "")).strip() and str(e.get("successor_name", "")).strip()) print(f"\n{a.file}: {len(entries)} entries\n") if pr.errors: print(f"{len(pr.errors)} error(s) — nothing should be sent until these are fixed:\n") for w, m, fix in pr.errors[:60]: print(f" {w}: {m}") if fix: print(f" → {fix}") if len(pr.errors) > 60: print(f" … and {len(pr.errors) - 60} more") print() if pr.warnings: print(f"{len(pr.warnings)} note(s) — not errors, just things worth knowing:\n") for w, m, fix in pr.warnings[:20]: print(f" {w}: {m}") if fix: print(f" → {fix}") if len(pr.warnings) > 20: print(f" … and {len(pr.warnings) - 20} more") print() print("What would happen to these entries:\n") for k, (action, why) in CONSENT.items(): n = tally.get(k, 0) if n: print(f" {str(n).rjust(5)} {action.ljust(12)} {k} — {why}") unset = sum(v for k, v in tally.items() if k not in CONSENT) if unset: print(f" {str(unset).rjust(5)} rejected consent missing or not recognised") print(f"\n {named} {'entry names' if named == 1 else 'entries name'} both parties " f"and can carry a Lineage Number.") if pr.errors: print("\nNot conforming. Fix the errors above and run this again.") return 1 print("\nConforming. This file can be submitted as it stands.") print("Nothing has been sent anywhere — this script does not touch the network.") return 0 if __name__ == "__main__": sys.exit(main())