diff --git a/.gitignore b/.gitignore index 32ba4299..ce47748b 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,27 @@ docs/research/provision-matching/probes/merged_labels.json # No trailing slash, per the #319 symlink reasoning above. docs/research/pdf-backend-bakeoff/holdout +# External-validity probe evidence (A46). Running any xNN probe rewrites its own evidence +# file, so after the A46 cleanup these reappear as untracked artifacts and a later `git add -A` +# would silently re-commit the working material that was retired. Ignored rather than left +# loose. They are regenerated by running the probe; git history holds the removed versions. +# +# The negations are the artifacts something actually READS or that a byte-frozen document +# cites — dropping one un-tracks a live gate input, so only change this list together with +# the consumer that justifies it. A new committed artifact needs a new negation here. +docs/research/pdf-backend-bakeoff/validation/external-validity/results/x*.json +# G2 reads this one; G6 defects ORACLE_INTEGRATION_NOT_VERIFIED without the other. +!docs/research/pdf-backend-bakeoff/validation/external-validity/results/x2_contract_assertions.json +!docs/research/pdf-backend-bakeoff/validation/external-validity/results/x26_control_oracle.json +# Cited as MEASURED by PRE-REGISTRATION.md, which is byte-frozen and can never be repointed. +!docs/research/pdf-backend-bakeoff/validation/external-validity/results/x00_design_pilot.json +!docs/research/pdf-backend-bakeoff/validation/external-validity/results/x02_oracle_reference_defects.json +# Named by retained METHODOLOGY_SURFACE files (cross_engine_control, run_hybrid, run_extended, +# reconstruct_extended_corrected), which may not be edited for a cosmetic reason. +!docs/research/pdf-backend-bakeoff/validation/external-validity/results/x09_skeleton_cross_engine.json +!docs/research/pdf-backend-bakeoff/validation/external-validity/results/x11_provenance_chain.json +!docs/research/pdf-backend-bakeoff/validation/external-validity/results/x13_x_arm.json + # Build artifacts. The project became buildable in #398, so `uv build` now produces # these where it never used to. `build/` matters beyond tidiness: some backends copy # the package's own .py files into it, and a directory holding Python that is neither diff --git a/AGENTS.md b/AGENTS.md index 17ea9bd0..ab370cf4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,13 +89,16 @@ A comment describes the code as it is **now**. History and rejected alternatives Probes, audits, spikes and study write-ups are **working material, not automatically permanent repository material.** At closure, retain an artifact only if it is needed to reproduce a consequential result, enforce an invariant, document a durable decision, or serve as a frozen input. **Delete the rest** — Git history preserves the investigative record. Retention is the exception that needs a reason, and "it was expensive to produce" is not one. +Apply that strictly: "useful history", "might be interesting later", "shows how we got here" and "was once reviewed" are not retention reasons, because every one of them is satisfied by `git log`. Judge an artifact by the consequential role it plays **now**, never by its filename, its age, or how much history it carries. The inverse error is just as real, so decide by tracing consumers — under `docs/research/` a generated JSON that looks like a stale run log may be a committed expected-output oracle a gate reads, and a probe that looks superseded may be the only executable negative control of a rule still in force. Before removing one, check imports, direct path reads, subprocess calls, gate inputs, and any doc that names it as a *current* requirement. + When deleting research, **remove or update live references to it**, and move any durable conclusion into its authoritative current home: an ADR, the architecture documentation, an executable test, or a frozen fixture. Do not write an archival summary, a tombstone map or a closure document whose only job is to record what was deleted — that is a new artifact with the same problem, and a pointer into history is one more thing that can be wrong. -The failure this prevents is subtler than rot. A probe that no longer runs at least announces itself; a probe that still runs can publish a quantity the project has since disowned, and passing a check makes it look current. So the question at closure is not "does it work?" but "does it still answer a live question?" +The failure this prevents is subtler than rot. A probe that no longer runs at least announces itself; a probe that still runs can publish a quantity the project has since disowned, and passing a check makes it look current. So the question at closure is not "does it work?" but "does it still answer a live question?" `tests/test_research_probes.py` runs the probes it declares runnable against a closed manifest, so a probe added later is either run or the gate fails. -So the gate executes (which catches the first two), and retirement is judged on whether the artifact still answers a live question (which catches the third). `tests/test_research_probes.py` runs the probes it declares runnable against a closed manifest, so a probe added later is either run or the gate fails. +Two rules that decide the ambiguous cases: -When retiring an artifact, record in the document that cited it **which executable test inherited its question**, and pin the commit it stays readable at. A reproduction claim pointing at a deleted file is the defect this is meant to avoid, not the price of avoiding it. +- **Prove claimed regenerability rather than assuming it.** If an artifact is being deleted because surviving machinery can reproduce it, re-run the producer and confirm that claim. Failure to regenerate is a reason to re-check the four criteria above; it is not, by itself, a retention reason — a one-off diagnostic that nothing needs is just as disposable when its producer is gone. +- **A dangling reference is acceptable only where the referring document can be appended to.** If a byte-frozen document (a pre-registration, a sealed manifest) cites the artifact, it can never be given a pointer to git history — so retain the artifact instead. An append-only ledger can simply record where the removed file resolves, which is the one place a history pointer earns its keep. ## Architecture decisions diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/HARNESS-PLAN.md b/docs/research/pdf-backend-bakeoff/validation/external-validity/HARNESS-PLAN.md index 1ad435e1..98892d08 100644 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/HARNESS-PLAN.md +++ b/docs/research/pdf-backend-bakeoff/validation/external-validity/HARNESS-PLAN.md @@ -45,8 +45,8 @@ never re-derives an earlier stage's decisions. ## 2. `build_frames.py` — **BUILT (A31)** -Implemented and tested on SYNTHETIC + DEVELOPMENT material only; evidence in -`results/x17_build_frames.json`, **56/56 controls passing**. No canonical `results/frames.json` +Implemented and tested on SYNTHETIC + DEVELOPMENT material only; evidence is regenerated by +running `x17_build_frames.py`, **56/56 controls passing**. No canonical `results/frames.json` exists. The D-frame is emitted as a **complete census** — the A10/A27.3 budget is not applied here. Anchor equality uses the **whole emitted production `Anchor` value**; no reduced signature was invented. No runner instrumentation was added: each arm's anchors come from the @@ -372,7 +372,7 @@ a pre-committed sentence, and must refuse to emit any comparative-accuracy claim ### RESOLVED by measurement — 7.4 anchor→neutral bridge -**Proven, not argued** (`x14_anchor_bridge.py`, `results/x14_anchor_bridge.json`): +**Proven, not argued** (run `x14_anchor_bridge.py`): ``` Anchor(page_number, line_number) <- line_number is the GPO PRINTED margin number @@ -426,7 +426,7 @@ enforce it. | §4.5 P-head was a caller obligation | **A30.4** — `filter_keys` now takes `(key, kind, population)` and applies both restrictions itself | | blind-ID uniqueness was only synthetic | **A30.5** — asserted over the **realized** set; a collision aborts the build | -**Proven, not argued** (`x16_occurrence_identity.py`, `results/x16_occurrence_identity.json`): +**Proven, not argued** (run `x16_occurrence_identity.py`): ``` instrumented mirror == production extract_anchors every DEVELOPMENT page consumed diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/PRE-EXECUTION-AMENDMENTS.md b/docs/research/pdf-backend-bakeoff/validation/external-validity/PRE-EXECUTION-AMENDMENTS.md index 33f85829..dafdb335 100644 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/PRE-EXECUTION-AMENDMENTS.md +++ b/docs/research/pdf-backend-bakeoff/validation/external-validity/PRE-EXECUTION-AMENDMENTS.md @@ -6071,6 +6071,74 @@ opening any document and nothing measured, since the gate reads keys only. --- +## A46 — SUBSTANTIVE. Design-era working material is retired from the tree + +```json +{"id": "A46", "class": "SUBSTANTIVE", + "commits": ["4b808e2"], + "confirmatory_output_at_time": "none", + "affects_membership": false, "affects_scoring_rule": false, + "files_touched": ["probes/x06_m6_feasibility.py", "probes/x19_raster_edge_diagnostic.py", + "HARNESS-PLAN.md"], + "supersedes_text_in": "none -- NO frozen rule changes. No metric, threshold, denominator, control fixture, population, adjudication route, decision rule or F/G gate moves. G5's surface stays 15 files and no member of it is touched", + "status": "COMPLETE. Boundary ABSENT, execution FORBIDDEN"} +``` + +**The retention rule being applied.** Research artifacts are working material, not +automatically permanent repository material. At closure, an artifact is retained only if it +is needed to reproduce a consequential result, enforce an invariant, document a durable +decision, or serve as a frozen input — plus, here, anything the still-active Run 2 execution +path needs. Git history preserves the investigative record. "Useful history", "shows how we +got here" and "was once reviewed" are not retention reasons. + +**Two probes are removed**, and neither owns a live invariant: + +- **`x06_m6_feasibility.py`** measured whether M6 could be adjudicated inside a 6–10 line + region. **A20 STRUCK M6**, and HARNESS-PLAN implements "§6 (M0–M9 minus M6)", so the + question is no longer in the methodology. It also drives the superseded `pdfium_hybrid` / + `reconstruct_hybrid` arms rather than the frozen H/X pair, so it does not exercise any + current component. A17.1's table transcribes its finding inline. +- **`x19_raster_edge_diagnostic.py`** asked whether the visible left edge is displaced from + the geometric `x0`. It **states no pass threshold** by construction, and the ruling it + informed — A30.3 stands, no tolerance — is frozen in A32. Its 16 development raster PNGs + are referenced only by its own evidence file. + +**Twenty generated evidence artifacts are removed.** Their only role was recording that an +earlier probe run passed; nothing reads them. Each is regenerated by a probe that remains in +the tree, and regenerability was measured rather than assumed: every committed probe artifact +in this study was re-run before the deletion and rewrote **byte-identically**, leaving the +tree clean. + +**Earlier links in this ledger resolve in git history.** A17/A19/A21/A22/A23/A24.2 and A41 +link `results/x06_m6_feasibility.json`, `x07_neutral_geometry.json`, +`x08_neutral_identity.json`, `x10_reconstruction_signature.json`, +`x12_skeleton_eligibility.json` and `x27_score_metrics.json` as MEASURED evidence. Those +amendments are **not rewritten** — the ledger is an append-only record, and editing a past +entry to hide a removal would be the defect, not the fix. Read those links at **`4b808e2~1`**. +Each amendment already transcribes its numbers inline, so the reasoning stands without them. + +**What was deliberately NOT removed, and why.** The line is that a dangling reference is +acceptable only where the referring document can be appended to, as this ledger can: + +| retained | why it could not go | +|---|---| +| `holdout/`, `control_fixtures/`, `holdout_membership.json`, `contamination.json`, `design_exposure.json`, `control_fixtures.json` | frozen inputs to F1/F2/F3/F7/F8/F10/F11, G4 and G6 | +| `results/x2_contract_assertions.json` | G2 reads it | +| `results/x26_control_oracle.json` | G6 defects `ORACLE_INTEGRATION_NOT_VERIFIED` without it | +| `results/design_runs/*.log` | `x05` **derives** the design-exposure list from them; a primary source, not a log dump | +| `results/x00_design_pilot.json`, `results/x02_oracle_reference_defects.json` | cited as MEASURED by **`PRE-REGISTRATION.md`**, which is byte-frozen and can never carry a pointer to git history | +| `results/x09_skeleton_cross_engine.json`, `x11_provenance_chain.json`, `x13_x_arm.json` | named by retained METHODOLOGY_SURFACE files; repointing those would edit a result-bearing file for a cosmetic reason | +| every other `xNN` probe | each is the executable control over a retained live component | + +**Why SUBSTANTIVE and not TOOLING or CLERICAL.** All three protected files are already +declared under SUBSTANTIVE amendments (A17/A18/A20, A32, A27/A28/A30/A31/A33/A34), and x04 +refuses a file declared under both a SUBSTANTIVE and a TOOLING amendment. CLERICAL would +misdescribe the removal of two executable probes as a typo fix. The change alters no scoring +rule, so `affects_scoring_rule` is **false** — but removing study machinery is a material act +and is recorded as one, which also means it would be forbidden after the boundary opens. + +--- + ## A18 — the commit ↔ file accounting of record ```json diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/probes/x06_m6_feasibility.py b/docs/research/pdf-backend-bakeoff/validation/external-validity/probes/x06_m6_feasibility.py deleted file mode 100644 index a0d7d898..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/probes/x06_m6_feasibility.py +++ /dev/null @@ -1,174 +0,0 @@ -"""x06 -- DESIGN MATERIAL. Can M6 be adjudicated inside a 6-10 line region? - -NOT CONFIRMATORY. DEVELOPMENT documents only. No holdout document is opened. - -PRE-REGISTRATION.md section 6 defines M6 as: - - "for each dollar amount in a C-region, emitted nearest heading-ish ancestor vs - adjudicated" - -and section 5.3 fixes the adjudicated unit as a region "spanning 6-10 printed lines", with -the adjudicator seeing ONLY that region's rendered image. - -Those two clauses are in tension, and the tension is measurable rather than arguable: an -appropriations account heading governs a long run of prose, so most amounts sit many lines -below the heading that owns them. If the governing heading is outside the rendered region, -the adjudicator cannot state the amount's account from the evidence it is shown, and M6 has -no oracle for that amount. - -WHAT THIS MEASURES, AND WHAT IT DOES NOT. The reference here is the nearest preceding -account/agency anchor that PRODUCTION's `extract_anchors` emits from the HYBRID -reconstruction. That is an architecture-derived reference, NOT independent truth. So this -probe measures: - - the distance from each amount to the nearest preceding HYBRID-PRODUCED anchor - -and NOT: - - the distance from each amount to its TRUE governing account heading. - -The distinction matters: if hybrid misses a heading, the measured distance to the next one -back is too large; if hybrid invents one, too small. The measurement is still decisive for -the design question -- a 6-10 line viewport is far too small for a claim about long-range -attribution -- but it does not establish the true distance distribution, and A17 must not -be worded as though it does. - -Uses the hybrid path only: the question is about the PROTOCOL's geometry, not about a -difference between architectures, and both arms see the same headings here. -""" - -from __future__ import annotations - -import json -import re -import sys -from pathlib import Path - -HERE = Path(__file__).resolve() -EV = HERE.parents[1] -BAKE = EV.parents[1] -REPO = BAKE.parents[2] - -sys.path.insert(0, str(REPO / "src")) -sys.path.insert(0, str(BAKE / "probes")) -sys.path.insert(0, str(BAKE / "probes" / "backends")) - -import pdfium_hybrid # noqa: E402 -import reconstruct_hybrid # noqa: E402 - -from deltatrack.parsers.pdf_anchors import extract_anchors # noqa: E402 - -OUT = EV / "results" / "x06_m6_feasibility.json" -AMOUNT_RE = re.compile(r"\$[\d,]+(?:\.\d+)?") -HEADING_KINDS = ("account", "agency") -REGION_SIZES = (6, 8, 10, 15, 25, 50) - -DOCS = [ - ("114-hr-2029/4", REPO / "tests/corpus/114-hr-2029/4_reported-in-senate.pdf"), - ("118-hr-4366/5", REPO / "tests/corpus/118-hr-4366/5_engrossed-amendment-house.pdf"), - ("118-s-4795/1", REPO / "tests/corpus/118-s-4795/1_reported-in-senate.pdf"), -] - - -def main(limit: int = 30) -> int: - per_doc = [] - all_distances: list[int] = [] - - for name, path in DOCS: - if not path.exists(): - print(f"MISSING {path}", file=sys.stderr) - continue - pages, _ = pdfium_hybrid.extract(path, limit=limit) - recon, _ = reconstruct_hybrid.reconstruct(pages) - anchors = extract_anchors(recon) - - # Flatten to a document-order line stream, and mark which lines are headings. - flat: list[tuple[int, int | None, str]] = [] - for p in recon: - for ln in p.lines: - flat.append((p.page_number, ln.line_number, ln.text)) - heading_at = {(a.page_number, a.line_number) for a in anchors if a.kind in HEADING_KINDS} - - distances: list[int] = [] - unattributable = 0 - for idx, (page, lineno, text) in enumerate(flat): - if not AMOUNT_RE.search(text): - continue - for amount in AMOUNT_RE.findall(text): - del amount - # Walk back to the nearest preceding account/agency heading. - d = None - for back in range(idx, -1, -1): - if (flat[back][0], flat[back][1]) in heading_at: - d = idx - back - break - if d is None: - unattributable += 1 - else: - distances.append(d) - all_distances.extend(distances) - - share = { - str(n): round(sum(1 for d in distances if d < n) / len(distances), 4) if distances else None - for n in REGION_SIZES - } - per_doc.append( - { - "document": name, - "pages_scored": limit, - "amounts_found": len(distances) + unattributable, - "amounts_with_a_preceding_heading": len(distances), - "amounts_with_NO_preceding_heading_in_document": unattributable, - "median_lines_to_governing_heading": sorted(distances)[len(distances) // 2] if distances else None, - "max_lines_to_governing_heading": max(distances) if distances else None, - "share_answerable_within_region_of_N_lines": share, - } - ) - print( - f"{name:16} amounts={len(distances) + unattributable:5} " - f"median_dist={sorted(distances)[len(distances) // 2] if distances else '-':>4} " - f"max={max(distances) if distances else '-':>4} " - f"answerable@10={share.get('10')}", - flush=True, - ) - - pooled = { - str(n): round(sum(1 for d in all_distances if d < n) / len(all_distances), 4) if all_distances else None - for n in REGION_SIZES - } - # DESCRIPTIVE, not a pass/fail. A first version of this probe applied a >50 % threshold - # and printed "may be implementable as frozen" at 57 %, which is far too lenient: it - # would license a metric whose oracle is silently absent for two amounts in five. The - # judgement belongs in the amendment, next to the number, not in a canned string here. - verdict = ( - f"On three development documents, the nearest preceding HYBRID-PRODUCED " - f"account/agency anchor lies within 6 reconstructed lines for {pooled.get('6')} of " - f"observed amounts and within 10 for {pooled.get('10')}; a 50-line span reaches " - f"{pooled.get('50')}. The reference is architecture-derived, so this is NOT the " - f"share of amounts whose TRUE governing heading is in range. It does show that a " - f"6-10 line viewport cannot support a long-range attribution claim. " - f"See PRE-EXECUTION-AMENDMENTS.md A17." - ) - doc = { - "population": "DEVELOPMENT -- not a holdout", - "question": "can M6's amount->account attribution be adjudicated inside a 6-10 printed-line region?", - "reference_is_architecture_derived": ( - "distances are to the nearest preceding anchor emitted by production " - "extract_anchors over the HYBRID reconstruction, not to independent truth" - ), - "protocol_region_size": "6-10 printed lines (PRE-REGISTRATION.md 5.3)", - "pooled_share_answerable_within_region_of_N_lines": pooled, - "amounts_total": len(all_distances), - "verdict": verdict, - "documents": per_doc, - } - OUT.parent.mkdir(parents=True, exist_ok=True) - OUT.write_text(json.dumps(doc, indent=1)) - print(f"\npooled share answerable by region size: {json.dumps(pooled)}") - print(f"\n{verdict}") - print(f"wrote {OUT}") - return 0 - - -if __name__ == "__main__": - sys.exit(main(int(sys.argv[1]) if len(sys.argv) > 1 else 30)) diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/probes/x19_raster_edge_diagnostic.py b/docs/research/pdf-backend-bakeoff/validation/external-validity/probes/x19_raster_edge_diagnostic.py deleted file mode 100644 index ae5937ec..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/probes/x19_raster_edge_diagnostic.py +++ /dev/null @@ -1,366 +0,0 @@ -"""x19 -- A32 raster diagnostic: is the VISIBLE left edge displaced from the geometric x0? - -NOT CONFIRMATORY. DEVELOPMENT only, worst-case targets from `x18`. Nothing is scored and no -oracle artifact is created. Evidence: `results/x19_raster_edge_diagnostic.json`. - -NARROW PURPOSE, per the A32 contract: does the MuPDF-rendered raster reveal a SYSTEMATIC reason -that "left edge of the first printed character" -- what A30.3 asks the adjudicator for -- would -sit materially away from the neutral glyph's geometric `x0`? - -WHAT THIS DELIBERATELY DOES NOT DO. No OCR. No pixel-intensity threshold invented and then -treated as truth. The only edge rule used is EXACT RASTER OCCUPANCY: with the page rendered on -a transparent ground, the visible edge is the leftmost pixel column receiving ANY alpha at all. -That is a property MuPDF computes, not a number chosen here. - -THE MEASUREMENT IS RESTRICTED GEOMETRICALLY, NOT HEURISTICALLY. A neighbouring glyph's ink -inside the search window would contaminate "leftmost inked column", and separating it would -require exactly the segmentation heuristic the contract forbids. So a target is measured only -when NO other glyph's ink box reaches into its window -- a purely geometric exclusion, decided -from `x18`'s recorded neighbour boxes, with the excluded count reported rather than hidden. - -RUN IT WITH AN INTERPRETER THAT HAS `pymupdf`. It is deliberately absent from the project venv: -that venv is shared across worktrees and PyMuPDF is a rejected EXTRACTOR under ADR 0002. Here -it is only the renderer PRE-REGISTRATION 5.2 already names for adjudication stimuli. -""" - -from __future__ import annotations - -import hashlib -import json -import sys -from pathlib import Path - -import pymupdf - -HERE = Path(__file__).resolve() -EV = HERE.parents[1] -X18 = EV / "results" / "x18_start_x_discriminability.json" -OUT = EV / "results" / "x19_raster_edge_diagnostic.json" -IMG_DIR = EV / "results" / "x19_raster_dev" - -PRIMARY_DPI, R1_DPI = 300, 330 -WINDOW_PT = 3.0 # how far left of x0 the search window reaches -N_IMAGES = 4 # worst cases kept as crops, per population -- minimal by design - -ROWS: list[dict] = [] -FAILED: list[str] = [] - - -def check(name, expected, observed, fails_when=""): - ok = expected == observed - ROWS.append({"test": name, "expected": expected, "observed": observed, "pass": ok, "fails_when": fails_when}) - print(f"[PASS] {name}" if ok else f"[FAIL] {name}\n expected={expected!r}\n observed={observed!r}") - if not ok: - FAILED.append(name) - - -def leftmost_inked_column(pix): - """The leftmost pixel column carrying ANY alpha, or None if the crop is empty. - - Exact occupancy. No intensity threshold: a column counts if the renderer put any coverage - in it at all, which is what "the glyph is visible here" means without inventing a cutoff. - """ - n, w, h = pix.n, pix.width, pix.height - samples = pix.samples - for x in range(w): - for y in range(h): - if samples[(y * w + x) * n + (n - 1)]: - return x - return None - - -def measure(target, dpi, save_as=None): - """Render one target's window and report where visible ink actually starts.""" - doc = pymupdf.open(target["pdf"]) - try: - page = doc[target["page"] - 1] - if page.rotation: - return {"error": "PAGE_ROTATED", "rotation": page.rotation} - # PDF user space is bottom-up; MuPDF's page space is top-down. Converting wrongly - # would silently render the wrong band, so the empty-crop check below is the guard. - ph = page.rect.height - x_left = target["x0"] - WINDOW_PT - rect = pymupdf.Rect(x_left, ph - target["y1"], target["x1"], ph - target["y0"]) - - # NON-GLYPH VECTOR INK. GPO reported prints show matter deleted in committee as - # STRUCK THROUGH, and that strike is a drawn rule, not a character: it carries no text - # index, so it has no `ngid` and is correctly absent from the neutral skeleton -- yet - # MuPDF renders it across every column of the line. Raster occupancy cannot tell rule - # ink from glyph ink, and separating them is precisely the segmentation heuristic the - # A32 contract forbids. `get_drawings()` reports vector paths STRUCTURALLY, so the - # contaminated case is excluded exactly rather than guessed at or silently reported as - # a displaced glyph edge. - # NB: a strike rule is a ZERO-HEIGHT rect, and PyMuPDF's Rect.intersects() is False for - # a degenerate rect -- so the obvious spelling of this test can never fire. Overlap is - # therefore tested by coordinate, which is what makes this exclusion actually work. - drawings = [ - dr - for dr in page.get_drawings() - if dr["rect"].x0 <= rect.x1 - and dr["rect"].x1 >= rect.x0 - and dr["rect"].y0 <= rect.y1 - and dr["rect"].y1 >= rect.y0 - ] - if drawings: - return { - "dpi": dpi, - "measured": False, - "why_not_measured": "NON_GLYPH_VECTOR_INK_IN_BAND", - "n_vector_paths_in_band": len(drawings), - "note": "struck-through matter: a drawn rule, not a glyph; no ngid by construction", - } - pix = page.get_pixmap(matrix=pymupdf.Matrix(dpi / 72.0, dpi / 72.0), clip=rect, alpha=True) - col = leftmost_inked_column(pix) - out = { - "dpi": dpi, - "crop_px": [pix.width, pix.height], - "window_pt": WINDOW_PT, - "leftmost_inked_column_px": col, - } - if col is None: - out["error"] = "EMPTY_CROP" - return out - # offset of the visible edge from the geometric x0, in pixels at this scale - x0_col = (target["x0"] - x_left) * dpi / 72.0 - out["geometric_x0_column_px"] = x0_col - out["visible_edge_offset_px"] = col - x0_col - if save_as: - IMG_DIR.mkdir(parents=True, exist_ok=True) - data = pix.tobytes("png") - (IMG_DIR / save_as).write_bytes(data) - out["image"] = save_as - out["image_sha256"] = hashlib.sha256(data).hexdigest() - return out - finally: - doc.close() - - -def mupdf_glyph_bbox_delta(target): - """MuPDF's reported glyph bbox x0 minus PDFium's char-box x0, in points. - - A SEPARATE quantity from the visible-ink edge, reported because it is systematic and - would otherwise look like a discrepancy. A glyph bbox includes the left side bearing -- - whitespace the font reserves before the ink starts -- so this delta is NOT what a human - marking "the left edge of the first printed character" would point at. The raster - occupancy figure above is the task-relevant one. - """ - doc = pymupdf.open(target["pdf"]) - try: - page = doc[target["page"] - 1] - ph = page.rect.height - band = pymupdf.Rect(target["x0"] - 0.2, ph - target["y1"], target["x1"] + 0.2, ph - target["y0"]) - best = None - for b in page.get_text("rawdict", clip=band)["blocks"]: - for line in b.get("lines", []): - for s in line.get("spans", []): - for c in s.get("chars", []): - cb = c["bbox"] - if abs(cb[0] - target["x0"]) < 3.0 or cb[0] <= target["x0"] <= cb[2]: - if best is None or abs(cb[0] - target["x0"]) < abs(best["mupdf_x0"] - target["x0"]): - best = {"mupdf_x0": cb[0], "char": c["c"], "font": s["font"], "size": s["size"]} - if best is None: - return {"found": False} - best["pdfium_x0"] = target["x0"] - best["delta_pt"] = best["mupdf_x0"] - target["x0"] - best["delta_px_300"] = best["delta_pt"] * PRIMARY_DPI / 72.0 - best["found"] = True - return best - finally: - doc.close() - - -def main() -> int: - if not X18.exists(): - print(f"FATAL: run x18 first ({X18} missing)") - return 2 - x18 = json.loads(X18.read_text()) - worst = x18["development"]["worst_cases"] - - print(f"pymupdf {pymupdf.__doc__ or ''} version={pymupdf.version}") - results = {"H": [], "C": []} - excluded = {"H": 0, "C": 0} - empty = 0 - - for pop in ("H", "C"): - for i, t in enumerate(worst[pop]): - # purely geometric exclusion: any neighbouring ink reaching into the window makes - # "leftmost inked column" ambiguous, and segmenting it is forbidden - intruding = [n for n in t.get("ink_within_6pt_left", []) if n["x1"] > t["x0"] - WINDOW_PT] - if intruding: - excluded[pop] += 1 - results[pop].append( - { - "document": t["document"], - "page": t["page"], - "ngid": t["ngid"], - "m_pt": t["m_pt"], - "measured": False, - "why_not_measured": "NEIGHBOUR_INK_IN_WINDOW", - "intruding": intruding, - } - ) - continue - row = { - "document": t["document"], - "page": t["page"], - "ngid": t["ngid"], - "kind": t.get("kind"), - "m_pt": t["m_pt"], - } - keep = i < N_IMAGES - for dpi, tag in ((PRIMARY_DPI, "300"), (R1_DPI, "330")): - name = f"{pop}{i}_{t['document'].replace('/', '-')}_p{t['page']}_g{t['ngid']}_{tag}dpi.png" - m = measure(t, dpi, save_as=name if keep else None) - row[f"dpi_{tag}"] = m - if m.get("error") == "EMPTY_CROP": - empty += 1 - row["mupdf_glyph_bbox_delta"] = mupdf_glyph_bbox_delta(t) - # `measure()` is the AUTHORITY on whether a raster edge was obtained. An earlier - # cut set measured=True on this row BEFORE those calls ran, so a target measure() - # had excluded still counted in the denominator -- 16 reported where 15 were - # measurable. The flag is now derived from the per-scale results, never asserted - # ahead of them. - reasons = sorted({row[k].get("why_not_measured") for k in ("dpi_300", "dpi_330")} - {None}) - row["measured"] = not reasons - if reasons: - row["why_not_measured"] = reasons[0] - results[pop].append(row) - - all_rows = [r for p in results.values() for r in p] - cleanly = [r for r in all_rows if r.get("measured") and "dpi_300" in r] - offsets300 = [ - r["dpi_300"]["visible_edge_offset_px"] - for r in cleanly - if r["dpi_300"].get("visible_edge_offset_px") is not None - ] - offsets330 = [ - r["dpi_330"]["visible_edge_offset_px"] - for r in cleanly - if r["dpi_330"].get("visible_edge_offset_px") is not None - ] - vector_excluded = [ - { - "document": r["document"], - "page": r["page"], - "ngid": r["ngid"], - "n_vector_paths_in_band": r["dpi_300"].get("n_vector_paths_in_band"), - } - for r in all_rows - if r.get("why_not_measured") == "NON_GLYPH_VECTOR_INK_IN_BAND" - ] - - # NEGATIVE CONTROL for the denominator defect itself: an excluded target may neither be - # counted as measured nor contribute an offset. Asserted on the rows, not on a helper - # comparing itself. - check( - "a vector-ink exclusion is NOT counted in the measured denominator", - [], - [r["ngid"] for r in cleanly if r.get("why_not_measured") == "NON_GLYPH_VECTOR_INK_IN_BAND"], - "the outer row once said measured=True before measure() ran, inflating n_measured by one", - ) - check( - "...and contributes NO value to either offset distribution", - (0, 0), - tuple( - sum( - 1 - for r in all_rows - if r.get("why_not_measured") == "NON_GLYPH_VECTOR_INK_IN_BAND" - and r[k].get("visible_edge_offset_px") is not None - ) - for k in ("dpi_300", "dpi_330") - ), - ) - check( - "the denominator is exact: cleanly measured + excluded == attempted", - len(all_rows), - len(cleanly) + len([r for r in all_rows if not r.get("measured")]), - ) - - check( - "every rendered crop contained ink, so the PDF->MuPDF coordinate mapping is right", - 0, - empty, - "an empty crop would mean the bottom-up/top-down y conversion is wrong and every " - "offset below would be measuring blank paper", - ) - check( - "at least one worst-case target admitted a clean, unsegmented measurement", - True, - len(cleanly) > 0, - "if all were excluded, this diagnostic would report nothing and must say so", - ) - - summary = { - "renderer": "MuPDF (pymupdf)", - "renderer_version": str(pymupdf.version), - "edge_rule": "exact raster occupancy -- leftmost column with ANY alpha; no intensity threshold", - "ocr_used": False, - "segmentation_heuristic_used": False, - "window_pt": WINDOW_PT, - "n_attempted": len(all_rows), - "n_cleanly_measured": len(cleanly), - "n_excluded_neighbour_ink": excluded, - "n_excluded_non_glyph_vector_ink": len(vector_excluded), - "excluded_non_glyph_vector_ink": vector_excluded, - "visible_edge_offset_px_300": { - "min": min(offsets300) if offsets300 else None, - "max": max(offsets300) if offsets300 else None, - "values": offsets300, - }, - "visible_edge_offset_px_330": { - "min": min(offsets330) if offsets330 else None, - "max": max(offsets330) if offsets330 else None, - "values": offsets330, - }, - "limitation": ( - "Two exclusions, both STRUCTURAL rather than heuristic. (1) A target whose window " - "contains a neighbouring GLYPH's ink is not measured; its intruding boxes are " - "listed. (2) A target whose band is crossed by NON-GLYPH VECTOR INK is not " - "measured: GPO reported prints strike through matter deleted in committee, and " - "that rule is a drawn path with no text index, hence no ngid -- correctly outside " - "the neutral skeleton, but rendered by MuPDF across the whole line. Raster " - "occupancy cannot separate rule ink from glyph ink, and doing so would need the " - "segmentation heuristic the A32 contract forbids. Crops for the worst cases are " - "committed for human inspection." - ), - "mupdf_vs_pdfium_glyph_bbox": ( - "Separately measured and reported so it cannot look like an unexplained " - "discrepancy: MuPDF's glyph BBOX x0 sits systematically left of PDFium's char-box " - "x0 -- about -1.23 pt for '(' and -0.90 pt for the open quote at DeVinne/14. That " - "is the font's left side bearing, i.e. reserved whitespace before the ink, and it " - "is NOT what a human marking the left edge of a printed character would point at. " - "The raster-occupancy offsets above are the task-relevant quantity." - ), - "for_the_reviewer": ( - "The struck-through case is not a defect in A30.3's geometry: the glyph's x0 is " - "unchanged and the resolver never reads pixels. It is a PROMPT-DESIGN observation " - "-- an adjudicator asked for 'the left edge of the first printed character' will " - "see a horizontal rule running through that character on reported prints. Whether " - "that needs wording in adjudicator_prompt.md is the reviewer's call; A32 states " - "no ruling." - ), - } - print(json.dumps(summary, indent=1)[:1200]) - - OUT.parent.mkdir(parents=True, exist_ok=True) - OUT.write_text( - json.dumps( - { - "population": "DEVELOPMENT worst cases from x18 -- no holdout opened, nothing scored", - "summary": summary, - "H": results["H"], - "C": results["C"], - "tests": ROWS, - "failures": FAILED, - }, - indent=1, - ) - ) - print(f"\n{len(ROWS) - len(FAILED)}/{len(ROWS)} checks pass") - print(f"wrote {OUT}") - return 1 if FAILED else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x06_m6_feasibility.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x06_m6_feasibility.json deleted file mode 100644 index 94ae3754..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x06_m6_feasibility.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "population": "DEVELOPMENT -- not a holdout", - "question": "can M6's amount->account attribution be adjudicated inside a 6-10 printed-line region?", - "reference_is_architecture_derived": "distances are to the nearest preceding anchor emitted by production extract_anchors over the HYBRID reconstruction, not to independent truth", - "protocol_region_size": "6-10 printed lines (PRE-REGISTRATION.md 5.3)", - "pooled_share_answerable_within_region_of_N_lines": { - "6": 0.4336, - "8": 0.4912, - "10": 0.5708, - "15": 0.677, - "25": 0.7743, - "50": 0.823 - }, - "amounts_total": 226, - "verdict": "On three development documents, the nearest preceding HYBRID-PRODUCED account/agency anchor lies within 6 reconstructed lines for 0.4336 of observed amounts and within 10 for 0.5708; a 50-line span reaches 0.823. The reference is architecture-derived, so this is NOT the share of amounts whose TRUE governing heading is in range. It does show that a 6-10 line viewport cannot support a long-range attribution claim. See PRE-EXECUTION-AMENDMENTS.md A17.", - "documents": [ - { - "document": "114-hr-2029/4", - "pages_scored": 30, - "amounts_found": 74, - "amounts_with_a_preceding_heading": 74, - "amounts_with_NO_preceding_heading_in_document": 0, - "median_lines_to_governing_heading": 5, - "max_lines_to_governing_heading": 210, - "share_answerable_within_region_of_N_lines": { - "6": 0.5135, - "8": 0.5676, - "10": 0.6622, - "15": 0.8243, - "25": 0.8649, - "50": 0.8919 - } - }, - { - "document": "118-hr-4366/5", - "pages_scored": 30, - "amounts_found": 70, - "amounts_with_a_preceding_heading": 70, - "amounts_with_NO_preceding_heading_in_document": 0, - "median_lines_to_governing_heading": 12, - "max_lines_to_governing_heading": 288, - "share_answerable_within_region_of_N_lines": { - "6": 0.3714, - "8": 0.4429, - "10": 0.4714, - "15": 0.5286, - "25": 0.5571, - "50": 0.5714 - } - }, - { - "document": "118-s-4795/1", - "pages_scored": 30, - "amounts_found": 82, - "amounts_with_a_preceding_heading": 82, - "amounts_with_NO_preceding_heading_in_document": 0, - "median_lines_to_governing_heading": 9, - "max_lines_to_governing_heading": 54, - "share_answerable_within_region_of_N_lines": { - "6": 0.4146, - "8": 0.4634, - "10": 0.5732, - "15": 0.6707, - "25": 0.878, - "50": 0.9756 - } - } - ] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x07_neutral_geometry.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x07_neutral_geometry.json deleted file mode 100644 index e6573d6b..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x07_neutral_geometry.json +++ /dev/null @@ -1,366 +0,0 @@ -{ - "population": "SYNTHETIC + DEVELOPMENT -- no holdout opened", - "purpose": "test the neutral ink-line skeleton proposed to resolve A17.2-A17.4", - "rule": { - "neutral_facts": [ - "baseline (pen origin y)", - "glyph box x0/y0/x1/y1", - "page number" - ], - "forbidden": [ - "codepoint", - "font name", - "word spacing", - "case", - "heading label", - "architecture line ordinal" - ], - "tolerance": "0.5 * median ink height on the page (derived, not a constant)", - "ordering": "descending baseline = top-to-bottom; ordinal is the index", - "anchor": "first glyph of the cluster, so drift cannot walk a cluster down the page", - "regions": "non-overlapping windows of 8 neutral lines, aligned to page start, short tail kept", - "projection": "nearest neutral baseline within tolerance; no text similarity" - }, - "synthetic_tests": [ - { - "test": "two ordinary adjacent lines", - "expected": 2, - "observed": 2, - "pass": true, - "implication": "ordinary leading separates" - }, - { - "test": "two lines 2pt apart merge (tol=5pt)", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "sub-tolerance baselines are one physical line; this is the documented limit" - }, - { - "test": "superscript rides with its body line", - "expected": 2, - "observed": 2, - "pass": true, - "implication": "footnote markers do not fabricate a line" - }, - { - "test": "subscript rides with its body line", - "expected": 2, - "observed": 2, - "pass": true, - "implication": "same, downward" - }, - { - "test": "large heading then body", - "expected": 2, - "observed": 2, - "pass": true, - "implication": "display type does not swallow body" - }, - { - "test": "letter-spaced display heading is one line", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "the C O N T E N T S case does not become 8 lines" - }, - { - "test": "margin line number joins its body line", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "GPO sets the number on the body baseline" - }, - { - "test": "one physical line from 3 text objects", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "text-object boundaries are invisible to the skeleton" - }, - { - "test": "two-column page MERGES columns (known limit)", - "expected": 2, - "observed": 2, - "pass": true, - "implication": "single-column GPO bills are unaffected; report tables are not, and A17-N records it" - }, - { - "test": "table row of fragments is one line", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "column fragments on a shared baseline are one neutral row" - }, - { - "test": "neutral ids are (page, ordinal) top-to-bottom", - "expected": [ - [ - 7, - 0 - ], - [ - 7, - 1 - ], - [ - 7, - 2 - ] - ], - "observed": [ - [ - 7, - 0 - ], - [ - 7, - 1 - ], - [ - 7, - 2 - ] - ], - "pass": true, - "implication": "identity is a pure function of geometry" - }, - { - "test": "identity is stable when glyph ORDER is shuffled", - "expected": [ - [ - 7, - 0 - ], - [ - 7, - 1 - ], - [ - 7, - 2 - ] - ], - "observed": [ - [ - 7, - 0 - ], - [ - 7, - 1 - ], - [ - 7, - 2 - ] - ], - "pass": true, - "implication": "input order cannot change identity" - }, - { - "test": "H line projects onto its neutral line", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "geometric projection, no text similarity" - }, - { - "test": "X line at a 1pt offset still projects to the same neutral line", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "sub-tolerance jitter does not move the slot" - }, - { - "test": "a line far from any neutral baseline projects to None", - "expected": null, - "observed": null, - "pass": true, - "implication": "fails closed rather than snapping to the nearest" - }, - { - "test": "baseline-proximity projection LOSES a merged line", - "expected": null, - "observed": null, - "pass": true, - "implication": "why membership projection replaces it" - }, - { - "test": "membership projection puts a merged line on one slot", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "plurality of glyphs; ties to the lowest ordinal" - }, - { - "test": "membership projection handles a SPLIT line", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "a split half lands on its own neutral line" - }, - { - "test": "membership projection with no glyphs is None", - "expected": null, - "observed": null, - "pass": true, - "implication": "fails closed" - }, - { - "test": "centred-narrow flags the centred line only", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "separates a heading from body prose and from a left-aligned paragraph tail" - }, - { - "test": "20 neutral lines -> 3 non-overlapping regions", - "expected": 3, - "observed": 3, - "pass": true, - "implication": "aligned to page start" - }, - { - "test": "regions partition the lines exactly once", - "expected": 20, - "observed": 20, - "pass": true, - "implication": "no line is dropped or double-counted" - }, - { - "test": "the short trailing region is kept", - "expected": 4, - "observed": 4, - "pass": true, - "implication": "page bottoms carry continuation text and must not be silently dropped" - } - ], - "synthetic_failures": [], - "development": [ - { - "document": "114-hr-2029/4", - "pages": 20, - "neutral_lines": 522, - "lines_per_page_median": 26.1, - "body_height_q0.5": 6.5, - "pages_enriched_q0.5": 20, - "pages_enriched_share": 1.0, - "quantum_sensitivity": { - "0.25": { - "body_height": 6.25, - "pages_enriched": 20 - }, - "0.5": { - "body_height": 6.5, - "pages_enriched": 20 - }, - "1.0": { - "body_height": 6.0, - "pages_enriched": 1 - } - }, - "centred_narrow_lines": 25, - "centred_narrow_validation": { - "pages_with_a_hybrid_account_or_agency_anchor": 9, - "pages_flagged_by_centred_narrow": 20, - "overlap": 9, - "recall_of_anchor_pages": 1.0, - "share_of_all_pages_flagged": 1.0 - } - }, - { - "document": "118-hr-4366/5", - "pages": 20, - "neutral_lines": 550, - "lines_per_page_median": 27.5, - "body_height_q0.5": 6.5, - "pages_enriched_q0.5": 20, - "pages_enriched_share": 1.0, - "quantum_sensitivity": { - "0.25": { - "body_height": 6.25, - "pages_enriched": 20 - }, - "0.5": { - "body_height": 6.5, - "pages_enriched": 20 - }, - "1.0": { - "body_height": 6.0, - "pages_enriched": 2 - } - }, - "centred_narrow_lines": 23, - "centred_narrow_validation": { - "pages_with_a_hybrid_account_or_agency_anchor": 12, - "pages_flagged_by_centred_narrow": 20, - "overlap": 12, - "recall_of_anchor_pages": 1.0, - "share_of_all_pages_flagged": 1.0 - } - }, - { - "document": "118-s-4795/1", - "pages": 20, - "neutral_lines": 527, - "lines_per_page_median": 26.4, - "body_height_q0.5": 6.5, - "pages_enriched_q0.5": 20, - "pages_enriched_share": 1.0, - "quantum_sensitivity": { - "0.25": { - "body_height": 6.25, - "pages_enriched": 20 - }, - "0.5": { - "body_height": 6.5, - "pages_enriched": 20 - }, - "1.0": { - "body_height": 6.0, - "pages_enriched": 1 - } - }, - "centred_narrow_lines": 23, - "centred_narrow_validation": { - "pages_with_a_hybrid_account_or_agency_anchor": 13, - "pages_flagged_by_centred_narrow": 20, - "overlap": 13, - "recall_of_anchor_pages": 1.0, - "share_of_all_pages_flagged": 1.0 - } - }, - { - "document": "CRPT-118srpt198", - "pages": 20, - "neutral_lines": 942, - "lines_per_page_median": 47.1, - "body_height_q0.5": 5.0, - "pages_enriched_q0.5": 7, - "pages_enriched_share": 0.35, - "quantum_sensitivity": { - "0.25": { - "body_height": 5.0, - "pages_enriched": 14 - }, - "0.5": { - "body_height": 5.0, - "pages_enriched": 7 - }, - "1.0": { - "body_height": 5.0, - "pages_enriched": 7 - } - }, - "centred_narrow_lines": 44, - "centred_narrow_validation": { - "pages_with_a_hybrid_account_or_agency_anchor": 0, - "pages_flagged_by_centred_narrow": 20, - "overlap": 0, - "recall_of_anchor_pages": null, - "share_of_all_pages_flagged": 1.0 - } - } - ] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x08_neutral_identity.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x08_neutral_identity.json deleted file mode 100644 index e39faaa1..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x08_neutral_identity.json +++ /dev/null @@ -1,338 +0,0 @@ -{ - "population": "SYNTHETIC + DEVELOPMENT (hybrid only) -- no holdout opened", - "neutral_glyph_id": "(document_sha256, page_number, source_char_index)", - "eligibility": "valid finite positive-area ink box AND upright AND gid is not None", - "projection": "set membership on gids; Model G source-glyph partition", - "model_comparison": [ - { - "case": "merge 3/1", - "plurality": [ - "FH!P", - "ABSENT" - ], - "partition": [ - "FH!", - "P" - ], - "preferred": "G", - "why": "P attributes line 1's glyph to line 0 AND blanks line 1: one glyph is double-counted and one physical line vanishes" - }, - { - "case": "merge 50/50", - "plurality": "tie-break decides which line is blanked", - "partition": [ - "A", - "D" - ], - "preferred": "G", - "why": "G needs no tie-break; P needs an arbitrary one" - }, - { - "case": "split 1->2", - "plurality": "both emitted lines -> same slot; second is lost or overwrites", - "partition": "rejoined in source order", - "preferred": "G", - "why": "P has no defined aggregation for two emitted lines on one slot" - } - ], - "tests": [ - { - "test": "OLD project_by_glyphs forces a y=-5000 glyph onto a line", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "nearest-baseline inference with no maximum distance -- not membership" - }, - { - "test": "NEW membership: a foreign gid contributes nothing", - "expected": "", - "observed": "", - "pass": true, - "implication": "set membership cannot invent ownership" - }, - { - "test": "1 neutral -> 1 emitted line", - "expected": "ABC", - "observed": "ABC", - "pass": true, - "implication": "" - }, - { - "test": "1 neutral -> 2 emitted lines: the SPLIT is preserved as a line break", - "expected": "A\nBC", - "observed": "A\nBC", - "pass": true, - "implication": "joining by '' would assert an adjacency the architecture never emitted" - }, - { - "test": "1 neutral -> 3 emitted lines", - "expected": "A\nB\nC", - "observed": "A\nB\nC", - "pass": true, - "implication": "" - }, - { - "test": "emitted-line order REVERSED gives the same contribution", - "expected": "A\nB\nC", - "observed": "A\nB\nC", - "pass": true, - "implication": "ordering is a function of source identity, not of emission order" - }, - { - "test": "many neutral -> 1 merged line: line 0 keeps only its own glyphs", - "expected": "ABC", - "observed": "ABC", - "pass": true, - "implication": "MODEL G -- a merge no longer blanks the other line" - }, - { - "test": "many neutral -> 1 merged line: line 1 keeps its own glyphs", - "expected": "DEF", - "observed": "DEF", - "pass": true, - "implication": "" - }, - { - "test": "exact 50/50 merge splits by ownership, not by plurality", - "expected": [ - "A", - "D" - ], - "observed": [ - "A", - "D" - ], - "pass": true, - "implication": "no tie-break is needed at all" - }, - { - "test": "duplicate gid is retained in the contribution", - "expected": "AAB", - "observed": "AAB", - "pass": true, - "implication": "duplication is visible, then flagged as a diagnostic" - }, - { - "test": "unowned glyph contributes nothing", - "expected": "AB", - "observed": "AB", - "pass": true, - "implication": "" - }, - { - "test": "lost glyph simply does not appear", - "expected": "AC", - "observed": "AC", - "pass": true, - "implication": "" - }, - { - "test": "MODEL P double-counts a merged glyph", - "expected": true, - "observed": true, - "pass": true, - "implication": "whole merged text lands on line 0 while line 1 is blanked" - }, - { - "test": "MODEL G conserves every glyph exactly once", - "expected": [ - "!", - "F", - "H", - "P" - ], - "observed": [ - "!", - "F", - "H", - "P" - ], - "pass": true, - "implication": "partition is lossless; plurality is not" - }, - { - "test": "same gids, H welds -> no space", - "expected": "FH", - "observed": "FH", - "pass": true, - "implication": "" - }, - { - "test": "same gids, X inserts a space -> space preserved", - "expected": "F H", - "observed": "F H", - "pass": true, - "implication": "" - }, - { - "test": "the weld/space difference is TEXT_DIFFERS", - "expected": "TEXT_DIFFERS", - "observed": "TEXT_DIFFERS", - "pass": true, - "implication": "the seam difference survives projection -- it is not normalised away" - }, - { - "test": "R E P vs REP is TEXT_DIFFERS", - "expected": "TEXT_DIFFERS", - "observed": "TEXT_DIFFERS", - "pass": true, - "implication": "" - }, - { - "test": "a dropped glyph is TEXT_DIFFERS", - "expected": "TEXT_DIFFERS", - "observed": "TEXT_DIFFERS", - "pass": true, - "implication": "" - }, - { - "test": "...and is recorded as X source-glyph loss", - "expected": [ - 1 - ], - "observed": [ - 1 - ], - "pass": true, - "implication": "" - }, - { - "test": "a duplicated glyph is flagged", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "D-frame membership is invariant under swapping H and X", - "expected": [], - "observed": [], - "pass": true, - "implication": "D(H,X) == D(X,H) for every cardinality case" - }, - { - "test": "...and each case has the EXPECTED membership, so symmetry is not vacuous", - "expected": [], - "observed": [], - "pass": true, - "implication": "False == False no longer passes for a case that must be True" - }, - { - "test": "the asymmetric states are explicit mirrors", - "expected": [ - "X_ABSENT", - "H_ABSENT" - ], - "observed": [ - "X_ABSENT", - "H_ABSENT" - ], - "pass": true, - "implication": "" - }, - { - "test": "generated space (gid None) is ineligible", - "expected": false, - "observed": false, - "pass": true, - "implication": "" - }, - { - "test": "missing box is ineligible", - "expected": false, - "observed": false, - "pass": true, - "implication": "" - }, - { - "test": "zero-area box is ineligible", - "expected": false, - "observed": false, - "pass": true, - "implication": "" - }, - { - "test": "zero-width box is ineligible", - "expected": false, - "observed": false, - "pass": true, - "implication": "" - }, - { - "test": "non-upright glyph is ineligible", - "expected": false, - "observed": false, - "pass": true, - "implication": "" - }, - { - "test": "a real ink mark is eligible", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "None coordinate is ineligible", - "expected": false, - "observed": false, - "pass": true, - "implication": "" - }, - { - "test": "A24.2: a content-stream U+0020 with a positive-area box is INELIGIBLE", - "expected": false, - "observed": false, - "pass": true, - "implication": "the box PDFium reports for a real space is 3.6 x 0.014 pt and clears 'positive area'; geometry alone cannot express the ink/non-ink distinction" - }, - { - "test": "...while the same geometry with an ink codepoint stays eligible", - "expected": true, - "observed": true, - "pass": true, - "implication": "the exclusion is exactly U+0020, not a size threshold" - }, - { - "test": "adversarial geometry matrix matches the frozen invariant exactly", - "expected": [], - "observed": [], - "pass": true, - "implication": "" - }, - { - "test": "...and the predicate returns BOTH answers across it", - "expected": [ - false, - true - ], - "observed": [ - false, - true - ], - "pass": true, - "implication": "a matrix that only ever returns False would pass while proving nothing" - } - ], - "failures": [], - "development": [ - { - "document": "114-hr-2029/4", - "pages": 12, - "chars": 15453, - "generated": 2461, - "geometrically_eligible": 12691, - "excluded": 2762, - "excluded_ONLY_by_a_codepoint_rule": 301 - }, - { - "document": "118-s-4795/1", - "pages": 12, - "chars": 16276, - "generated": 788, - "geometrically_eligible": 13317, - "excluded": 2959, - "excluded_ONLY_by_a_codepoint_rule": 2171 - } - ] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x10_reconstruction_signature.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x10_reconstruction_signature.json deleted file mode 100644 index 7f7dc12c..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x10_reconstruction_signature.json +++ /dev/null @@ -1,1652 +0,0 @@ -{ - "population": "SYNTHETIC only -- no holdout opened, no architecture run", - "supersedes": "A21's D-frame membership rule (any line not SAME); A22's gid-subset signature", - "defect": [ - { - "scenario": "H merges N0+N1, X emits them separately", - "lines": [ - { - "line": [ - 1, - 0 - ], - "state": "SAME", - "h_text": "ABC", - "x_text": "ABC", - "H_CROSS_LINE_MERGE": true, - "X_CROSS_LINE_MERGE": false, - "superseded_d_frame": false, - "new_d_frame": true - }, - { - "line": [ - 1, - 1 - ], - "state": "SAME", - "h_text": "DEF", - "x_text": "DEF", - "H_CROSS_LINE_MERGE": true, - "X_CROSS_LINE_MERGE": false, - "superseded_d_frame": false, - "new_d_frame": true - } - ] - }, - { - "scenario": "the inverse: H separate, X merges", - "lines": [ - { - "line": [ - 1, - 0 - ], - "state": "SAME", - "h_text": "ABC", - "x_text": "ABC", - "H_CROSS_LINE_MERGE": false, - "X_CROSS_LINE_MERGE": true, - "superseded_d_frame": false, - "new_d_frame": true - }, - { - "line": [ - 1, - 1 - ], - "state": "SAME", - "h_text": "DEF", - "x_text": "DEF", - "H_CROSS_LINE_MERGE": false, - "X_CROSS_LINE_MERGE": true, - "superseded_d_frame": false, - "new_d_frame": true - } - ] - }, - { - "scenario": "pure within-line split of N0, rejoined in source order", - "lines": [ - { - "line": [ - 1, - 0 - ], - "state": "TEXT_DIFFERS", - "h_text": "ABC", - "x_text": "A\nBC", - "H_CROSS_LINE_MERGE": false, - "X_CROSS_LINE_MERGE": false, - "superseded_d_frame": true, - "new_d_frame": true - }, - { - "line": [ - 1, - 1 - ], - "state": "BOTH_ABSENT", - "h_text": "", - "x_text": "", - "H_CROSS_LINE_MERGE": false, - "X_CROSS_LINE_MERGE": false, - "superseded_d_frame": true, - "new_d_frame": false - } - ] - } - ], - "matrix": [ - { - "case": "H merge / X split", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABC", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": true - }, - { - "case": "H merge / X split", - "line": [ - 1, - 1 - ], - "h_text": "DEF", - "x_text": "DEF", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": true - }, - { - "case": "H merge / X split", - "line": [ - 1, - 2 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "H merge / X split", - "REGION_IN_D_FRAME": true - }, - { - "case": "H split / X merge", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABC", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": true - }, - { - "case": "H split / X merge", - "line": [ - 1, - 1 - ], - "h_text": "DEF", - "x_text": "DEF", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": true - }, - { - "case": "H split / X merge", - "line": [ - 1, - 2 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "H split / X merge", - "REGION_IN_D_FRAME": true - }, - { - "case": "same merge on both arms", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABC", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "same merge on both arms", - "line": [ - 1, - 1 - ], - "h_text": "DEF", - "x_text": "DEF", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "same merge on both arms", - "line": [ - 1, - 2 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "same merge on both arms", - "REGION_IN_D_FRAME": false - }, - { - "case": "different merge spans (H: N0+N1, X: N1+N2)", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABC", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": true - }, - { - "case": "different merge spans (H: N0+N1, X: N1+N2)", - "line": [ - 1, - 1 - ], - "h_text": "DEF", - "x_text": "DEF", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": true - }, - { - "case": "different merge spans (H: N0+N1, X: N1+N2)", - "line": [ - 1, - 2 - ], - "h_text": "GHI", - "x_text": "GHI", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": true - }, - { - "case": "different merge spans (H: N0+N1, X: N1+N2)", - "REGION_IN_D_FRAME": true - }, - { - "case": "same glyphs / different emitted-line cardinality", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "A\nBC", - "expected_text_discordance": true, - "observed_text_discordance": true, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": false - }, - { - "case": "same glyphs / different emitted-line cardinality", - "line": [ - 1, - 1 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "same glyphs / different emitted-line cardinality", - "line": [ - 1, - 2 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "same glyphs / different emitted-line cardinality", - "REGION_IN_D_FRAME": true - }, - { - "case": "different text / same cardinality", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABZ", - "expected_text_discordance": true, - "observed_text_discordance": true, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": true, - "m3_reads_same": false - }, - { - "case": "different text / same cardinality", - "line": [ - 1, - 1 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "different text / same cardinality", - "line": [ - 1, - 2 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "different text / same cardinality", - "REGION_IN_D_FRAME": true - }, - { - "case": "same text / same cardinality", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABC", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "same text / same cardinality", - "line": [ - 1, - 1 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "same text / same cardinality", - "line": [ - 1, - 2 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "same text / same cardinality", - "REGION_IN_D_FRAME": false - }, - { - "case": "merge WITH a spacing difference", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "A BC", - "expected_text_discordance": true, - "observed_text_discordance": true, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": false - }, - { - "case": "merge WITH a spacing difference", - "line": [ - 1, - 1 - ], - "h_text": "DEF", - "x_text": "DEF", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": true - }, - { - "case": "merge WITH a spacing difference", - "line": [ - 1, - 2 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "merge WITH a spacing difference", - "REGION_IN_D_FRAME": true - }, - { - "case": "split WITHOUT a spacing difference", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "AB\nC", - "expected_text_discordance": true, - "observed_text_discordance": true, - "expected_segmentation_discordance": true, - "observed_segmentation_discordance": true, - "in_d_frame": true, - "m3_reads_same": false - }, - { - "case": "split WITHOUT a spacing difference", - "line": [ - 1, - 1 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "split WITHOUT a spacing difference", - "line": [ - 1, - 2 - ], - "h_text": "", - "x_text": "", - "expected_text_discordance": false, - "observed_text_discordance": false, - "expected_segmentation_discordance": false, - "observed_segmentation_discordance": false, - "in_d_frame": false, - "m3_reads_same": true - }, - { - "case": "split WITHOUT a spacing difference", - "REGION_IN_D_FRAME": true - } - ], - "coverage_vs_grouping": [ - { - "case": "same one-line grouping, X drops a glyph", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "AC", - "common_gids": [ - 0, - 2 - ], - "expected": { - "text": true, - "segmentation": false, - "diagnostic": "X_LOSS" - }, - "observed": { - "text": true, - "segmentation": false, - "diagnostic": "X_LOSS" - }, - "in_d_frame": true - }, - { - "case": "same one-line grouping, X duplicates a glyph", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABBC", - "common_gids": [ - 0, - 1, - 2 - ], - "expected": { - "text": true, - "segmentation": false, - "diagnostic": "X_DUP" - }, - "observed": { - "text": true, - "segmentation": false, - "diagnostic": "X_DUP" - }, - "in_d_frame": true - }, - { - "case": "same glyphs, 1 emitted line vs 2", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "A\nBC", - "common_gids": [ - 0, - 1, - 2 - ], - "expected": { - "text": true, - "segmentation": true, - "diagnostic": null - }, - "observed": { - "text": true, - "segmentation": true, - "diagnostic": null - }, - "in_d_frame": true - }, - { - "case": "H merges N0+N1, X emits separately", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABC", - "common_gids": [ - 0, - 1, - 2 - ], - "expected": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "observed": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "in_d_frame": true - }, - { - "case": "H merges N0+N1, X emits separately", - "line": [ - 1, - 1 - ], - "h_text": "DEF", - "x_text": "DEF", - "common_gids": [ - 3, - 4, - 5 - ], - "expected": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "observed": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "in_d_frame": true - }, - { - "case": "H merges N0+N1 AND loses gid 5, X emits separately", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABC", - "common_gids": [ - 0, - 1, - 2 - ], - "expected": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "observed": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "in_d_frame": true - }, - { - "case": "H merges N0+N1 AND loses gid 5, X emits separately", - "line": [ - 1, - 1 - ], - "h_text": "DE", - "x_text": "DEF", - "common_gids": [ - 3, - 4 - ], - "expected": { - "text": true, - "segmentation": true, - "diagnostic": "H_LOSS" - }, - "observed": { - "text": true, - "segmentation": true, - "diagnostic": "H_LOSS" - }, - "in_d_frame": true - }, - { - "case": "both arms lose the same glyph, grouping identical", - "line": [ - 1, - 0 - ], - "h_text": "AC", - "x_text": "AC", - "common_gids": [ - 0, - 2 - ], - "expected": { - "text": false, - "segmentation": false, - "diagnostic": "SHARED_LOSS" - }, - "observed": { - "text": false, - "segmentation": false, - "diagnostic": "SHARED_LOSS" - }, - "in_d_frame": false - }, - { - "case": "both arms merge N0+N1 identically", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABC", - "common_gids": [ - 0, - 1, - 2 - ], - "expected": { - "text": false, - "segmentation": false, - "diagnostic": null - }, - "observed": { - "text": false, - "segmentation": false, - "diagnostic": null - }, - "in_d_frame": false - }, - { - "case": "both arms merge N0+N1 identically", - "line": [ - 1, - 1 - ], - "h_text": "DEF", - "x_text": "DEF", - "common_gids": [ - 3, - 4, - 5 - ], - "expected": { - "text": false, - "segmentation": false, - "diagnostic": null - }, - "observed": { - "text": false, - "segmentation": false, - "diagnostic": null - }, - "in_d_frame": false - }, - { - "case": "different merge spans (H: N0+N1, X: N1+N2)", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "ABC", - "common_gids": [ - 0, - 1, - 2 - ], - "expected": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "observed": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "in_d_frame": true - }, - { - "case": "different merge spans (H: N0+N1, X: N1+N2)", - "line": [ - 1, - 1 - ], - "h_text": "DEF", - "x_text": "DEF", - "common_gids": [ - 3, - 4, - 5 - ], - "expected": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "observed": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "in_d_frame": true - }, - { - "case": "different merge spans (H: N0+N1, X: N1+N2)", - "line": [ - 1, - 2 - ], - "h_text": "GHI", - "x_text": "GHI", - "common_gids": [ - 6, - 7, - 8 - ], - "expected": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "observed": { - "text": false, - "segmentation": true, - "diagnostic": null - }, - "in_d_frame": true - }, - { - "case": "spacing-only difference", - "line": [ - 1, - 0 - ], - "h_text": "ABC", - "x_text": "A BC", - "common_gids": [ - 0, - 1, - 2 - ], - "expected": { - "text": true, - "segmentation": false, - "diagnostic": null - }, - "observed": { - "text": true, - "segmentation": false, - "diagnostic": null - }, - "in_d_frame": true - } - ], - "denominator": { - "population": "1 discordant + 1 agreeing + 3 jointly absent", - "all_lines_denominator": 5, - "risk_set_denominator": 2, - "M0_any_rate_risk_set": 0.5, - "M0_any_rate_all_lines_superseded": 0.2 - }, - "frame_conditioning": { - "fine_frame": { - "lines": 3, - "M0_any": 2, - "rate": 0.666667 - }, - "coarse_frame": { - "lines": 2, - "M0_any": 1, - "rate": 0.5 - } - }, - "provenance_vs_identity": [ - { - "case": "content-stream space between ink of one neutral line", - "expected": "A B", - "observed": "A B" - }, - { - "case": "generated space between ink of one neutral line", - "expected": "A B", - "observed": "A B" - }, - { - "case": "X-inserted space between ink of one neutral line", - "expected": "A B", - "observed": "A B" - }, - { - "case": "space across two neutral lines, seen from N0", - "expected": "A", - "observed": "A" - }, - { - "case": "space across two neutral lines, seen from N1", - "expected": "D", - "observed": "D" - }, - { - "case": "leading space before the first ink glyph", - "expected": "A", - "observed": "A" - }, - { - "case": "trailing space after the last ink glyph", - "expected": "A", - "observed": "A" - }, - { - "case": "consecutive spaces between owned ink", - "expected": "A B", - "observed": "A B" - }, - { - "case": "space between owned ink and a foreign glyph", - "expected": "A", - "observed": "A" - } - ], - "hr_fixture": { - "h_text": "H. R. 2029", - "x_text": "H.R.2029", - "text_discordance": true, - "segmentation_discordance": false, - "m3_outcome": "HeadingOutcome.X_REGRESSES", - "m3_x_weld": 2, - "m3_x_split": 0 - }, - "m0": { - "neutral_lines_in_scope": 5, - "risk_set": 4, - "M0a_text": 2, - "M0b_segmentation": 2, - "M0_any": 3, - "M0b_only_segmentation": 1, - "M0a_only_text": 1, - "both_absent": 1, - "M0b_defined": 4, - "M0a_text_rate": 0.5, - "M0b_segmentation_rate": 0.5, - "M0_any_rate": 0.75, - "M0b_rate_on_defined": 0.5, - "M0_any_rate_ALL_LINES_superseded": 0.6 - }, - "tests": [ - { - "test": "REPRODUCED: a cross-line merge/split leaves both text states SAME", - "expected": [ - "SAME", - "SAME" - ], - "observed": [ - "SAME", - "SAME" - ], - "pass": true, - "implication": "partition hands each neutral line back its own glyphs, so the text cannot differ" - }, - { - "test": "REPRODUCED: the superseded rule therefore drops it from the D-frame", - "expected": [ - false, - false - ], - "observed": [ - false, - false - ], - "pass": true, - "implication": "the merge/split disagreement A17.4 exists to observe was erased" - }, - { - "test": "...while the merge WAS visible all along, as a diagnostic that entered nothing", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "implication": "" - }, - { - "test": "CLOSED: the new rule puts both neutral lines in the D-frame", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "implication": "" - }, - { - "test": "REPRODUCED: a line NEITHER arm emits was in the D-frame under the superseded rule", - "expected": [ - true, - "BOTH_ABSENT" - ], - "observed": [ - true, - "BOTH_ABSENT" - ], - "pass": true, - "implication": "every running head, page number and VerDate stamp both arms correctly drop as chrome entered a census frame as discordance" - }, - { - "test": "CLOSED: a shared drop is not a comparative discordance", - "expected": false, - "observed": false, - "pass": true, - "implication": "PRE-REGISTRATION 5.8: the D-frame 'cannot see a failure both architectures share. That is exactly why the C-frame exists'" - }, - { - "test": "every adversarial case matches its expected text/segmentation verdict", - "expected": [], - "observed": [], - "pass": true, - "implication": "the two predicates are independent and each fires only where it should" - }, - { - "test": "region-level D-frame membership matches the frozen expectation", - "expected": [ - true, - true, - false, - true, - true, - true, - false, - true, - true - ], - "observed": [ - true, - true, - false, - true, - true, - true, - false, - true, - true - ], - "pass": true, - "implication": "" - }, - { - "test": "D(H,X) == D(X,H) on every adversarial case", - "expected": [], - "observed": [], - "pass": true, - "implication": "" - }, - { - "test": "...and the membership is the EXPECTED one, so symmetry is not vacuous", - "expected": [], - "observed": [], - "pass": true, - "implication": "the prior test passed on False == False while the disagreement was being erased" - }, - { - "test": "merge/split: D(H,X) and D(X,H) are both TRUE, not merely equal", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "implication": "" - }, - { - "test": "weld vs space is TEXT_DIFFERS", - "expected": "TEXT_DIFFERS", - "observed": "TEXT_DIFFERS", - "pass": true, - "implication": "" - }, - { - "test": "...and carries NO segmentation discordance", - "expected": false, - "observed": false, - "pass": true, - "implication": "" - }, - { - "test": "...and reaches M3 as X_CORRECTS", - "expected": "X_CORRECTS", - "observed": "X_CORRECTS", - "pass": true, - "implication": "" - }, - { - "test": "...with H scoring exactly one weld", - "expected": [ - 1, - 0 - ], - "observed": [ - 1, - 0 - ], - "pass": true, - "implication": "" - }, - { - "test": "...and X clean", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "a split heading IS a segmentation discordance", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "...and enters the D-frame", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "...but fabricates NO M3 word-boundary error", - "expected": [ - "BOTH_CLEAN", - 0, - 0 - ], - "observed": [ - "BOTH_CLEAN", - 0, - 0 - ], - "pass": true, - "implication": "the '\\n' join is read as a word boundary by m3_boundaries.decompose, so a split at a word boundary costs nothing -- joining by '' would have scored a false weld" - }, - { - "test": "...and both arms stay clean against the oracle", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "implication": "" - }, - { - "test": "a split MID-WORD still scores a real M3 split", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "the arm genuinely broke FAMILY across two printed lines" - }, - { - "test": "a dropped character is TEXT discordance with NO segmentation discordance", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "implication": "" - }, - { - "test": "...and reaches M3 as a dirty X against a clean H", - "expected": [ - "X_REGRESSES", - true, - false - ], - "observed": [ - "X_REGRESSES", - true, - false - ], - "pass": true, - "implication": "M3 consumes text and oracle evidence, never an M0b label" - }, - { - "test": "...scored as a text error, not a boundary error", - "expected": [ - 1, - 0, - 0 - ], - "observed": [ - 1, - 0, - 0 - ], - "pass": true, - "implication": "" - }, - { - "test": "a duplicated character is TEXT discordance with NO segmentation discordance", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "implication": "EmittedLine.gids is a set, so a repeated gid cannot reach the signature" - }, - { - "test": "...and reaches M3 as a dirty X", - "expected": [ - "X_REGRESSES", - false - ], - "observed": [ - "X_REGRESSES", - false - ], - "pass": true, - "implication": "" - }, - { - "test": "M0's components decompose the population exactly", - "expected": { - "in_scope": 5, - "risk_set": 4, - "M0a_text": 2, - "M0b_segmentation": 2, - "M0_any": 3, - "M0b_only": 1, - "M0a_only": 1, - "both_absent": 1 - }, - "observed": { - "in_scope": 5, - "risk_set": 4, - "M0a_text": 2, - "M0b_segmentation": 2, - "M0_any": 3, - "M0b_only": 1, - "M0a_only": 1, - "both_absent": 1 - }, - "pass": true, - "implication": "M0b_only is the count that was structurally unreachable under the superseded rule" - }, - { - "test": "M0_any is the UNION, never a sum", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "the risk set excludes BOTH_ABSENT and nothing else", - "expected": [ - 4, - 1 - ], - "observed": [ - 4, - 1 - ], - "pass": true, - "implication": "" - }, - { - "test": "the denominator change RAISES the rate, so it cannot be motivated by the number", - "expected": [ - 0.75, - 0.6 - ], - "observed": [ - 0.75, - 0.6 - ], - "pass": true, - "implication": "RQ1 seeks an equivalence statement, so a higher discordance rate makes the study's own claim harder to support, not easier" - }, - { - "test": "TEXT_DISCORDANCE returns both True and False on the matrix", - "expected": [ - false, - true - ], - "observed": [ - false, - true - ], - "pass": true, - "implication": "" - }, - { - "test": "SEGMENTATION_DISCORDANCE returns both True and False", - "expected": [ - false, - true - ], - "observed": [ - false, - true - ], - "pass": true, - "implication": "" - }, - { - "test": "line D-frame membership returns both True and False", - "expected": [ - false, - true - ], - "observed": [ - false, - true - ], - "pass": true, - "implication": "" - }, - { - "test": "ANCHOR_DISCORDANCE returns both", - "expected": [ - false, - true - ], - "observed": [ - false, - true - ], - "pass": true, - "implication": "" - }, - { - "test": "the signature separates merge, plain and split", - "expected": 3, - "observed": 3, - "pass": true, - "implication": "" - }, - { - "test": "the signature is INSENSITIVE to an inserted space", - "expected": [ - [ - [ - 0, - 1, - 2 - ], - [] - ] - ], - "observed": [ - [ - [ - 0, - 1, - 2 - ], - [] - ] - ], - "pass": true, - "implication": "a word-space decision must never register as a segmentation difference, or M3 would see a boundary error that does not exist" - }, - { - "test": "the signature names the merge SPAN, so different spans are distinguishable", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "the signature is INSENSITIVE to a repeated gid", - "expected": [ - [ - [ - 0, - 1, - 2 - ], - [] - ] - ], - "observed": [ - [ - [ - 0, - 1, - 2 - ], - [] - ] - ], - "pass": true, - "implication": "EmittedLine.gids is a set, so duplication could never move the signature -- measured, not assumed: duplication was already classified correctly before the common-domain repair, and only LOSS was mis-classified" - }, - { - "test": "coverage defects and grouping defects are classified independently", - "expected": [], - "observed": [], - "pass": true, - "implication": "loss and duplication must never move the segmentation metric; a merge must still be detected when the merging arm ALSO loses a glyph" - }, - { - "test": "the two denominators give materially different rates on the same population", - "expected": [ - 0.5, - 0.2 - ], - "observed": [ - 0.5, - 0.2 - ], - "pass": true, - "implication": "3 of 5 lines are page furniture neither arm emitted; counting them as agreements makes the reported rate a function of how much chrome the document carries" - }, - { - "test": "jointly absent lines are retained as a raw count, not discarded", - "expected": 3, - "observed": 3, - "pass": true, - "implication": "" - }, - { - "test": "identical architecture output scores a DIFFERENT M0 under a different frame", - "expected": true, - "observed": true, - "pass": true, - "implication": "so RQ1's reported numbers are conditional on the PDFium frame, and a failed cross-engine control must qualify RQ1 as well as RQ2" - }, - { - "test": "...while the per-line comparative verdict itself survived both frames", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "implication": "the conditioning is on denominators and populations, NOT on verdict direction -- stated precisely rather than overclaimed" - }, - { - "test": "Model G attachment holds for every space provenance and position", - "expected": [], - "observed": [], - "pass": true, - "implication": "" - }, - { - "test": "a content-stream space contributes TEXT but never IDENTITY", - "expected": [ - [ - 0, - 1 - ], - "A B" - ], - "observed": [ - [ - 0, - 1 - ], - "A B" - ], - "pass": true, - "implication": "gids carries neutral ink only; the space is visible in text and absent from identity" - }, - { - "test": "provenance survives on the cell even with no neutral identity", - "expected": [ - null, - 1, - false - ], - "observed": [ - null, - 1, - false - ], - "pass": true, - "implication": "" - }, - { - "test": "no space provenance can move the reconstruction signature", - "expected": [ - [ - [ - [ - 0, - 1 - ], - [] - ] - ], - [ - [ - [ - 0, - 1 - ], - [] - ] - ], - [ - [ - [ - 0, - 1 - ], - [] - ] - ] - ], - "observed": [ - [ - [ - [ - 0, - 1 - ], - [] - ] - ], - [ - [ - [ - 0, - 1 - ], - [] - ] - ], - [ - [ - [ - 0, - 1 - ], - [] - ] - ] - ], - "pass": true, - "implication": "segmentation must respond to grouping, never to spacing" - }, - { - "test": "H.R. fixture: H projects the content-stream spaces", - "expected": "H. R. 2029", - "observed": "H. R. 2029", - "pass": true, - "implication": "" - }, - { - "test": "H.R. fixture: X projects no space it did not derive", - "expected": "H.R.2029", - "observed": "H.R.2029", - "pass": true, - "implication": "" - }, - { - "test": "H.R. fixture: this is TEXT discordance", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "H.R. fixture: and NOT segmentation discordance", - "expected": false, - "observed": false, - "pass": true, - "implication": "the arms grouped the same ink identically; only the spacing differs" - }, - { - "test": "H.R. fixture: it enters the D-frame", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "H.R. fixture: it stays scorable by M3 rather than being filtered by the contract", - "expected": [ - "X_REGRESSES", - true, - false - ], - "observed": [ - "X_REGRESSES", - true, - false - ], - "pass": true, - "implication": "the eligibility gate does not decide correctness -- the oracle does" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x12_skeleton_eligibility.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x12_skeleton_eligibility.json deleted file mode 100644 index 1e84ce58..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x12_skeleton_eligibility.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "population": "DEVELOPMENT -- no holdout opened", - "superseded_rule": "valid finite positive-area box AND upright; no codepoint consulted", - "frozen_rule_A24_2": "source char exists AND codepoint != U+0020 AND valid positive-area box AND upright", - "why": "positive-area PDFium geometry does not identify ink: a content-stream U+0020 gets a ~3.6 x 0.014 pt box. A21 measured only the exclusion direction, so the inclusion was invisible. The exception is lexical but legitimate -- U+0020 is a below-seam source fact, X-2 already froze it as carrying no ink, it reads no H/X output, and it cannot favour either arm.", - "scope": "ONLY U+0020. Not a whitespace blacklist; no ink-height threshold is introduced.", - "documents": [ - { - "document": "114-hr-2029/4", - "pages": 8, - "u0020_source_chars": 229, - "u0020_admitted_by_superseded_rule": 229, - "u0020_admitted_by_A24_2_rule": 0, - "space_box_median_width": 3.626, - "space_box_median_height": 0.014, - "capital_box_median_height": 7.944, - "height_ratio_capital_over_space": 568.4, - "neutral_lines_before": 205, - "neutral_lines_after": 205, - "lines_touching_a_space_before": 141, - "pages_where_INK_membership_changed": 0, - "neutral_lines_whose_x_extent_changed": 132, - "x_loss_lines_after": 188, - "x_loss_lines_SHARED_with_H": 188, - "x_loss_lines_X_ONLY": 0, - "neutral_lines_scored_after": 205 - }, - { - "document": "118-s-4795/1", - "pages": 8, - "u0020_source_chars": 1398, - "u0020_admitted_by_superseded_rule": 1398, - "u0020_admitted_by_A24_2_rule": 0, - "space_box_median_width": 3.626, - "space_box_median_height": 0.014, - "capital_box_median_height": 7.4651, - "height_ratio_capital_over_space": 532.9, - "neutral_lines_before": 205, - "neutral_lines_after": 205, - "lines_touching_a_space_before": 190, - "pages_where_INK_membership_changed": 0, - "neutral_lines_whose_x_extent_changed": 138, - "x_loss_lines_after": 188, - "x_loss_lines_SHARED_with_H": 188, - "x_loss_lines_X_ONLY": 0, - "neutral_lines_scored_after": 205 - } - ], - "tests": [ - { - "test": "114-hr-2029/4: no remaining X loss is X's own", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "what survives is SHARED loss -- the GPO margin number, which 3.3 has both arms strip identically. X fails to carry nothing that H also does not carry, so the diagnostic is interpretable again rather than saturated by excluded spaces" - }, - { - "test": "114-hr-2029/4: the repair removes every U+0020 from the skeleton", - "expected": [ - true, - 0 - ], - "observed": [ - true, - 0 - ], - "pass": true, - "implication": "before/after paired on one population and one page limit" - }, - { - "test": "114-hr-2029/4: the INK partition is unchanged by removing spaces", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "removing non-ink glyphs must not move which ink glyphs share a physical line; if this ever fires, the clustering tolerance shifted enough to matter" - }, - { - "test": "118-s-4795/1: no remaining X loss is X's own", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "what survives is SHARED loss -- the GPO margin number, which 3.3 has both arms strip identically. X fails to carry nothing that H also does not carry, so the diagnostic is interpretable again rather than saturated by excluded spaces" - }, - { - "test": "118-s-4795/1: the repair removes every U+0020 from the skeleton", - "expected": [ - true, - 0 - ], - "observed": [ - true, - 0 - ], - "pass": true, - "implication": "before/after paired on one population and one page limit" - }, - { - "test": "118-s-4795/1: the INK partition is unchanged by removing spaces", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "removing non-ink glyphs must not move which ink glyphs share a physical line; if this ever fires, the clustering tolerance shifted enough to matter" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x14_anchor_bridge.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x14_anchor_bridge.json deleted file mode 100644 index a1ccc2f2..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x14_anchor_bridge.json +++ /dev/null @@ -1,254 +0,0 @@ -{ - "population": "DEVELOPMENT + synthetic -- no holdout opened, nothing scored", - "chain": "Anchor(page, margin line_number) -> Page.print_lines index -> emitted[i] -> ngid -> NeutralLine -> region", - "bilateral": "the SAME rule is applied to H and X; neither arm has a private matching rule", - "text_matching_used": false, - "fallback_exists": false, - "region_size_neutral_lines": 8, - "documents": [ - { - "document": "114-hr-2029/4", - "pages": 8, - "arms": [ - { - "arm": "H", - "anchors_total": 11, - "placed_uniquely": 11, - "unplaceable": {}, - "examples": [ - { - "kind": "title", - "page": 2, - "margin_line": 8, - "region": 1, - "neutral_line": [ - 2, - 8 - ] - }, - { - "kind": "major", - "page": 2, - "margin_line": 9, - "region": 1, - "neutral_line": [ - 2, - 9 - ] - } - ] - }, - { - "arm": "X", - "anchors_total": 11, - "placed_uniquely": 11, - "unplaceable": {}, - "examples": [ - { - "kind": "title", - "page": 2, - "margin_line": 8, - "region": 1, - "neutral_line": [ - 2, - 8 - ] - }, - { - "kind": "major", - "page": 2, - "margin_line": 9, - "region": 1, - "neutral_line": [ - 2, - 9 - ] - } - ] - } - ] - }, - { - "document": "118-s-4795/1", - "pages": 8, - "arms": [ - { - "arm": "H", - "anchors_total": 16, - "placed_uniquely": 16, - "unplaceable": {}, - "examples": [ - { - "kind": "title", - "page": 2, - "margin_line": 1, - "region": 0, - "neutral_line": [ - 2, - 1 - ] - }, - { - "kind": "major", - "page": 2, - "margin_line": 2, - "region": 0, - "neutral_line": [ - 2, - 2 - ] - } - ] - }, - { - "arm": "X", - "anchors_total": 16, - "placed_uniquely": 16, - "unplaceable": {}, - "examples": [ - { - "kind": "title", - "page": 2, - "margin_line": 1, - "region": 0, - "neutral_line": [ - 2, - 1 - ] - }, - { - "kind": "major", - "page": 2, - "margin_line": 2, - "region": 0, - "neutral_line": [ - 2, - 2 - ] - } - ] - } - ] - } - ], - "tests": [ - { - "test": "114-hr-2029/4: both arms bridge onto the SAME neutral skeleton", - "expected": [], - "observed": [], - "pass": true, - "implication": "" - }, - { - "test": "114-hr-2029/4 [H]: print_lines matches emitted index-for-index on EVERY page", - "expected": [], - "observed": [], - "pass": true, - "implication": "the bridge's index step is only sound while this holds, on every consumed page" - }, - { - "test": "114-hr-2029/4 [H]: every anchor on a sampled page places uniquely", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "any residue makes anchor discordance non-executable as frozen" - }, - { - "test": "114-hr-2029/4 [X]: print_lines matches emitted index-for-index on EVERY page", - "expected": [], - "observed": [], - "pass": true, - "implication": "the bridge's index step is only sound while this holds, on every consumed page" - }, - { - "test": "114-hr-2029/4 [X]: every anchor on a sampled page places uniquely", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "any residue makes anchor discordance non-executable as frozen" - }, - { - "test": "118-s-4795/1: both arms bridge onto the SAME neutral skeleton", - "expected": [], - "observed": [], - "pass": true, - "implication": "" - }, - { - "test": "118-s-4795/1 [H]: print_lines matches emitted index-for-index on EVERY page", - "expected": [], - "observed": [], - "pass": true, - "implication": "the bridge's index step is only sound while this holds, on every consumed page" - }, - { - "test": "118-s-4795/1 [H]: every anchor on a sampled page places uniquely", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "any residue makes anchor discordance non-executable as frozen" - }, - { - "test": "118-s-4795/1 [X]: print_lines matches emitted index-for-index on EVERY page", - "expected": [], - "observed": [], - "pass": true, - "implication": "the bridge's index step is only sound while this holds, on every consumed page" - }, - { - "test": "118-s-4795/1 [X]: every anchor on a sampled page places uniquely", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "any residue makes anchor discordance non-executable as frozen" - }, - { - "test": "negative: a margin number present twice on a page is refused, not guessed", - "expected": "AMBIGUOUS_MARGIN_NUMBER_ON_PAGE", - "observed": "AMBIGUOUS_MARGIN_NUMBER_ON_PAGE", - "pass": true, - "implication": "" - }, - { - "test": "negative: a margin number on no print line is refused, not guessed", - "expected": "NO_PRINT_LINE_WITH_THAT_MARGIN_NUMBER", - "observed": "NO_PRINT_LINE_WITH_THAT_MARGIN_NUMBER", - "pass": true, - "implication": "" - }, - { - "test": "negative: an emitted line with no neutral ink is refused, not guessed", - "expected": "EMITTED_LINE_CARRIES_NO_NEUTRAL_INK", - "observed": "EMITTED_LINE_CARRIES_NO_NEUTRAL_INK", - "pass": true, - "implication": "" - }, - { - "test": "negative: a first gid the skeleton does not own is refused, not guessed", - "expected": "FIRST_GID_NOT_OWNED_BY_ANY_NEUTRAL_LINE", - "observed": "FIRST_GID_NOT_OWNED_BY_ANY_NEUTRAL_LINE", - "pass": true, - "implication": "" - }, - { - "test": "positive: a clean anchor places on its region", - "expected": [ - 0, - [ - 1, - 0 - ] - ], - "observed": [ - 0, - [ - 1, - 0 - ] - ], - "pass": true, - "implication": "" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x15_methodology_contracts.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x15_methodology_contracts.json deleted file mode 100644 index bab8756f..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x15_methodology_contracts.json +++ /dev/null @@ -1,1760 +0,0 @@ -{ - "population": "SYNTHETIC only -- no PDF opened, no architecture run, nothing scored", - "adequacy_kinds": [ - "account", - "agency", - "grouping" - ], - "selection_seed": 20260807, - "primary_dpi": 300, - "r1_repeat_dpi": 330, - "a37_bootstrap": { - "statistic_identity": [ - "section8", - "document-heading-discordance", - "P-head" - ], - "namespace": "bootstrap-document", - "resamples": 10000, - "endpoint_indices": [ - 249, - 9749 - ], - "gate_vector": [ - "R1", - "N-A", - "N-B", - "N-C", - "S1", - "confirmatory X2-a", - "confirmatory X2-b", - "M9 evaluability", - "4.5 adequacy" - ], - "fixture_n_documents": 9, - "fixture_events": 3, - "interval": [ - 0.0, - 0.6666666666666666 - ], - "replicates_with_a_repeated_document_in_200": 200, - "zero_event_branch": { - "reported": false, - "reason": "ZERO_EVENTS_BOOTSTRAP_REFUSED", - "n_documents": 14, - "events": 0, - "clopper_pearson_upper_bound": 0.19263617565013536, - "gating": false - } - }, - "a39_rule0_margin": { - "rule": "count of Page.lines where line_number is not None; any positive deficit fires" - }, - "a39_cross_engine": { - "namespace": "cross-engine-page", - "fraction": 0.1, - "gate_owner": "x09_skeleton_cross_engine.gate -- not duplicated here", - "k_by_page_count": { - "1": 1, - "5": 1, - "9": 1, - "10": 1, - "11": 2, - "20": 2, - "21": 3 - }, - "sample_40_pages": [ - 14, - 22, - 34, - 37 - ] - }, - "tests": [ - { - "test": "4.5 state machine matches the frozen rows on every branch", - "expected": [], - "observed": [], - "pass": true, - "implication": "" - }, - { - "test": "...and all three states are reachable", - "expected": [ - "GENERALISABLE", - "INADEQUATE", - "LIMITED" - ], - "observed": [ - "GENERALISABLE", - "INADEQUATE", - "LIMITED" - ], - "pass": true, - "implication": "" - }, - { - "test": "no (strata, occurrences) pair is unclassified", - "expected": [], - "observed": [], - "pass": true, - "implication": "" - }, - { - "test": "one physical occurrence emitted by BOTH arms counts once", - "expected": 1, - "observed": 1, - "pass": true, - "implication": "" - }, - { - "test": "an occurrence emitted by ONE arm still counts", - "expected": 2, - "observed": 2, - "pass": true, - "implication": "" - }, - { - "test": "one arm missing an occurrence cannot remove the other arm's key", - "expected": 3, - "observed": 3, - "pass": true, - "implication": "an arm's own failure must never shrink the adequacy denominator" - }, - { - "test": "two occurrences on ONE neutral line stay distinct", - "expected": 2, - "observed": 2, - "pass": true, - "implication": "" - }, - { - "test": "only account/agency/grouping contribute to adequacy", - "expected": 3, - "observed": 3, - "pass": true, - "implication": "title/section/division must not inflate the denominator the frozen quantity is compared against" - }, - { - "test": "adding 120 P-robust account/agency/grouping keys does not change adequacy_occurrences", - "expected": 3, - "observed": 3, - "pass": true, - "implication": "the frozen P-head clause is now a gate rather than a caller obligation" - }, - { - "test": "...and the intruders really were adequacy-KIND keys, so the control is not vacuous", - "expected": 120, - "observed": 120, - "pass": true, - "implication": "" - }, - { - "test": "a P-robust key is excluded even when it is the ONLY input", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "" - }, - { - "test": "selection is reproducible", - "expected": [ - [ - "region", - "sha1", - 1, - 1 - ], - [ - "region", - "sha1", - 1, - 2 - ], - [ - "region", - "sha0", - 3, - 1 - ], - [ - "region", - "sha0", - 2, - 2 - ], - [ - "region", - "sha0", - 2, - 1 - ] - ], - "observed": [ - [ - "region", - "sha1", - 1, - 1 - ], - [ - "region", - "sha1", - 1, - 2 - ], - [ - "region", - "sha0", - 3, - 1 - ], - [ - "region", - "sha0", - 2, - 2 - ], - [ - "region", - "sha0", - 2, - 1 - ] - ], - "pass": true, - "implication": "" - }, - { - "test": "selection does not depend on input listing order", - "expected": [ - [ - "region", - "sha1", - 1, - 1 - ], - [ - "region", - "sha1", - 1, - 2 - ], - [ - "region", - "sha0", - 3, - 1 - ], - [ - "region", - "sha0", - 2, - 2 - ], - [ - "region", - "sha0", - 2, - 1 - ] - ], - "observed": [ - [ - "region", - "sha1", - 1, - 1 - ], - [ - "region", - "sha1", - 1, - 2 - ], - [ - "region", - "sha0", - 3, - 1 - ], - [ - "region", - "sha0", - 2, - 2 - ], - [ - "region", - "sha0", - 2, - 1 - ] - ], - "pass": true, - "implication": "" - }, - { - "test": "different purposes select differently", - "expected": true, - "observed": true, - "pass": true, - "implication": "domain separation, so one namespace's draw cannot leak into another's" - }, - { - "test": "canonical form is stable across tuple/list spelling", - "expected": "821b664b1f62c2f97820a841cb1dbbf8d86fbe27251750b992134afdf83ebeb4", - "observed": "821b664b1f62c2f97820a841cb1dbbf8d86fbe27251750b992134afdf83ebeb4", - "pass": true, - "implication": "" - }, - { - "test": "changing the blind-ID scheme changes NO selection and NO presentation rank", - "expected": { - "audit": [ - [ - "region", - "sha1", - 3, - 2 - ], - [ - "region", - "sha1", - 2, - 1 - ], - [ - "region", - "sha0", - 1, - 2 - ], - [ - "region", - "sha0", - 1, - 1 - ] - ], - "r1": [ - [ - "region", - "sha0", - 3, - 1 - ], - [ - "region", - "sha1", - 3, - 1 - ], - [ - "region", - "sha0", - 2, - 1 - ] - ], - "order": [ - [ - "region", - "sha0", - 1, - 1 - ], - [ - "region", - "sha1", - 2, - 2 - ], - [ - "r1-repeat", - [ - "region", - "sha0", - 1, - 2 - ] - ], - [ - "region", - "sha0", - 2, - 1 - ], - [ - "region", - "sha1", - 3, - 2 - ], - [ - "r1-repeat", - [ - "region", - "sha0", - 1, - 1 - ] - ], - [ - "region", - "sha1", - 1, - 2 - ], - [ - "region", - "sha0", - 2, - 2 - ], - [ - "region", - "sha1", - 1, - 0 - ], - [ - "r1-repeat", - [ - "region", - "sha0", - 2, - 0 - ] - ], - [ - "region", - "sha1", - 2, - 1 - ], - [ - "region", - "sha0", - 3, - 2 - ], - [ - "region", - "sha1", - 3, - 0 - ], - [ - "region", - "sha0", - 3, - 0 - ], - [ - "region", - "sha0", - 3, - 1 - ], - [ - "region", - "sha1", - 3, - 1 - ], - [ - "region", - "sha0", - 2, - 0 - ], - [ - "region", - "sha0", - 1, - 0 - ], - [ - "region", - "sha0", - 1, - 2 - ], - [ - "r1-repeat", - [ - "region", - "sha0", - 1, - 0 - ] - ], - [ - "region", - "sha1", - 1, - 1 - ], - [ - "region", - "sha1", - 2, - 0 - ] - ] - }, - "observed": { - "audit": [ - [ - "region", - "sha1", - 3, - 2 - ], - [ - "region", - "sha1", - 2, - 1 - ], - [ - "region", - "sha0", - 1, - 2 - ], - [ - "region", - "sha0", - 1, - 1 - ] - ], - "r1": [ - [ - "region", - "sha0", - 3, - 1 - ], - [ - "region", - "sha1", - 3, - 1 - ], - [ - "region", - "sha0", - 2, - 1 - ] - ], - "order": [ - [ - "region", - "sha0", - 1, - 1 - ], - [ - "region", - "sha1", - 2, - 2 - ], - [ - "r1-repeat", - [ - "region", - "sha0", - 1, - 2 - ] - ], - [ - "region", - "sha0", - 2, - 1 - ], - [ - "region", - "sha1", - 3, - 2 - ], - [ - "r1-repeat", - [ - "region", - "sha0", - 1, - 1 - ] - ], - [ - "region", - "sha1", - 1, - 2 - ], - [ - "region", - "sha0", - 2, - 2 - ], - [ - "region", - "sha1", - 1, - 0 - ], - [ - "r1-repeat", - [ - "region", - "sha0", - 2, - 0 - ] - ], - [ - "region", - "sha1", - 2, - 1 - ], - [ - "region", - "sha0", - 3, - 2 - ], - [ - "region", - "sha1", - 3, - 0 - ], - [ - "region", - "sha0", - 3, - 0 - ], - [ - "region", - "sha0", - 3, - 1 - ], - [ - "region", - "sha1", - 3, - 1 - ], - [ - "region", - "sha0", - 2, - 0 - ], - [ - "region", - "sha0", - 1, - 0 - ], - [ - "region", - "sha0", - 1, - 2 - ], - [ - "r1-repeat", - [ - "region", - "sha0", - 1, - 0 - ] - ], - [ - "region", - "sha1", - 1, - 1 - ], - [ - "region", - "sha1", - 2, - 0 - ] - ] - }, - "pass": true, - "implication": "sampling ranks canonical pre-blinding identities; the blind id is an alias only" - }, - { - "test": "...and the alias itself did change, so the control is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "blind ids are unique across distinct stimuli", - "expected": 22, - "observed": 22, - "pass": true, - "implication": "" - }, - { - "test": "the realized-set check passes on a clean stimulus set", - "expected": 22, - "observed": 22, - "pass": true, - "implication": "" - }, - { - "test": "an injected blind-ID collision aborts the build", - "expected": "BlindIdCollision", - "observed": "BlindIdCollision", - "pass": true, - "implication": "no overwrite, merge, last-write-wins, salt or re-roll is permitted" - }, - { - "test": "a duplicated stimulus identity aborts the build", - "expected": "DuplicateStimulusIdentity", - "observed": "DuplicateStimulusIdentity", - "pass": true, - "implication": "" - }, - { - "test": "...and the collision injection did not leak past its scope", - "expected": 22, - "observed": 22, - "pass": true, - "implication": "" - }, - { - "test": "primary stimuli render at exactly 300 dpi", - "expected": 300, - "observed": 300, - "pass": true, - "implication": "" - }, - { - "test": "R1 repeats render at exactly 330 dpi", - "expected": 330, - "observed": 330, - "pass": true, - "implication": "" - }, - { - "test": "the R1 scale is the frozen 300 x 1.10", - "expected": 330, - "observed": 330, - "pass": true, - "implication": "" - }, - { - "test": "primary and repeat scales differ, so R1 is a re-render and not a cache hit", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "1. identical inputs reproduce resample 0 exactly", - "expected": [ - [ - "doc-4", - false - ], - [ - "doc-5", - false - ], - [ - "doc-2", - false - ], - [ - "doc-8", - false - ], - [ - "doc-3", - true - ], - [ - "doc-2", - false - ], - [ - "doc-0", - true - ], - [ - "doc-5", - false - ], - [ - "doc-6", - true - ] - ], - "observed": [ - [ - "doc-4", - false - ], - [ - "doc-5", - false - ], - [ - "doc-2", - false - ], - [ - "doc-8", - false - ], - [ - "doc-3", - true - ], - [ - "doc-2", - false - ], - [ - "doc-0", - true - ], - [ - "doc-5", - false - ], - [ - "doc-6", - true - ] - ], - "pass": true, - "implication": "the draw consults something other than the committed identity and the two ordinals" - }, - { - "test": "2. identical inputs reproduce the WHOLE result", - "expected": { - "reported": true, - "statistic": [ - "section8", - "document-heading-discordance", - "P-head" - ], - "resamples": 10000, - "namespace": "bootstrap-document", - "seed": 20260807, - "unit": "document", - "n_documents": 9, - "events": 3, - "observed_rate": 0.3333333333333333, - "endpoint_indices": [ - 249, - 9749 - ], - "interval": [ - 0.0, - 0.6666666666666666 - ], - "gating": false - }, - "observed": { - "reported": true, - "statistic": [ - "section8", - "document-heading-discordance", - "P-head" - ], - "resamples": 10000, - "namespace": "bootstrap-document", - "seed": 20260807, - "unit": "document", - "n_documents": 9, - "events": 3, - "observed_rate": 0.3333333333333333, - "endpoint_indices": [ - 249, - 9749 - ], - "interval": [ - 0.0, - 0.6666666666666666 - ], - "gating": false - }, - "pass": true, - "implication": "two runs of the same committed inputs print different intervals" - }, - { - "test": "3. REVERSED input order reproduces every resample", - "expected": [ - [ - [ - "doc-4", - false - ], - [ - "doc-5", - false - ], - [ - "doc-2", - false - ], - [ - "doc-8", - false - ], - [ - "doc-3", - true - ], - [ - "doc-2", - false - ], - [ - "doc-0", - true - ], - [ - "doc-5", - false - ], - [ - "doc-6", - true - ] - ], - [ - [ - "doc-5", - false - ], - [ - "doc-0", - true - ], - [ - "doc-2", - false - ], - [ - "doc-5", - false - ], - [ - "doc-5", - false - ], - [ - "doc-8", - false - ], - [ - "doc-1", - false - ], - [ - "doc-0", - true - ], - [ - "doc-5", - false - ] - ], - [ - [ - "doc-7", - false - ], - [ - "doc-3", - true - ], - [ - "doc-2", - false - ], - [ - "doc-0", - true - ], - [ - "doc-4", - false - ], - [ - "doc-2", - false - ], - [ - "doc-1", - false - ], - [ - "doc-1", - false - ], - [ - "doc-6", - true - ] - ], - [ - [ - "doc-2", - false - ], - [ - "doc-1", - false - ], - [ - "doc-0", - true - ], - [ - "doc-2", - false - ], - [ - "doc-3", - true - ], - [ - "doc-8", - false - ], - [ - "doc-1", - false - ], - [ - "doc-3", - true - ], - [ - "doc-2", - false - ] - ], - [ - [ - "doc-5", - false - ], - [ - "doc-4", - false - ], - [ - "doc-5", - false - ], - [ - "doc-2", - false - ], - [ - "doc-1", - false - ], - [ - "doc-0", - true - ], - [ - "doc-8", - false - ], - [ - "doc-7", - false - ], - [ - "doc-2", - false - ] - ] - ], - "observed": [ - [ - [ - "doc-4", - false - ], - [ - "doc-5", - false - ], - [ - "doc-2", - false - ], - [ - "doc-8", - false - ], - [ - "doc-3", - true - ], - [ - "doc-2", - false - ], - [ - "doc-0", - true - ], - [ - "doc-5", - false - ], - [ - "doc-6", - true - ] - ], - [ - [ - "doc-5", - false - ], - [ - "doc-0", - true - ], - [ - "doc-2", - false - ], - [ - "doc-5", - false - ], - [ - "doc-5", - false - ], - [ - "doc-8", - false - ], - [ - "doc-1", - false - ], - [ - "doc-0", - true - ], - [ - "doc-5", - false - ] - ], - [ - [ - "doc-7", - false - ], - [ - "doc-3", - true - ], - [ - "doc-2", - false - ], - [ - "doc-0", - true - ], - [ - "doc-4", - false - ], - [ - "doc-2", - false - ], - [ - "doc-1", - false - ], - [ - "doc-1", - false - ], - [ - "doc-6", - true - ] - ], - [ - [ - "doc-2", - false - ], - [ - "doc-1", - false - ], - [ - "doc-0", - true - ], - [ - "doc-2", - false - ], - [ - "doc-3", - true - ], - [ - "doc-8", - false - ], - [ - "doc-1", - false - ], - [ - "doc-3", - true - ], - [ - "doc-2", - false - ] - ], - [ - [ - "doc-5", - false - ], - [ - "doc-4", - false - ], - [ - "doc-5", - false - ], - [ - "doc-2", - false - ], - [ - "doc-1", - false - ], - [ - "doc-0", - true - ], - [ - "doc-8", - false - ], - [ - "doc-7", - false - ], - [ - "doc-2", - false - ] - ] - ], - "pass": true, - "implication": "canonical sorting is missing or ineffective, so the caller's listing order selects different documents -- the defect withdrawn A29 measured" - }, - { - "test": "4. several permuted input orders give the IDENTICAL interval", - "expected": [ - [ - 0.0, - 0.6666666666666666 - ], - [ - 0.0, - 0.6666666666666666 - ], - [ - 0.0, - 0.6666666666666666 - ], - [ - 0.0, - 0.6666666666666666 - ] - ], - "observed": [ - [ - 0.0, - 0.6666666666666666 - ], - [ - 0.0, - 0.6666666666666666 - ], - [ - 0.0, - 0.6666666666666666 - ], - [ - 0.0, - 0.6666666666666666 - ] - ], - "pass": true, - "implication": "the reported interval depends on how the caller happened to list its documents" - }, - { - "test": "5. a different event pattern CHANGES the interval, so the statistic is live", - "expected": true, - "observed": true, - "pass": true, - "implication": "the interval is constant across event patterns, so it measures nothing and every reproducibility control above would pass on a dead quantity" - }, - { - "test": "6. changing the statistic identity CHANGES the draws, so domain separation is live", - "expected": true, - "observed": true, - "pass": true, - "implication": "two different statistics share a draw sequence, so the population component of the identity is decorative and a P-robust variant could silently reuse P-head's draws" - }, - { - "test": "7. NEGATIVE -- a duplicate document identity REFUSES", - "expected": "DUPLICATE_DOCUMENT_IDENTITY", - "observed": "DUPLICATE_DOCUMENT_IDENTITY", - "pass": true, - "implication": "a repeated document is silently weighted twice, though the document is the independent unit (8.3, red-team #7)" - }, - { - "test": "8. NEGATIVE -- an empty document set REFUSES", - "expected": "EMPTY_DOCUMENT_SET", - "observed": "EMPTY_DOCUMENT_SET", - "pass": true, - "implication": "N = 0 produces a division by zero or a vacuous interval instead of refusing" - }, - { - "test": "9. NEGATIVE -- a non-boolean event REFUSES", - "expected": "NON_BOOLEAN_EVENT", - "observed": "NON_BOOLEAN_EVENT", - "pass": true, - "implication": "a bare 0/1 is accepted -- it would work silently in sum() because bool subclasses int, so the wrong type would never be seen" - }, - { - "test": "17. NEGATIVE -- a HEADINGS-as-rows table cannot be passed as documents", - "expected": "DUPLICATE_DOCUMENT_IDENTITY", - "observed": "DUPLICATE_DOCUMENT_IDENTITY", - "pass": true, - "implication": "headings enter as independent observations, which is exactly the per-heading probability 8.3 forbids (8.1 measured 0.1926 vs 0.00498, a 39x ratio)" - }, - { - "test": "10. zero events REFUSE the bootstrap", - "expected": [ - false, - "ZERO_EVENTS_BOOTSTRAP_REFUSED" - ], - "observed": [ - false, - "ZERO_EVENTS_BOOTSTRAP_REFUSED" - ], - "pass": true, - "implication": "a bootstrap is reported at zero events, where 8.1 measured every resample degenerate" - }, - { - "test": "11. ...and no [0, 0] interval is emitted", - "expected": true, - "observed": true, - "pass": true, - "implication": "a degenerate [0.0, 0.0] is emitted and would read as a real confidence interval" - }, - { - "test": "...and the zero-event branch still yields the frozen closed form 1 - 0.05**(1/N)", - "expected": 0.19263617565013536, - "observed": 0.19263617565013536, - "pass": true, - "implication": "the licensed number is missing, leaving only an absence where 8.3 specifies a bound" - }, - { - "test": "12. a non-zero statistic executes ALL 10,000 replicates", - "expected": 10000, - "observed": 10000, - "pass": true, - "implication": "fewer replicates run than the frozen B, so the interval is computed on a smaller resample set than the amendment states" - }, - { - "test": "13. every replicate draws EXACTLY N documents", - "expected": [ - 9 - ], - "observed": [ - 9 - ], - "pass": true, - "implication": "a replicate draws a different number of documents than the vector holds" - }, - { - "test": "14. replacement is REAL -- documents repeat within a replicate", - "expected": true, - "observed": true, - "pass": true, - "implication": "no replicate ever picks a document twice in 200 tries, which would mean the draw is a permutation rather than sampling with replacement" - }, - { - "test": "...and the counted run agrees with the uninstrumented one", - "expected": { - "reported": true, - "statistic": [ - "section8", - "document-heading-discordance", - "P-head" - ], - "resamples": 10000, - "namespace": "bootstrap-document", - "seed": 20260807, - "unit": "document", - "n_documents": 9, - "events": 3, - "observed_rate": 0.3333333333333333, - "endpoint_indices": [ - 249, - 9749 - ], - "interval": [ - 0.0, - 0.6666666666666666 - ], - "gating": false - }, - "observed": { - "reported": true, - "statistic": [ - "section8", - "document-heading-discordance", - "P-head" - ], - "resamples": 10000, - "namespace": "bootstrap-document", - "seed": 20260807, - "unit": "document", - "n_documents": 9, - "events": 3, - "observed_rate": 0.3333333333333333, - "endpoint_indices": [ - 249, - 9749 - ], - "interval": [ - 0.0, - 0.6666666666666666 - ], - "gating": false - }, - "pass": true, - "implication": "instrumenting the resample changed the result, so control 12 measured a different run" - }, - { - "test": "15. the endpoint ranks are exactly 249 and 9749 at B = 10,000", - "expected": [ - 249, - 9749 - ], - "observed": [ - 249, - 9749 - ], - "pass": true, - "implication": "the order statistics move, so the interval is not the one A37.6 froze" - }, - { - "test": "16. the endpoints are OBSERVED replicate values -- no interpolation", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "implication": "an endpoint lies between two achievable k/N rates, i.e. a library interpolated it (NumPy's percentile default would) and the value was never actually resampled" - }, - { - "test": "...and the reported endpoint indices are the frozen ones", - "expected": [ - 249, - 9749 - ], - "observed": [ - 249, - 9749 - ], - "pass": true, - "implication": "the result reports different indices from the ones it used" - }, - { - "test": "18. the gate vector is exactly A27.6's nine decision-blocking conditions", - "expected": [ - "R1", - "N-A", - "N-B", - "N-C", - "S1", - "confirmatory X2-a", - "confirmatory X2-b", - "M9 evaluability", - "4.5 adequacy" - ], - "observed": [ - "R1", - "N-A", - "N-B", - "N-C", - "S1", - "confirmatory X2-a", - "confirmatory X2-b", - "M9 evaluability", - "4.5 adequacy" - ], - "pass": true, - "implication": "a condition was added to or removed from the decision gate" - }, - { - "test": "18b. NO bootstrap field appears anywhere in the gate vector", - "expected": [], - "observed": [], - "pass": true, - "implication": "the bootstrap became a decision-blocking condition, promoting a supplementary number into evidence" - }, - { - "test": "18c. ...and no key of the bootstrap result is a gate input", - "expected": [], - "observed": [], - "pass": true, - "implication": "a field the bootstrap emits is consumed by the architecture decision" - }, - { - "test": "18d. ...and both branches declare themselves non-gating", - "expected": [ - false, - false - ], - "observed": [ - false, - false - ], - "pass": true, - "implication": "a result does not carry gating: False, so a downstream reader could treat it as evidence" - }, - { - "test": "19. WITHOUT canonical sorting the same set in another order resamples DIFFERENTLY", - "expected": true, - "observed": true, - "pass": true, - "implication": "the unsorted counterfactual is also order-independent, which would mean control 3 passes for some other reason and proves nothing about the sorting" - }, - { - "test": "H recovering FEWER margin lines makes H the loser", - "expected": [ - "H", - true, - 1 - ], - "observed": [ - "H", - true, - 1 - ], - "pass": true, - "implication": "a one-line deficit does not fire, i.e. a tolerance was invented where the frozen text says 'loses', not 'loses more than N'" - }, - { - "test": "X recovering FEWER margin lines makes X the loser", - "expected": [ - "X", - true, - 1 - ], - "observed": [ - "X", - true, - 1 - ], - "pass": true, - "implication": "the clause is not symmetric between the two architectures" - }, - { - "test": "EQUAL counts do not fire the margin-line clause", - "expected": [ - null, - false, - 0 - ], - "observed": [ - null, - false, - 0 - ], - "pass": true, - "implication": "equality fires, so every document would carry a Rule 0 margin loss" - }, - { - "test": "a LARGE deficit fires exactly as a one-line deficit does", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "implication": "the clause is graded by severity, which the frozen text does not license" - }, - { - "test": "zero recovered on BOTH sides is still not a comparative loss", - "expected": false, - "observed": false, - "pass": true, - "implication": "a document where neither architecture recovers a margin line is reported as one architecture losing to the other" - }, - { - "test": "the same inputs select the same pages", - "expected": [ - 14, - 22, - 34, - 37 - ], - "observed": [ - 14, - 22, - 34, - 37 - ], - "pass": true, - "implication": "selection is not reproducible from committed inputs" - }, - { - "test": "input page PERMUTATION changes nothing", - "expected": [ - [ - 14, - 22, - 34, - 37 - ], - [ - 14, - 22, - 34, - 37 - ] - ], - "observed": [ - [ - 14, - 22, - 34, - 37 - ], - [ - 14, - 22, - 34, - 37 - ] - ], - "pass": true, - "implication": "the sample depends on the order the caller happened to list pages in" - }, - { - "test": "changing the document SHA changes the ranking", - "expected": true, - "observed": true, - "pass": true, - "implication": "two documents share a page sample, so the identity component is not consumed" - }, - { - "test": "changing the NAMESPACE changes the ranking", - "expected": true, - "observed": true, - "pass": true, - "implication": "domain separation is not live, so another purpose could reuse this draw sequence" - }, - { - "test": "k = max(1, ceil(0.10 * page_count)) on every boundary", - "expected": { - "1": 1, - "5": 1, - "9": 1, - "10": 1, - "11": 2, - "20": 2, - "21": 3 - }, - "observed": { - "1": 1, - "5": 1, - "9": 1, - "10": 1, - "11": 2, - "20": 2, - "21": 3 - }, - "pass": true, - "implication": "a short document selects ZERO pages and silently escapes the control, or a boundary rounds the wrong way" - }, - { - "test": "every document gets at least one sampled page", - "expected": true, - "observed": true, - "pass": true, - "implication": "the per-document consequence could attach to a document with no sampled page" - }, - { - "test": "an empty page set selects nothing rather than inventing a page", - "expected": [], - "observed": [], - "pass": true, - "implication": "a document with no pages yields a sampled page that does not exist" - }, - { - "test": "this module exposes NO cross-engine gate of its own", - "expected": [], - "observed": [], - "pass": true, - "implication": "a second executable copy of the gate exists here, so it can drift from X09.gate without any control noticing" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x16_occurrence_identity.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x16_occurrence_identity.json deleted file mode 100644 index 7016ab8f..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x16_occurrence_identity.json +++ /dev/null @@ -1,661 +0,0 @@ -{ - "population": "DEVELOPMENT + synthetic -- no holdout opened, nothing scored", - "builds_no_harness_component": true, - "occurrence_key": "(document_sha256, page_number, start_neutral_line_key, start_ngid)", - "ngid_is_identity_not_order": true, - "text_matching_used": false, - "kind_matching_used": false, - "emitted_ordinal_used": false, - "fidelity_contract": "strip_to_production(instrumented) == extract_anchors, element for element, on every DEVELOPMENT page consumed. When confirmatory execution is authorized the same assertion must cover every consumed confirmatory page.", - "refusal_classes": [ - "AMBIGUOUS_SOURCE_POSITION", - "CELLS_NOT_ALIGNED_WITH_PRINT_TEXT", - "MERGE_RECONSTRUCTION_MISMATCH", - "NO_NEUTRAL_INK_AT_OR_AFTER_START", - "NO_NEUTRAL_INK_ON_LINE", - "OFFSET_PAST_END_OF_LINE", - "PAGE_HAS_NO_PRINT_LINE_PROVENANCE", - "PRINT_LINE_INDEX_UNRESOLVED", - "START_NGID_NOT_OWNED_BY_NEUTRAL_LINE" - ], - "census": { - "documents": [ - { - "document": "114-hr-2029/4", - "document_sha256": "7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2", - "pages": 150, - "H": { - "occurrences": 339, - "keys_derived": 339, - "collision_lines": 2, - "collision_shapes": { - "section+subsection": 2 - } - }, - "X": { - "occurrences": 339, - "keys_derived": 339, - "collision_lines": 2, - "collision_shapes": { - "section+subsection": 2 - } - }, - "shared": 339, - "identity_agrees": 339, - "identity_disagrees": [], - "text_discordant": 0, - "text_discordant_identity_agrees": 0 - }, - { - "document": "118-s-4795/1", - "document_sha256": "7f7d022501496236b1b3fcc5dea7a8af4eed7ea9d272fd3568d71d8bc01766e8", - "pages": 150, - "H": { - "occurrences": 212, - "keys_derived": 212, - "collision_lines": 0, - "collision_shapes": {} - }, - "X": { - "occurrences": 212, - "keys_derived": 212, - "collision_lines": 0, - "collision_shapes": {} - }, - "shared": 212, - "identity_agrees": 212, - "identity_disagrees": [], - "text_discordant": 0, - "text_discordant_identity_agrees": 0 - }, - { - "document": "115-hr-5895/1", - "document_sha256": "3450f4825ff273988f9fe6b77e244a04f163f67fff9c4b8ee42588d81cea5157", - "pages": 150, - "H": { - "occurrences": 129, - "keys_derived": 129, - "collision_lines": 2, - "collision_shapes": { - "section+subsection": 2 - } - }, - "X": { - "occurrences": 129, - "keys_derived": 129, - "collision_lines": 2, - "collision_shapes": { - "section+subsection": 2 - } - }, - "shared": 129, - "identity_agrees": 129, - "identity_disagrees": [], - "text_discordant": 1, - "text_discordant_identity_agrees": 1 - }, - { - "document": "118-hr-8752/1", - "document_sha256": "d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0", - "pages": 150, - "H": { - "occurrences": 203, - "keys_derived": 203, - "collision_lines": 1, - "collision_shapes": { - "section+subsection": 1 - } - }, - "X": { - "occurrences": 203, - "keys_derived": 203, - "collision_lines": 1, - "collision_shapes": { - "section+subsection": 1 - } - }, - "shared": 203, - "identity_agrees": 203, - "identity_disagrees": [], - "text_discordant": 1, - "text_discordant_identity_agrees": 1 - }, - { - "document": "119-hr-1/1", - "document_sha256": "d324ddfc44c2d277ea095938ec94b3f78bb2cffd0bf02c92382d4bd1a40846e0", - "pages": 150, - "H": { - "occurrences": 169, - "keys_derived": 169, - "collision_lines": 0, - "collision_shapes": {} - }, - "X": { - "occurrences": 169, - "keys_derived": 169, - "collision_lines": 0, - "collision_shapes": {} - }, - "shared": 169, - "identity_agrees": 169, - "identity_disagrees": [], - "text_discordant": 28, - "text_discordant_identity_agrees": 28 - }, - { - "document": "118-hr-4366/1", - "document_sha256": "9c801bffcf796935315ba8cf17897389430e885a8aa6fc0dab1efe193d5e93df", - "pages": 150, - "H": { - "occurrences": 197, - "keys_derived": 197, - "collision_lines": 2, - "collision_shapes": { - "section+subsection": 2 - } - }, - "X": { - "occurrences": 197, - "keys_derived": 197, - "collision_lines": 2, - "collision_shapes": { - "section+subsection": 2 - } - }, - "shared": 197, - "identity_agrees": 197, - "identity_disagrees": [], - "text_discordant": 0, - "text_discordant_identity_agrees": 0 - } - ], - "totals": { - "H_occurrences": 1249, - "X_occurrences": 1249, - "shared": 1249, - "agree": 1249, - "disagree": 0, - "text_discordant": 30, - "text_discordant_agree": 30, - "H_collision_lines": 7, - "X_collision_lines": 7 - }, - "refusals": {}, - "fidelity_failures": [] - }, - "adversarial": { - "H": { - "section": { - "offset": 0, - "key": [ - "devsha", - 1, - [ - 1, - 0 - ], - 0 - ] - }, - "subsection": { - "offset": 10, - "key": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ] - } - }, - "X": { - "section": { - "offset": 0, - "key": [ - "devsha", - 1, - [ - 1, - 0 - ], - 0 - ] - }, - "subsection": { - "offset": 9, - "key": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ] - } - }, - "rejected_ordinal_with_A": 1, - "rejected_ordinal_without_A": 0, - "neutral_line": [ - 1, - 0 - ] - }, - "oracle_position": { - "key_A": [ - "devsha", - 1, - [ - 1, - 0 - ], - 0 - ], - "key_B": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "tie_refusal": "AMBIGUOUS_SOURCE_POSITION" - }, - "r1": { - "R1_start_identity_agreement": "4/4", - "items": [ - { - "ngid_truth": 0, - "agree": true, - "at_bbox_origin": true, - "primary_px": 0, - "repeat_px": 0 - }, - { - "ngid_truth": 8, - "agree": true, - "at_bbox_origin": false, - "primary_px": 200, - "repeat_px": 220 - }, - { - "ngid_truth": 22, - "agree": true, - "at_bbox_origin": false, - "primary_px": 550, - "repeat_px": 605 - }, - { - "ngid_truth": 40, - "agree": true, - "at_bbox_origin": false, - "primary_px": 1000, - "repeat_px": 1100 - } - ] - }, - "pdfium_conditioned_frame_note": "This occurrence-position join reads the neutral skeleton, so it inherits the already-frozen PDFIUM-CONDITIONED FRAME qualification when the cross-engine neutral-frame control (x09) fails. It does not change the oracle's heading text/role/parent truth source, which stays independently adjudicated.", - "tests": [ - { - "test": "the instrumented mirror reproduces production extract_anchors on EVERY page", - "expected": [], - "observed": [], - "pass": true, - "implication": "any drift makes every derived offset describe a recognition this study does not run" - }, - { - "test": "every cross-arm occurrence identity agrees", - "expected": 0, - "observed": 0, - "pass": true, - "implication": "" - }, - { - "test": "...and text-discordant occurrences were actually exercised, so it is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "implication": "if no occurrence had discordant text, space-invariance would be untested on real material" - }, - { - "test": "every text-discordant occurrence still agrees on identity", - "expected": 30, - "observed": 30, - "pass": true, - "implication": "" - }, - { - "test": "no occurrence was refused on development material", - "expected": {}, - "observed": {}, - "pass": true, - "implication": "" - }, - { - "test": "both arms recognise the section and its inline subsection on one physical line", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "the arms genuinely disagree about spacing, so space-invariance is exercised", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "A's identity is arm-invariant", - "expected": [ - "devsha", - 1, - [ - 1, - 0 - ], - 0 - ], - "observed": [ - "devsha", - 1, - [ - 1, - 0 - ], - 0 - ], - "pass": true, - "implication": "" - }, - { - "test": "B's identity is arm-invariant", - "expected": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "observed": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "pass": true, - "implication": "" - }, - { - "test": "A and B are separately identifiable on the one line", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "key_H(B) == key_X(B) when X is missing A", - "expected": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "observed": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "pass": true, - "implication": "" - }, - { - "test": "mirror: key_H(B) == key_X(B) when H is missing A", - "expected": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "observed": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "pass": true, - "implication": "" - }, - { - "test": "the rejected arm-local ordinal DOES renumber B when A is missing", - "expected": true, - "observed": true, - "pass": true, - "implication": "ordinal(B) 1 -> 0 while start_ngid stays fixed; a green run here proves the control can fail" - }, - { - "test": "oracle control 1: two occurrences on one line resolve to two distinct identities", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "the oracle's identity for A equals the architecture's", - "expected": [ - "devsha", - 1, - [ - 1, - 0 - ], - 0 - ], - "observed": [ - "devsha", - 1, - [ - 1, - 0 - ], - 0 - ], - "pass": true, - "implication": "" - }, - { - "test": "the oracle's identity for B equals the architecture's", - "expected": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "observed": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "pass": true, - "implication": "" - }, - { - "test": "oracle control 2: dropping the FIRST architecture occurrence leaves B's identity", - "expected": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "observed": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "pass": true, - "implication": "the resolver reads the skeleton and the adjudicated position; no arm output enters" - }, - { - "test": "oracle control 3: mirror -- dropping the SECOND leaves A's identity", - "expected": [ - "devsha", - 1, - [ - 1, - 0 - ], - 0 - ], - "observed": [ - "devsha", - 1, - [ - 1, - 0 - ], - 0 - ], - "pass": true, - "implication": "" - }, - { - "test": "oracle control 4: H/X spacing before the occurrence does not move the identity", - "expected": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "observed": [ - "devsha", - 1, - [ - 1, - 0 - ], - 8 - ], - "pass": true, - "implication": "" - }, - { - "test": "oracle control 5: an exact geometric tie is UNMATCHED", - "expected": [ - null, - "AMBIGUOUS_SOURCE_POSITION" - ], - "observed": [ - null, - "AMBIGUOUS_SOURCE_POSITION" - ], - "pass": true, - "implication": "" - }, - { - "test": "...but a non-tie still resolves, so the tie rule is not swallowing everything", - "expected": 12, - "observed": 12, - "pass": true, - "implication": "" - }, - { - "test": "oracle control 6: a start not owned by the reported neutral line is UNMATCHED", - "expected": [ - null, - "START_NGID_NOT_OWNED_BY_NEUTRAL_LINE" - ], - "observed": [ - null, - "START_NGID_NOT_OWNED_BY_NEUTRAL_LINE" - ], - "pass": true, - "implication": "" - }, - { - "test": "oracle control 7: a line with no neutral ink is UNMATCHED", - "expected": [ - null, - "NO_NEUTRAL_INK_ON_LINE" - ], - "observed": [ - null, - "NO_NEUTRAL_INK_ON_LINE" - ], - "pass": true, - "implication": "" - }, - { - "test": "oracle control 8: the source identity is fixed while the rejected ordinal moves", - "expected": true, - "observed": true, - "pass": true, - "implication": "" - }, - { - "test": "the repeat is genuinely a re-render: pixel coordinates differ away from the bbox origin", - "expected": true, - "observed": true, - "pass": true, - "implication": "if the coordinates matched, R1 would be testing a cache rather than a re-read" - }, - { - "test": "...and the bbox-origin occurrence is pixel 0 at both scales, as arithmetic requires", - "expected": [ - 0, - 0 - ], - "observed": [ - 0, - 0 - ], - "pass": true, - "implication": "" - }, - { - "test": "R1_start_identity_agreement: every repeated item resolves to the primary's identity", - "expected": 4, - "observed": 4, - "pass": true, - "implication": "" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x17_build_frames.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x17_build_frames.json deleted file mode 100644 index bec2dbb7..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x17_build_frames.json +++ /dev/null @@ -1,1380 +0,0 @@ -{ - "population": "SYNTHETIC + DEVELOPMENT -- no holdout opened, nothing scored", - "canonical_frames_json_created": false, - "d_frame_budget_applied": false, - "d_frame_sampled_or_truncated": false, - "region_size": 8, - "c_frame_max_per_document": 8, - "c_frame_namespace": "cframe-select", - "selection_seed": 20260807, - "anchor_equality": "the WHOLE emitted production Anchor value (page, line, kind, text, division); no reduced signature is invented for frames", - "development_note": "DEVELOPMENT figures are diagnostics that exercise the code paths. They are not study results and no metric is scored from them.", - "public_entrypoint": "build_frames.build_document_frame(sha, id, population, h_pages, x_pages)", - "private_synthetic_constructor": "build_frames._build_document_frame_from_inputs(...)", - "region_size_is_not_a_parameter_on_any_result_bearing_path": true, - "structural_preconditions_fail_closed": [ - "DUPLICATE_PAGE_NUMBER", - "PAGE_SET_MISMATCH", - "NEUTRAL_SKELETON_MISMATCH", - "PRINT_LINES_EMITTED_DRIFT", - "ANCHOR_PLACEMENT_REFUSED", - "UNKNOWN_POPULATION" - ], - "synthetic_precondition_evidence": { - "synthetic_scope_document": [ - [ - 2, - 1, - "account", - "OPERATIONS AND SUPPORT", - "" - ] - ], - "synthetic_scope_per_page": [] - }, - "synthetic_d_evidence": { - "text_injection": [ - "TEXT_DISCORDANCE" - ], - "merge_injection": [ - "SEGMENTATION_DISCORDANCE" - ], - "anchor_injection": [ - "ANCHOR_DISCORDANCE" - ], - "d_census_61": 61 - }, - "synthetic_c_evidence": { - "regions": 15, - "selected": 8 - }, - "development": { - "documents": [ - { - "document": "114-hr-2029/4", - "document_sha256": "7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2", - "pages": 60, - "neutral_lines": 1594, - "regions": 238, - "short_trailing_regions": 59, - "c_frame_selected": 8, - "d_frame_census": 170, - "d_text": 170, - "d_segmentation": 0, - "d_anchor": 0, - "both_absent_lines": 118, - "m0_risk_set_lines": 1476 - }, - { - "document": "118-s-4795/1", - "document_sha256": "7f7d022501496236b1b3fcc5dea7a8af4eed7ea9d272fd3568d71d8bc01766e8", - "pages": 60, - "neutral_lines": 1583, - "regions": 235, - "short_trailing_regions": 58, - "c_frame_selected": 8, - "d_frame_census": 165, - "d_text": 165, - "d_segmentation": 0, - "d_anchor": 0, - "both_absent_lines": 118, - "m0_risk_set_lines": 1465 - }, - { - "document": "115-hr-5895/1", - "document_sha256": "3450f4825ff273988f9fe6b77e244a04f163f67fff9c4b8ee42588d81cea5157", - "pages": 60, - "neutral_lines": 1651, - "regions": 239, - "short_trailing_regions": 60, - "c_frame_selected": 8, - "d_frame_census": 177, - "d_text": 177, - "d_segmentation": 0, - "d_anchor": 1, - "both_absent_lines": 178, - "m0_risk_set_lines": 1473 - }, - { - "document": "118-hr-8752/1", - "document_sha256": "d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0", - "pages": 60, - "neutral_lines": 1654, - "regions": 239, - "short_trailing_regions": 60, - "c_frame_selected": 8, - "d_frame_census": 171, - "d_text": 171, - "d_segmentation": 0, - "d_anchor": 1, - "both_absent_lines": 178, - "m0_risk_set_lines": 1476 - }, - { - "document": "119-hr-1/1", - "document_sha256": "d324ddfc44c2d277ea095938ec94b3f78bb2cffd0bf02c92382d4bd1a40846e0", - "pages": 60, - "neutral_lines": 1894, - "regions": 267, - "short_trailing_regions": 58, - "c_frame_selected": 8, - "d_frame_census": 159, - "d_text": 159, - "d_segmentation": 0, - "d_anchor": 8, - "both_absent_lines": 179, - "m0_risk_set_lines": 1715 - }, - { - "document": "118-hr-4366/1", - "document_sha256": "9c801bffcf796935315ba8cf17897389430e885a8aa6fc0dab1efe193d5e93df", - "pages": 60, - "neutral_lines": 1631, - "regions": 236, - "short_trailing_regions": 60, - "c_frame_selected": 8, - "d_frame_census": 171, - "d_text": 171, - "d_segmentation": 0, - "d_anchor": 0, - "both_absent_lines": 178, - "m0_risk_set_lines": 1453 - } - ], - "anchor_scope_comparison": [ - { - "document": "114-hr-2029/4", - "arm": "H", - "document_scope": 138, - "per_page_would_have_been": 138, - "document_scope_kinds": { - "title": 3, - "major": 3, - "account": 42, - "agency": 15, - "grouping": 2, - "section": 72, - "subsection": 1 - }, - "per_page_kinds": { - "title": 3, - "major": 3, - "account": 42, - "agency": 15, - "grouping": 2, - "section": 72, - "subsection": 1 - }, - "identical": true - }, - { - "document": "114-hr-2029/4", - "arm": "X", - "document_scope": 138, - "per_page_would_have_been": 138, - "document_scope_kinds": { - "title": 3, - "major": 3, - "account": 42, - "agency": 15, - "grouping": 2, - "section": 72, - "subsection": 1 - }, - "per_page_kinds": { - "title": 3, - "major": 3, - "account": 42, - "agency": 15, - "grouping": 2, - "section": 72, - "subsection": 1 - }, - "identical": true - }, - { - "document": "118-s-4795/1", - "arm": "H", - "document_scope": 96, - "per_page_would_have_been": 96, - "document_scope_kinds": { - "title": 2, - "major": 2, - "agency": 25, - "account": 53, - "grouping": 1, - "section": 13 - }, - "per_page_kinds": { - "title": 2, - "major": 2, - "agency": 25, - "account": 53, - "grouping": 1, - "section": 13 - }, - "identical": true - }, - { - "document": "118-s-4795/1", - "arm": "X", - "document_scope": 96, - "per_page_would_have_been": 96, - "document_scope_kinds": { - "title": 2, - "major": 2, - "agency": 25, - "account": 53, - "grouping": 1, - "section": 13 - }, - "per_page_kinds": { - "title": 2, - "major": 2, - "agency": 25, - "account": 53, - "grouping": 1, - "section": 13 - }, - "identical": true - }, - { - "document": "115-hr-5895/1", - "arm": "H", - "document_scope": 121, - "per_page_would_have_been": 120, - "document_scope_kinds": { - "title": 5, - "major": 7, - "account": 59, - "agency": 14, - "section": 31, - "subsection": 5 - }, - "per_page_kinds": { - "title": 5, - "major": 6, - "account": 59, - "agency": 14, - "section": 31, - "subsection": 5 - }, - "identical": false - }, - { - "document": "115-hr-5895/1", - "arm": "X", - "document_scope": 121, - "per_page_would_have_been": 120, - "document_scope_kinds": { - "title": 5, - "major": 7, - "account": 59, - "agency": 14, - "section": 31, - "subsection": 5 - }, - "per_page_kinds": { - "title": 5, - "major": 6, - "account": 59, - "agency": 14, - "section": 31, - "subsection": 5 - }, - "identical": false - }, - { - "document": "118-hr-8752/1", - "arm": "H", - "document_scope": 137, - "per_page_would_have_been": 137, - "document_scope_kinds": { - "title": 4, - "major": 4, - "agency": 15, - "account": 37, - "grouping": 4, - "section": 73 - }, - "per_page_kinds": { - "title": 4, - "major": 4, - "agency": 15, - "account": 37, - "grouping": 4, - "section": 73 - }, - "identical": true - }, - { - "document": "118-hr-8752/1", - "arm": "X", - "document_scope": 137, - "per_page_would_have_been": 137, - "document_scope_kinds": { - "title": 4, - "major": 4, - "agency": 15, - "account": 37, - "grouping": 4, - "section": 73 - }, - "per_page_kinds": { - "title": 4, - "major": 4, - "agency": 15, - "account": 37, - "grouping": 4, - "section": 73 - }, - "identical": true - }, - { - "document": "119-hr-1/1", - "arm": "H", - "document_scope": 48, - "per_page_would_have_been": 46, - "document_scope_kinds": { - "section": 16, - "title": 1, - "account": 15, - "subsection": 16 - }, - "per_page_kinds": { - "section": 16, - "title": 1, - "account": 13, - "subsection": 16 - }, - "identical": false - }, - { - "document": "119-hr-1/1", - "arm": "X", - "document_scope": 48, - "per_page_would_have_been": 46, - "document_scope_kinds": { - "section": 16, - "title": 1, - "account": 15, - "subsection": 16 - }, - "per_page_kinds": { - "section": 16, - "title": 1, - "account": 13, - "subsection": 16 - }, - "identical": false - }, - { - "document": "118-hr-4366/1", - "arm": "H", - "document_scope": 123, - "per_page_would_have_been": 123, - "document_scope_kinds": { - "title": 2, - "major": 2, - "account": 42, - "agency": 14, - "grouping": 2, - "section": 61 - }, - "per_page_kinds": { - "title": 2, - "major": 2, - "account": 42, - "agency": 14, - "grouping": 2, - "section": 61 - }, - "identical": true - }, - { - "document": "118-hr-4366/1", - "arm": "X", - "document_scope": 123, - "per_page_would_have_been": 123, - "document_scope_kinds": { - "title": 2, - "major": 2, - "account": 42, - "agency": 14, - "grouping": 2, - "section": 61 - }, - "per_page_kinds": { - "title": 2, - "major": 2, - "account": 42, - "agency": 14, - "grouping": 2, - "section": 61 - }, - "identical": true - } - ], - "census_mismatch": [], - "placement_mismatch": [] - }, - "tests": [ - { - "test": "control 10: a 19-line page yields 3 regions, the last a SHORT TRAILING one", - "expected": [ - 8, - 8, - 3 - ], - "observed": [ - 8, - 8, - 3 - ], - "pass": true, - "fails_when": "a dropped tail would make the last 1-7 neutral lines of every page unsamplable" - }, - { - "test": "...and the trailing region is flagged as such", - "expected": [ - false, - false, - true - ], - "observed": [ - false, - false, - true - ], - "pass": true, - "fails_when": "" - }, - { - "test": "control 2: every neutral line belongs to exactly one region", - "expected": [ - [ - [ - 1, - 0 - ], - [ - 1, - 1 - ], - [ - 1, - 2 - ], - [ - 1, - 3 - ], - [ - 1, - 4 - ], - [ - 1, - 5 - ], - [ - 1, - 6 - ], - [ - 1, - 7 - ], - [ - 1, - 8 - ], - [ - 1, - 9 - ], - [ - 1, - 10 - ], - [ - 1, - 11 - ], - [ - 1, - 12 - ], - [ - 1, - 13 - ], - [ - 1, - 14 - ], - [ - 1, - 15 - ], - [ - 1, - 16 - ], - [ - 1, - 17 - ], - [ - 1, - 18 - ] - ], - 19 - ], - "observed": [ - [ - [ - 1, - 0 - ], - [ - 1, - 1 - ], - [ - 1, - 2 - ], - [ - 1, - 3 - ], - [ - 1, - 4 - ], - [ - 1, - 5 - ], - [ - 1, - 6 - ], - [ - 1, - 7 - ], - [ - 1, - 8 - ], - [ - 1, - 9 - ], - [ - 1, - 10 - ], - [ - 1, - 11 - ], - [ - 1, - 12 - ], - [ - 1, - 13 - ], - [ - 1, - 14 - ], - [ - 1, - 15 - ], - [ - 1, - 16 - ], - [ - 1, - 17 - ], - [ - 1, - 18 - ] - ], - 19 - ], - "pass": true, - "fails_when": "a line in zero regions is unsamplable; a line in two would double-count it" - }, - { - "test": "control 11: each page starts its own region grid at ordinal 0", - "expected": [ - [ - 1, - 0 - ], - [ - 2, - 0 - ] - ], - "observed": [ - [ - 1, - 0 - ], - [ - 2, - 0 - ] - ], - "pass": true, - "fails_when": "a continuing ordinal would let a region span two pages" - }, - { - "test": "control 11b: I5 -- no region contains lines from two pages", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "" - }, - { - "test": "a 5-line page still yields one short trailing region", - "expected": [ - 5 - ], - "observed": [ - 5 - ], - "pass": true, - "fails_when": "" - }, - { - "test": "control 4: a page neither arm emitted creates NO D-frame membership", - "expected": [ - false - ], - "observed": [ - false - ], - "pass": true, - "fails_when": "counting a shared drop as discordance would flood the D-frame with page furniture" - }, - { - "test": "...and its lines are still IN the artifact and the region grid (I3)", - "expected": 4, - "observed": 4, - "pass": true, - "fails_when": "" - }, - { - "test": "...and are excluded from the M0 comparative risk set (I3)", - "expected": [ - false, - false, - false, - false - ], - "observed": [ - false, - false, - false, - false - ], - "pass": true, - "fails_when": "" - }, - { - "test": "control 6: a text change on ONE arm puts its region in D", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "if this passed without the injection, the text predicate would never fire" - }, - { - "test": "...and ONLY the mutated line is recorded as text-discordant (I4)", - "expected": [ - [ - 1, - 1 - ] - ], - "observed": [ - [ - 1, - 1 - ] - ], - "pass": true, - "fails_when": "" - }, - { - "test": "control 8: concordant lines inside a D region are NOT individually qualifying (I4)", - "expected": [ - false, - true, - false, - false - ], - "observed": [ - false, - true, - false, - false - ], - "pass": true, - "fails_when": "marking the whole region's lines discordant would fabricate M0a numerator entries" - }, - { - "test": "...and a text-only difference does NOT move the segmentation signature", - "expected": [ - false, - false, - false, - false - ], - "observed": [ - false, - false, - false, - false - ], - "pass": true, - "fails_when": "the two quantities are deliberately orthogonal; coupling them would corrupt M3" - }, - { - "test": "control 5: a merge on ONE arm puts its region in D via SEGMENTATION", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "a merge that did not move the signature would make M0b structurally blind" - }, - { - "test": "...and the merge did NOT move the projected text (grouping vs characters)", - "expected": [ - false, - false, - false, - false - ], - "observed": [ - false, - false, - false, - false - ], - "pass": true, - "fails_when": "if a merge moved the text, a segmentation difference would fabricate an M3 boundary error" - }, - { - "test": "control 7: an anchor-set difference alone puts the region in D", - "expected": [ - true, - [ - "ANCHOR_DISCORDANCE" - ] - ], - "observed": [ - true, - [ - "ANCHOR_DISCORDANCE" - ] - ], - "pass": true, - "fails_when": "PRE-REGISTRATION 5.8 makes a differing emitted anchor set region-level evidence" - }, - { - "test": "...with no line recorded as text- or segmentation-discordant (I4)", - "expected": [ - [], - [] - ], - "observed": [ - [], - [] - ], - "pass": true, - "fails_when": "" - }, - { - "test": "baseline: with no injection the region is NOT in D", - "expected": [ - false - ], - "observed": [ - false - ], - "pass": true, - "fails_when": "a baseline that was already in D would make all four injections vacuous" - }, - { - "test": "control 12: a 61-region D census is emitted in full, not truncated at 60", - "expected": 61, - "observed": 61, - "pass": true, - "fails_when": "truncating here would destroy the count A27.3 uses to decide whether Rule 1 may run" - }, - { - "test": "...and the artifact says so explicitly", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "" - }, - { - "test": "the complete region grid is enumerated from the skeleton (I1)", - "expected": 15, - "observed": 15, - "pass": true, - "fails_when": "" - }, - { - "test": "control 3: a body line BOTH arms dropped is still in the grid, in region 0, BOTH_ABSENT", - "expected": [ - "BOTH_ABSENT", - 0, - false - ], - "observed": [ - "BOTH_ABSENT", - 0, - false - ], - "pass": true, - "fails_when": "enumerating regions from EMITTED lines would make this line unsamplable and the shared failure invisible -- exactly what the C-frame exists to catch" - }, - { - "test": "...and its region is C-frame ELIGIBLE, i.e. reachable by the frozen draw", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "" - }, - { - "test": "...and the drop did not put its region in the D-frame (I3)", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "a shared drop belongs to RQ2 via the C-frame, never to the comparative D-frame" - }, - { - "test": "the C-frame draw is capped at 8 regions per document", - "expected": 8, - "observed": 8, - "pass": true, - "fails_when": "" - }, - { - "test": "control 9: the same identities select the same set AND the same order", - "expected": [ - [ - "region", - "devsha", - 5, - 2 - ], - [ - "region", - "devsha", - 4, - 2 - ], - [ - "region", - "devsha", - 3, - 1 - ], - [ - "region", - "devsha", - 2, - 2 - ], - [ - "region", - "devsha", - 1, - 0 - ], - [ - "region", - "devsha", - 4, - 1 - ], - [ - "region", - "devsha", - 2, - 0 - ], - [ - "region", - "devsha", - 2, - 1 - ] - ], - "observed": [ - [ - "region", - "devsha", - 5, - 2 - ], - [ - "region", - "devsha", - 4, - 2 - ], - [ - "region", - "devsha", - 3, - 1 - ], - [ - "region", - "devsha", - 2, - 2 - ], - [ - "region", - "devsha", - 1, - 0 - ], - [ - "region", - "devsha", - 4, - 1 - ], - [ - "region", - "devsha", - 2, - 0 - ], - [ - "region", - "devsha", - 2, - 1 - ] - ], - "pass": true, - "fails_when": "any listing-order dependence would make the sample irreproducible" - }, - { - "test": "...and building the document with pages listed in reverse selects the same regions", - "expected": [ - [ - 1, - 0 - ], - [ - 2, - 0 - ], - [ - 2, - 1 - ], - [ - 2, - 2 - ], - [ - 3, - 1 - ], - [ - 4, - 1 - ], - [ - 4, - 2 - ], - [ - 5, - 2 - ] - ], - "observed": [ - [ - 1, - 0 - ], - [ - 2, - 0 - ], - [ - 2, - 1 - ], - [ - 2, - 2 - ], - [ - 3, - 1 - ], - [ - 4, - 1 - ], - [ - 4, - 2 - ], - [ - 5, - 2 - ] - ], - "pass": true, - "fails_when": "" - }, - { - "test": "a selected C region stays selected even when an arm drops all of its content", - "expected": [ - [ - 1, - 0 - ], - [ - 2, - 0 - ], - [ - 2, - 1 - ], - [ - 2, - 2 - ], - [ - 3, - 1 - ], - [ - 4, - 1 - ], - [ - 4, - 2 - ], - [ - 5, - 2 - ] - ], - "observed": [ - [ - 1, - 0 - ], - [ - 2, - 0 - ], - [ - 2, - 1 - ], - [ - 2, - 2 - ], - [ - 3, - 1 - ], - [ - 4, - 1 - ], - [ - 4, - 2 - ], - [ - 5, - 2 - ] - ], - "pass": true, - "fails_when": "replacing such a region would delete the shared-failure evidence RQ2 exists to collect" - }, - { - "test": "a P-robust document receives NO C-frame selection", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "the C-frame quantity is defined on P-head only" - }, - { - "test": "...but its regions are still enumerated in full", - "expected": 15, - "observed": 15, - "pass": true, - "fails_when": "if enumeration were skipped too, the D-frame would silently lose the document" - }, - { - "test": "H carries a page X lacks -> ABORT", - "expected": "PAGE_SET_MISMATCH", - "observed": "PAGE_SET_MISMATCH", - "pass": true, - "fails_when": "dropping or intersecting the page set would silently shrink every denominator" - }, - { - "test": "X carries a page H lacks -> ABORT", - "expected": "PAGE_SET_MISMATCH", - "observed": "PAGE_SET_MISMATCH", - "pass": true, - "fails_when": "" - }, - { - "test": "matched page sets do NOT abort", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "if the clean case also aborted, the two controls above would prove nothing" - }, - { - "test": "a one-glyph neutral skeleton difference -> ABORT", - "expected": "NEUTRAL_SKELETON_MISMATCH", - "observed": "NEUTRAL_SKELETON_MISMATCH", - "pass": true, - "fails_when": "two skeletons would mean the arms' frames are not the same frame" - }, - { - "test": "an emitted line DELETED on one arm -> ABORT", - "expected": "PRINT_LINES_EMITTED_DRIFT", - "observed": "PRINT_LINES_EMITTED_DRIFT", - "pass": true, - "fails_when": "the bridge reads emitted[i] for print line i; a deletion shifts every later index" - }, - { - "test": "an emitted line's TEXT altered at equal list length -> ABORT", - "expected": "PRINT_LINES_EMITTED_DRIFT", - "observed": "PRINT_LINES_EMITTED_DRIFT", - "pass": true, - "fails_when": "equal cardinality is not enough; the texts must correspond index for index" - }, - { - "test": "an unplaceable anchor in H -> ABORT", - "expected": "ANCHOR_PLACEMENT_REFUSED", - "observed": "ANCHOR_PLACEMENT_REFUSED", - "pass": true, - "fails_when": "dropping it and comparing the rest would fabricate an ANCHOR_DISCORDANCE" - }, - { - "test": "...and the refusal is NOT interpreted as ANCHOR_DISCORDANCE", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "" - }, - { - "test": "an unplaceable anchor in X (asymmetric, different refusal class) -> ABORT", - "expected": "ANCHOR_PLACEMENT_REFUSED", - "observed": "ANCHOR_PLACEMENT_REFUSED", - "pass": true, - "fails_when": "" - }, - { - "test": "...and it too is NOT interpreted as ANCHOR_DISCORDANCE", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "" - }, - { - "test": "an UNRECOGNISED population string -> ABORT, not a quiet empty C-frame", - "expected": "UNKNOWN_POPULATION", - "observed": "UNKNOWN_POPULATION", - "pass": true, - "fails_when": "'not P-head' would otherwise draw zero C regions and look like a valid frame" - }, - { - "test": "...while the two real populations still build", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "" - }, - { - "test": "an UNRECOGNISED population reaching the PUBLIC entrypoint -> ABORT", - "expected": "UNKNOWN_POPULATION", - "observed": "UNKNOWN_POPULATION", - "pass": true, - "fails_when": "" - }, - { - "test": "a duplicate page record in H -> ABORT", - "expected": "DUPLICATE_PAGE_NUMBER", - "observed": "DUPLICATE_PAGE_NUMBER", - "pass": true, - "fails_when": "set equality is blind to duplicates, and the later page map silently collapses them" - }, - { - "test": "a duplicate page record in X -> ABORT", - "expected": "DUPLICATE_PAGE_NUMBER", - "observed": "DUPLICATE_PAGE_NUMBER", - "pass": true, - "fails_when": "" - }, - { - "test": "no result-bearing signature accepts a region size", - "expected": [ - false, - false, - false - ], - "observed": [ - false, - false, - false - ], - "pass": true, - "fails_when": "a region_size on any of these lets placement and the grid disagree while both look valid" - }, - { - "test": "the public entrypoint takes RUNNER OUTPUTS, not PageInputs, so gates cannot be skipped", - "expected": [ - "document_id", - "document_sha256", - "h_pages", - "population", - "x_pages" - ], - "observed": [ - "document_id", - "document_sha256", - "h_pages", - "population", - "x_pages" - ], - "pass": true, - "fails_when": "" - }, - { - "test": "...and the public path always yields the frozen 8-line grid", - "expected": [ - 8, - true - ], - "observed": [ - 8, - true - ], - "pass": true, - "fails_when": "" - }, - { - "test": "per-page extraction produces a DIFFERENT anchor census than document-scope", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "if these agreed, the scope control could not detect the defect it exists to catch" - }, - { - "test": "...specifically, document scope finds the account heading and per-page finds nothing", - "expected": [ - [ - "account" - ], - [] - ], - "observed": [ - [ - "account" - ], - [] - ], - "pass": true, - "fails_when": "size bands are derived over the SUPPLIED collection: page 2 alone carries no body-size prose, so no band exists and the account level vanishes" - }, - { - "test": "H anchor census is document-scoped and reaches the frame intact", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a per-page census would re-derive size bands per page and cut cross-page runs" - }, - { - "test": "X anchor census is document-scoped and reaches the frame intact", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "" - }, - { - "test": "H placement reproduces x14's over the DOCUMENT-SCOPE census", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a divergence would mean the frames use a bridge the study never approved" - }, - { - "test": "X placement reproduces x14's over the DOCUMENT-SCOPE census", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "" - }, - { - "test": "every development document retained at least one short trailing region", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "" - }, - { - "test": "no development document exceeded the 8-region C-frame cap", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x18_start_x_discriminability.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x18_start_x_discriminability.json deleted file mode 100644 index bd83fec0..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x18_start_x_discriminability.json +++ /dev/null @@ -1,793 +0,0 @@ -{ - "population": "SYNTHETIC + DEVELOPMENT -- no holdout opened, nothing scored", - "contract": "A32, committed before this probe existed; no pass threshold is stated here", - "supersedes_a30_3": false, - "primary_dpi": 300, - "r1_dpi": 330, - "translation_invariant": true, - "crop_rule_used": null, - "resolver": { - "controls": "10 boundary controls + translation invariance, both sides injected" - }, - "development": { - "documents": [ - { - "document": "114-hr-2029/4", - "G_min_margin_px_300": 2.7708053588867188, - "H_min_margin_px_300": 7.699998219807942, - "C_min_margin_px_300": 8.224995930989584, - "n_G": 154243, - "n_H": 339, - "n_C": 4 - }, - { - "document": "118-s-4795/1", - "G_min_margin_px_300": 4.695828755696614, - "H_min_margin_px_300": 9.35665766398112, - "C_min_margin_px_300": null, - "n_G": 153563, - "n_H": 212, - "n_C": 0 - }, - { - "document": "115-hr-5895/1", - "G_min_margin_px_300": 2.3697535196940103, - "H_min_margin_px_300": 7.699998219807942, - "C_min_margin_px_300": 9.12917455037435, - "n_G": 74932, - "n_H": 129, - "n_C": 4 - }, - { - "document": "118-hr-8752/1", - "G_min_margin_px_300": 2.3697535196940103, - "H_min_margin_px_300": 7.699998219807942, - "C_min_margin_px_300": 9.12917455037435, - "n_G": 110393, - "n_H": 203, - "n_C": 2 - }, - { - "document": "119-hr-1/1", - "G_min_margin_px_300": 2.3697535196940103, - "H_min_margin_px_300": 5.5708567301432295, - "C_min_margin_px_300": null, - "n_G": 153962, - "n_H": 169, - "n_C": 0 - }, - { - "document": "118-hr-4366/1", - "G_min_margin_px_300": 2.3697535196940103, - "H_min_margin_px_300": 7.699998219807942, - "C_min_margin_px_300": 9.129142761230469, - "n_G": 107277, - "n_H": 197, - "n_C": 4 - } - ], - "populations": { - "G": { - "dpi_300": { - "dpi": 300, - "n_targets": 754370, - "n_exact_x_ambiguous": 0, - "n_no_competitor_on_line": 50, - "n_in_margin_stats": 754320, - "margin_px": { - "min": 2.3697535196940103, - "p01": 4.9833933512369795, - "p05": 7.3396046956380205, - "median": 11.637528737386068, - "p95": 16.712443033854168 - }, - "guaranteed_integer_error_px": { - "min": 1, - "n_none": 0, - "p01": 4, - "p05": 6, - "median": 11.0 - }, - "k_buckets": { - "1": 812, - "2": 504, - "3": 4682, - "4": 5154, - "5": 7800, - "6": 49613, - "7": 87032, - "8+": 598773 - } - }, - "dpi_330": { - "dpi": 330, - "n_targets": 754370, - "n_exact_x_ambiguous": 0, - "n_no_competitor_on_line": 50, - "n_in_margin_stats": 754320, - "margin_px": { - "min": 2.6067288716634116, - "p01": 5.481732686360677, - "p05": 8.073565165201822, - "median": 12.801281611124674, - "p95": 18.383687337239582 - }, - "guaranteed_integer_error_px": { - "min": 2, - "n_none": 0, - "p01": 4, - "p05": 7, - "median": 12.0 - }, - "k_buckets": { - "2": 1308, - "3": 2250, - "4": 4172, - "5": 5625, - "6": 10594, - "7": 80413, - "8+": 650008 - } - } - }, - "H": { - "dpi_300": { - "dpi": 300, - "n_targets": 1249, - "n_exact_x_ambiguous": 0, - "n_no_competitor_on_line": 0, - "n_in_margin_stats": 1249, - "margin_px": { - "min": 5.5708567301432295, - "p01": 7.554181416829427, - "p05": 8.254178365071615, - "median": 15.580622355143229, - "p95": 21.706231435139973 - }, - "guaranteed_integer_error_px": { - "min": 5, - "n_none": 0, - "p01": 7, - "p05": 7, - "median": 15 - }, - "k_buckets": { - "5": 1, - "6": 1, - "7": 67, - "8+": 1180 - } - }, - "dpi_330": { - "dpi": 330, - "n_targets": 1249, - "n_exact_x_ambiguous": 0, - "n_no_competitor_on_line": 0, - "n_in_margin_stats": 1249, - "margin_px": { - "min": 6.127942403157552, - "p01": 8.30959955851237, - "p05": 9.079596201578775, - "median": 17.13868459065755, - "p95": 23.876854578653973 - }, - "guaranteed_integer_error_px": { - "min": 5, - "n_none": 0, - "p01": 7, - "p05": 8, - "median": 16 - }, - "k_buckets": { - "5": 1, - "7": 59, - "8+": 1189 - } - } - }, - "C": { - "dpi_300": { - "dpi": 300, - "n_targets": 14, - "n_exact_x_ambiguous": 0, - "n_no_competitor_on_line": 0, - "n_in_margin_stats": 14, - "margin_px": { - "min": 8.224995930989584, - "p01": 8.224995930989584, - "p05": 8.224995930989584, - "median": 12.346156438191732, - "p95": 15.580622355143229 - }, - "guaranteed_integer_error_px": { - "min": 7, - "n_none": 0, - "p01": 7, - "p05": 7, - "median": 11.5 - }, - "k_buckets": { - "7": 1, - "8+": 13 - } - }, - "dpi_330": { - "dpi": 330, - "n_targets": 14, - "n_exact_x_ambiguous": 0, - "n_no_competitor_on_line": 0, - "n_in_margin_stats": 14, - "margin_px": { - "min": 9.047495524088541, - "p01": 9.047495524088541, - "p05": 9.047495524088541, - "median": 13.580772082010906, - "p95": 17.13868459065755 - }, - "guaranteed_integer_error_px": { - "min": 8, - "n_none": 0, - "p01": 8, - "p05": 8, - "median": 12.5 - }, - "k_buckets": { - "8+": 14 - } - } - } - }, - "per_document_minima": [ - { - "document": "114-hr-2029/4", - "G_min_margin_px_300": 2.7708053588867188, - "H_min_margin_px_300": 7.699998219807942, - "C_min_margin_px_300": 8.224995930989584, - "n_G": 154243, - "n_H": 339, - "n_C": 4 - }, - { - "document": "118-s-4795/1", - "G_min_margin_px_300": 4.695828755696614, - "H_min_margin_px_300": 9.35665766398112, - "C_min_margin_px_300": null, - "n_G": 153563, - "n_H": 212, - "n_C": 0 - }, - { - "document": "115-hr-5895/1", - "G_min_margin_px_300": 2.3697535196940103, - "H_min_margin_px_300": 7.699998219807942, - "C_min_margin_px_300": 9.12917455037435, - "n_G": 74932, - "n_H": 129, - "n_C": 4 - }, - { - "document": "118-hr-8752/1", - "G_min_margin_px_300": 2.3697535196940103, - "H_min_margin_px_300": 7.699998219807942, - "C_min_margin_px_300": 9.12917455037435, - "n_G": 110393, - "n_H": 203, - "n_C": 2 - }, - { - "document": "119-hr-1/1", - "G_min_margin_px_300": 2.3697535196940103, - "H_min_margin_px_300": 5.5708567301432295, - "C_min_margin_px_300": null, - "n_G": 153962, - "n_H": 169, - "n_C": 0 - }, - { - "document": "118-hr-4366/1", - "G_min_margin_px_300": 2.3697535196940103, - "H_min_margin_px_300": 7.699998219807942, - "C_min_margin_px_300": 9.129142761230469, - "n_G": 107277, - "n_H": 197, - "n_C": 4 - } - ], - "exact_x_collisions": { - "n": 0, - "n_involving_heading_start_or_collision": 0, - "cases": [] - }, - "kind_strata_H": { - "account": 363, - "agency": 121, - "grouping": 18, - "major": 30, - "section": 598, - "subsection": 88, - "title": 31 - }, - "worst_cases": { - "H": [ - { - "document": "119-hr-1/1", - "page": 51, - "line": [ - 51, - 9 - ], - "ngid": 404, - "m_pt": 1.337005615234375, - "exact_x_same": false, - "kind": "subsection", - "x0": 179.23159790039062, - "x1": 182.05960083007812, - "y0": 496.6091003417969, - "y1": 508.5931091308594, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/119-hr-1/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "119-hr-1/1", - "page": 86, - "line": [ - 86, - 10 - ], - "ngid": 424, - "m_pt": 1.7430038452148438, - "exact_x_same": false, - "kind": "subsection", - "x0": 179.23226928710938, - "x1": 182.06027221679688, - "y0": 470.6114196777344, - "y1": 482.5954284667969, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/119-hr-1/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "119-hr-1/1", - "page": 16, - "line": [ - 16, - 20 - ], - "ngid": 942, - "m_pt": 1.8130035400390625, - "exact_x_same": false, - "kind": "account", - "x0": 206.8962860107422, - "x1": 208.6182861328125, - "y0": 218.6394500732422, - "y1": 222.6014404296875, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/119-hr-1/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "119-hr-1/1", - "page": 22, - "line": [ - 22, - 5 - ], - "ngid": 203, - "m_pt": 1.8130035400390625, - "exact_x_same": false, - "kind": "account", - "x0": 234.8961944580078, - "x1": 236.61819458007812, - "y0": 612.63037109375, - "y1": 616.5924072265625, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/119-hr-1/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "119-hr-1/1", - "page": 23, - "line": [ - 23, - 17 - ], - "ngid": 647, - "m_pt": 1.8130035400390625, - "exact_x_same": false, - "kind": "account", - "x0": 206.89639282226562, - "x1": 208.61839294433594, - "y0": 296.6329040527344, - "y1": 300.59490966796875, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/119-hr-1/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "119-hr-1/1", - "page": 25, - "line": [ - 25, - 18 - ], - "ngid": 776, - "m_pt": 1.8130035400390625, - "exact_x_same": false, - "kind": "account", - "x0": 234.8961944580078, - "x1": 236.61819458007812, - "y0": 274.6304931640625, - "y1": 278.5924987792969, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/119-hr-1/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "119-hr-1/1", - "page": 28, - "line": [ - 28, - 19 - ], - "ngid": 832, - "m_pt": 1.8130035400390625, - "exact_x_same": false, - "kind": "account", - "x0": 206.89599609375, - "x1": 208.6179962158203, - "y0": 244.62989807128906, - "y1": 248.59188842773438, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/119-hr-1/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "119-hr-1/1", - "page": 31, - "line": [ - 31, - 13 - ], - "ngid": 529, - "m_pt": 1.8130035400390625, - "exact_x_same": false, - "kind": "account", - "x0": 206.89547729492188, - "x1": 208.6174774169922, - "y0": 400.6304931640625, - "y1": 404.5924987792969, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/119-hr-1/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - } - ], - "C": [ - { - "document": "114-hr-2029/4", - "page": 136, - "line": [ - 136, - 24 - ], - "ngid": 1262, - "m_pt": 1.9739990234375, - "exact_x_same": false, - "kind": "subsection", - "x0": 237.55599975585938, - "x1": 242.03599548339844, - "y0": 106.59210205078125, - "y1": 118.6041030883789, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/114-hr-2029/4_reported-in-senate.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "118-hr-4366/1", - "page": 88, - "line": [ - 88, - 23 - ], - "ngid": 1256, - "m_pt": 2.1909942626953125, - "exact_x_same": false, - "kind": "subsection", - "x0": 241.9912109375, - "x1": 244.81919860839844, - "y0": 132.60610961914062, - "y1": 144.59010314941406, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/118-hr-4366/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "114-hr-2029/4", - "page": 66, - "line": [ - 66, - 12 - ], - "ngid": 627, - "m_pt": 2.1910018920898438, - "exact_x_same": false, - "kind": "subsection", - "x0": 241.97869873046875, - "x1": 244.80670166015625, - "y0": 418.6060791015625, - "y1": 430.590087890625, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/114-hr-2029/4_reported-in-senate.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "115-hr-5895/1", - "page": 49, - "line": [ - 49, - 21 - ], - "ngid": 1105, - "m_pt": 2.1910018920898438, - "exact_x_same": false, - "kind": "subsection", - "x0": 239.8323974609375, - "x1": 242.660400390625, - "y0": 184.60610961914062, - "y1": 196.59010314941406, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/115-hr-5895/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "115-hr-5895/1", - "page": 52, - "line": [ - 52, - 3 - ], - "ngid": 159, - "m_pt": 2.1910018920898438, - "exact_x_same": false, - "kind": "subsection", - "x0": 246.19119262695312, - "x1": 249.01919555664062, - "y0": 652.6060180664062, - "y1": 664.5900268554688, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/115-hr-5895/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "118-hr-8752/1", - "page": 92, - "line": [ - 92, - 4 - ], - "ngid": 190, - "m_pt": 2.1910018920898438, - "exact_x_same": false, - "kind": "subsection", - "x0": 244.14999389648438, - "x1": 246.97799682617188, - "y0": 626.6061401367188, - "y1": 638.5901489257812, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/118-hr-8752/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "118-hr-4366/1", - "page": 90, - "line": [ - 90, - 12 - ], - "ngid": 639, - "m_pt": 2.1910018920898438, - "exact_x_same": false, - "kind": "subsection", - "x0": 245.07119750976562, - "x1": 247.89920043945312, - "y0": 418.6060791015625, - "y1": 430.590087890625, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/118-hr-4366/1_reported-in-house.pdf", - "ink_within_6pt_left": [] - }, - { - "document": "114-hr-2029/4", - "page": 136, - "line": [ - 136, - 24 - ], - "ngid": 1252, - "m_pt": 3.7351531982421875, - "exact_x_same": false, - "kind": "section", - "x0": 178.33599853515625, - "x1": 185.91000366210938, - "y0": 108.69691467285156, - "y1": 118.59491729736328, - "pdf": "/Users/williamhea/Documents/Code/civictech/appropriations_bills/.claude/worktrees/a30-occurrence-identity/tests/corpus/114-hr-2029/4_reported-in-senate.pdf", - "ink_within_6pt_left": [] - } - ] - } - }, - "hard_stop_conditions": [], - "tests": [ - { - "test": "1. a clean isolated target resolves to itself", - "expected": [ - 10, - null - ], - "observed": [ - 10, - null - ], - "pass": true, - "fails_when": "if the exact geometric start did not resolve, the interface would be unusable" - }, - { - "test": "2a. just BEFORE the midpoint resolves to the left candidate", - "expected": 10, - "observed": 10, - "pass": true, - "fails_when": "" - }, - { - "test": "2b. just AFTER the midpoint resolves to the right candidate", - "expected": 20, - "observed": 20, - "pass": true, - "fails_when": "both sides are injected; a one-sided test could not locate the boundary" - }, - { - "test": "3. the EXACT midpoint is UNMATCHED", - "expected": [ - null, - "AMBIGUOUS_SOURCE_POSITION" - ], - "observed": [ - null, - "AMBIGUOUS_SOURCE_POSITION" - ], - "pass": true, - "fails_when": "no tie is broken by ngid, text, kind, order or y" - }, - { - "test": "4. a duplicate candidate x, when nearest, is UNMATCHED", - "expected": [ - null, - "AMBIGUOUS_SOURCE_POSITION" - ], - "observed": [ - null, - "AMBIGUOUS_SOURCE_POSITION" - ], - "pass": true, - "fails_when": "" - }, - { - "test": "4b. ...but a duplicate elsewhere does NOT block a clear winner", - "expected": 30, - "observed": 30, - "pass": true, - "fails_when": "if this refused too, control 4 would prove only that duplicates always fail" - }, - { - "test": "5. an absent physical line is UNMATCHED", - "expected": [ - null, - "NO_NEUTRAL_INK_ON_LINE" - ], - "observed": [ - null, - "NO_NEUTRAL_INK_ON_LINE" - ], - "pass": true, - "fails_when": "" - }, - { - "test": "6. a line with no neutral ink is UNMATCHED", - "expected": [ - null, - "NO_NEUTRAL_INK_ON_LINE" - ], - "observed": [ - null, - "NO_NEUTRAL_INK_ON_LINE" - ], - "pass": true, - "fails_when": "" - }, - { - "test": "7. heading TEXT cannot change resolution (it is not an input)", - "expected": [ - 10, - null - ], - "observed": [ - 10, - null - ], - "pass": true, - "fails_when": "" - }, - { - "test": "8. anchor KIND cannot change resolution (it is not an input)", - "expected": [ - 10, - null - ], - "observed": [ - 10, - null - ], - "pass": true, - "fails_when": "reversing candidate order stands in for any kind/order-derived ranking" - }, - { - "test": "9. an H/X spacing disagreement does not move the target's geometry identity", - "expected": [ - 10, - null - ], - "observed": [ - 10, - null - ], - "pass": true, - "fails_when": "" - }, - { - "test": "10a. same-line section start resolves to the section", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "" - }, - { - "test": "10b. same-line inline-subsection start resolves to the subsection", - "expected": 8, - "observed": 8, - "pass": true, - "fails_when": "if these collapsed, A30's whole reason for existing would be defeated" - }, - { - "test": "translation invariance: the crop origin cannot change any resolution", - "expected": { - "0.0": 10, - "36.0": 10, - "-12.5": 10, - "1000.0": 10 - }, - "observed": { - "0.0": 10, - "36.0": 10, - "-12.5": 10, - "1000.0": 10 - }, - "pass": true, - "fails_when": "if it could, this study would require the unfrozen crop rule" - }, - { - "test": "...and the pixel driver agrees with the point-space resolver", - "expected": 10, - "observed": 10, - "pass": true, - "fails_when": "" - }, - { - "test": "population H is non-empty, so the task-relevant result is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "" - }, - { - "test": "population C is non-empty, so the adversarial case was actually exercised", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "" - }, - { - "test": "no HARD stop condition on DEVELOPMENT", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C0_114-hr-2029-4_p136_g1262_300dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C0_114-hr-2029-4_p136_g1262_300dpi.png deleted file mode 100644 index fd8d28f6..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C0_114-hr-2029-4_p136_g1262_300dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C0_114-hr-2029-4_p136_g1262_330dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C0_114-hr-2029-4_p136_g1262_330dpi.png deleted file mode 100644 index 15b2046b..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C0_114-hr-2029-4_p136_g1262_330dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C1_118-hr-4366-1_p88_g1256_300dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C1_118-hr-4366-1_p88_g1256_300dpi.png deleted file mode 100644 index 8bbac9a4..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C1_118-hr-4366-1_p88_g1256_300dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C1_118-hr-4366-1_p88_g1256_330dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C1_118-hr-4366-1_p88_g1256_330dpi.png deleted file mode 100644 index dfb92ee7..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C1_118-hr-4366-1_p88_g1256_330dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C2_114-hr-2029-4_p66_g627_300dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C2_114-hr-2029-4_p66_g627_300dpi.png deleted file mode 100644 index b58b2d99..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C2_114-hr-2029-4_p66_g627_300dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C2_114-hr-2029-4_p66_g627_330dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C2_114-hr-2029-4_p66_g627_330dpi.png deleted file mode 100644 index fc1a6736..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C2_114-hr-2029-4_p66_g627_330dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C3_115-hr-5895-1_p49_g1105_300dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C3_115-hr-5895-1_p49_g1105_300dpi.png deleted file mode 100644 index f9d6f510..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C3_115-hr-5895-1_p49_g1105_300dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C3_115-hr-5895-1_p49_g1105_330dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C3_115-hr-5895-1_p49_g1105_330dpi.png deleted file mode 100644 index 605442ad..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/C3_115-hr-5895-1_p49_g1105_330dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H0_119-hr-1-1_p51_g404_300dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H0_119-hr-1-1_p51_g404_300dpi.png deleted file mode 100644 index 5c83be9f..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H0_119-hr-1-1_p51_g404_300dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H0_119-hr-1-1_p51_g404_330dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H0_119-hr-1-1_p51_g404_330dpi.png deleted file mode 100644 index 58fb7bec..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H0_119-hr-1-1_p51_g404_330dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H1_119-hr-1-1_p86_g424_300dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H1_119-hr-1-1_p86_g424_300dpi.png deleted file mode 100644 index 8c0af8cc..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H1_119-hr-1-1_p86_g424_300dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H1_119-hr-1-1_p86_g424_330dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H1_119-hr-1-1_p86_g424_330dpi.png deleted file mode 100644 index 1db5b80c..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H1_119-hr-1-1_p86_g424_330dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H2_119-hr-1-1_p16_g942_300dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H2_119-hr-1-1_p16_g942_300dpi.png deleted file mode 100644 index bf171714..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H2_119-hr-1-1_p16_g942_300dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H2_119-hr-1-1_p16_g942_330dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H2_119-hr-1-1_p16_g942_330dpi.png deleted file mode 100644 index b50a0dc1..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H2_119-hr-1-1_p16_g942_330dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H3_119-hr-1-1_p22_g203_300dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H3_119-hr-1-1_p22_g203_300dpi.png deleted file mode 100644 index 4e8865ee..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H3_119-hr-1-1_p22_g203_300dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H3_119-hr-1-1_p22_g203_330dpi.png b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H3_119-hr-1-1_p22_g203_330dpi.png deleted file mode 100644 index eb0b399f..00000000 Binary files a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_dev/H3_119-hr-1-1_p22_g203_330dpi.png and /dev/null differ diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_edge_diagnostic.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_edge_diagnostic.json deleted file mode 100644 index 2e4aace5..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x19_raster_edge_diagnostic.json +++ /dev/null @@ -1,780 +0,0 @@ -{ - "population": "DEVELOPMENT worst cases from x18 -- no holdout opened, nothing scored", - "summary": { - "renderer": "MuPDF (pymupdf)", - "renderer_version": "('1.28.2', '1.28.2', None)", - "edge_rule": "exact raster occupancy -- leftmost column with ANY alpha; no intensity threshold", - "ocr_used": false, - "segmentation_heuristic_used": false, - "window_pt": 3.0, - "n_attempted": 16, - "n_cleanly_measured": 15, - "n_excluded_neighbour_ink": { - "H": 0, - "C": 0 - }, - "n_excluded_non_glyph_vector_ink": 1, - "excluded_non_glyph_vector_ink": [ - { - "document": "114-hr-2029/4", - "page": 66, - "ngid": 627, - "n_vector_paths_in_band": 1 - } - ], - "visible_edge_offset_px_300": { - "min": -0.5, - "max": 0.5, - "values": [ - 0.5, - 0.5, - -0.5, - -0.5, - -0.5, - -0.5, - -0.5, - -0.5, - -0.5, - 0.5, - 0.5, - 0.5, - 0.5, - 0.5, - 0.5 - ] - }, - "visible_edge_offset_px_330": { - "min": -0.75, - "max": 1.25, - "values": [ - 0.25, - 0.25, - 0.25, - 1.25, - 0.25, - 1.25, - 0.25, - 0.25, - -0.75, - -0.75, - 0.25, - 0.25, - -0.75, - 0.25, - 0.25 - ] - }, - "limitation": "Two exclusions, both STRUCTURAL rather than heuristic. (1) A target whose window contains a neighbouring GLYPH's ink is not measured; its intruding boxes are listed. (2) A target whose band is crossed by NON-GLYPH VECTOR INK is not measured: GPO reported prints strike through matter deleted in committee, and that rule is a drawn path with no text index, hence no ngid -- correctly outside the neutral skeleton, but rendered by MuPDF across the whole line. Raster occupancy cannot separate rule ink from glyph ink, and doing so would need the segmentation heuristic the A32 contract forbids. Crops for the worst cases are committed for human inspection.", - "mupdf_vs_pdfium_glyph_bbox": "Separately measured and reported so it cannot look like an unexplained discrepancy: MuPDF's glyph BBOX x0 sits systematically left of PDFium's char-box x0 -- about -1.23 pt for '(' and -0.90 pt for the open quote at DeVinne/14. That is the font's left side bearing, i.e. reserved whitespace before the ink, and it is NOT what a human marking the left edge of a printed character would point at. The raster-occupancy offsets above are the task-relevant quantity.", - "for_the_reviewer": "The struck-through case is not a defect in A30.3's geometry: the glyph's x0 is unchanged and the resolver never reads pixels. It is a PROMPT-DESIGN observation -- an adjudicator asked for 'the left edge of the first printed character' will see a horizontal rule running through that character on reported prints. Whether that needs wording in adjudicator_prompt.md is the reviewer's call; A32 states no ruling." - }, - "H": [ - { - "document": "119-hr-1/1", - "page": 51, - "ngid": 404, - "kind": "subsection", - "m_pt": 1.337005615234375, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 25, - 51 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": 0.5, - "image": "H0_119-hr-1-1_p51_g404_300dpi.png", - "image_sha256": "f6fa1428320b19befb1147c63f2ec5137ba661452fe4d95f2d8a0a41b112352e" - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 28, - 56 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25, - "image": "H0_119-hr-1-1_p51_g404_330dpi.png", - "image_sha256": "e8a15c2ab823f800bb318945190d1c60cbe97dc31912af24312fd5df9afb8fda" - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 177.9995880126953, - "char": "(", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 179.23159790039062, - "delta_pt": -1.2320098876953125, - "delta_px_300": -5.133374532063802, - "found": true - }, - "measured": true - }, - { - "document": "119-hr-1/1", - "page": 86, - "ngid": 424, - "kind": "subsection", - "m_pt": 1.7430038452148438, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 25, - 51 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": 0.5, - "image": "H1_119-hr-1-1_p86_g424_300dpi.png", - "image_sha256": "fdfc2d0091b3657cefd768ff05e635f81f6df979ba0bbea9bd979fc05fcee1b6" - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 28, - 56 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25, - "image": "H1_119-hr-1-1_p86_g424_330dpi.png", - "image_sha256": "55b9950309c02595160ab75b4cdb293b75edc20be42cd02ba488d2f842047698" - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 178.0002899169922, - "char": "(", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 179.23226928710938, - "delta_pt": -1.2319793701171875, - "delta_px_300": -5.133247375488281, - "found": true - }, - "measured": true - }, - { - "document": "119-hr-1/1", - "page": 16, - "ngid": 942, - "kind": "account", - "m_pt": 1.8130035400390625, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 21, - 18 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 12, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": -0.5, - "image": "H2_119-hr-1-1_p16_g942_300dpi.png", - "image_sha256": "0ba7c7651f0620fea0a8eff4bc3d412616fec3dd88d89f101d585c7084834d23" - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 23, - 19 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25, - "image": "H2_119-hr-1-1_p16_g942_330dpi.png", - "image_sha256": "b0e9c0dcd3c7db34cad03d5f74f446a5f857a8e301e99233cdd9cd8deada7a0b" - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 206.0002899169922, - "char": "\u2018", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 206.8962860107422, - "delta_pt": -0.89599609375, - "delta_px_300": -3.7333170572916665, - "found": true - }, - "measured": true - }, - { - "document": "119-hr-1/1", - "page": 22, - "ngid": 203, - "kind": "account", - "m_pt": 1.8130035400390625, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 20, - 18 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 12, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": -0.5, - "image": "H3_119-hr-1-1_p22_g203_300dpi.png", - "image_sha256": "82d1e0b0529a809cd2268ad3b742b0159bb6750d25acc20bb098d104b1498028" - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 23, - 20 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 15, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 1.25, - "image": "H3_119-hr-1-1_p22_g203_330dpi.png", - "image_sha256": "0ba32184928fc8523abcd292b21d8603f6ef64d9b4a892b00f8185c150cec243" - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 234.0001983642578, - "char": "\u2018", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 234.8961944580078, - "delta_pt": -0.89599609375, - "delta_px_300": -3.7333170572916665, - "found": true - }, - "measured": true - }, - { - "document": "119-hr-1/1", - "page": 23, - "ngid": 647, - "kind": "account", - "m_pt": 1.8130035400390625, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 21, - 18 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 12, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": -0.5 - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 23, - 19 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25 - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 206.00039672851562, - "char": "\u2018", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 206.89639282226562, - "delta_pt": -0.89599609375, - "delta_px_300": -3.7333170572916665, - "found": true - }, - "measured": true - }, - { - "document": "119-hr-1/1", - "page": 25, - "ngid": 776, - "kind": "account", - "m_pt": 1.8130035400390625, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 20, - 17 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 12, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": -0.5 - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 23, - 19 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 15, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 1.25 - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 234.0001983642578, - "char": "\u2018", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 234.8961944580078, - "delta_pt": -0.89599609375, - "delta_px_300": -3.7333170572916665, - "found": true - }, - "measured": true - }, - { - "document": "119-hr-1/1", - "page": 28, - "ngid": 832, - "kind": "account", - "m_pt": 1.8130035400390625, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 21, - 17 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 12, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": -0.5 - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 23, - 19 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25 - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 205.99998474121094, - "char": "\u2018", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 206.89599609375, - "delta_pt": -0.8960113525390625, - "delta_px_300": -3.7333806355794272, - "found": true - }, - "measured": true - }, - { - "document": "119-hr-1/1", - "page": 31, - "ngid": 529, - "kind": "account", - "m_pt": 1.8130035400390625, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 21, - 17 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 12, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": -0.5 - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 23, - 19 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25 - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 205.99948120117188, - "char": "\u2018", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 206.89547729492188, - "delta_pt": -0.89599609375, - "delta_px_300": -3.7333170572916665, - "found": true - }, - "measured": true - } - ], - "C": [ - { - "document": "114-hr-2029/4", - "page": 136, - "ngid": 1262, - "kind": "subsection", - "m_pt": 1.9739990234375, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 32, - 51 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 12, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": -0.5, - "image": "C0_114-hr-2029-4_p136_g1262_300dpi.png", - "image_sha256": "a30eb23f2b78a1d67fdebc65f8e1d3c4bbc2f58e0e40a870205f8ebba7c117ae" - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 35, - 56 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": -0.75, - "image": "C0_114-hr-2029-4_p136_g1262_330dpi.png", - "image_sha256": "5e752d5932c782b80f0a74fd8cf99f340564336a2812efcef9d365a3ae73e2d6" - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 236.5199432373047, - "char": "(", - "font": "DeVinne-Italic", - "size": 14.0, - "pdfium_x0": 237.55599975585938, - "delta_pt": -1.0360565185546875, - "delta_px_300": -4.316902160644531, - "found": true - }, - "measured": true - }, - { - "document": "118-hr-4366/1", - "page": 88, - "ngid": 1256, - "kind": "subsection", - "m_pt": 2.1909942626953125, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 26, - 51 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": 0.5, - "image": "C1_118-hr-4366-1_p88_g1256_300dpi.png", - "image_sha256": "899a180fcf2b2fe181171e00d9740e5f5e71037889b18c9dc15b4ede596eb26a" - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 28, - 56 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": -0.75, - "image": "C1_118-hr-4366-1_p88_g1256_330dpi.png", - "image_sha256": "ac679a5baf8e5a4c1a88fb284c810f0475f6601d255193fead34745ddc11b116" - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 240.7592315673828, - "char": "(", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 241.9912109375, - "delta_pt": -1.2319793701171875, - "delta_px_300": -5.133247375488281, - "found": true - }, - "measured": true - }, - { - "document": "114-hr-2029/4", - "page": 66, - "ngid": 627, - "kind": "subsection", - "m_pt": 2.1910018920898438, - "dpi_300": { - "dpi": 300, - "measured": false, - "why_not_measured": "NON_GLYPH_VECTOR_INK_IN_BAND", - "n_vector_paths_in_band": 1, - "note": "struck-through matter: a drawn rule, not a glyph; no ngid by construction" - }, - "dpi_330": { - "dpi": 330, - "measured": false, - "why_not_measured": "NON_GLYPH_VECTOR_INK_IN_BAND", - "n_vector_paths_in_band": 1, - "note": "struck-through matter: a drawn rule, not a glyph; no ngid by construction" - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 240.7467041015625, - "char": "(", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 241.97869873046875, - "delta_pt": -1.23199462890625, - "delta_px_300": -5.133310953776042, - "found": true - }, - "measured": false, - "why_not_measured": "NON_GLYPH_VECTOR_INK_IN_BAND" - }, - { - "document": "115-hr-5895/1", - "page": 49, - "ngid": 1105, - "kind": "subsection", - "m_pt": 2.1910018920898438, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 26, - 51 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": 0.5, - "image": "C3_115-hr-5895-1_p49_g1105_300dpi.png", - "image_sha256": "18b9d0c7e8fad8d22414ff824f1e7b9f373e5c0657cf1604f824951775faf294" - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 28, - 56 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25, - "image": "C3_115-hr-5895-1_p49_g1105_330dpi.png", - "image_sha256": "3781c7c72e26bf51f5ce53ce85c1cc27b9bf25e8abf5331db5b53beeb2e50a7c" - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 238.60044860839844, - "char": "(", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 239.8323974609375, - "delta_pt": -1.2319488525390625, - "delta_px_300": -5.133120218912761, - "found": true - }, - "measured": true - }, - { - "document": "115-hr-5895/1", - "page": 52, - "ngid": 159, - "kind": "subsection", - "m_pt": 2.1910018920898438, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 25, - 51 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": 0.5 - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 28, - 56 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25 - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 244.95924377441406, - "char": "(", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 246.19119262695312, - "delta_pt": -1.2319488525390625, - "delta_px_300": -5.133120218912761, - "found": true - }, - "measured": true - }, - { - "document": "118-hr-8752/1", - "page": 92, - "ngid": 190, - "kind": "subsection", - "m_pt": 2.1910018920898438, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 26, - 51 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": 0.5 - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 27, - 56 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": -0.75 - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 242.9180450439453, - "char": "(", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 244.14999389648438, - "delta_pt": -1.2319488525390625, - "delta_px_300": -5.133120218912761, - "found": true - }, - "measured": true - }, - { - "document": "118-hr-4366/1", - "page": 90, - "ngid": 639, - "kind": "subsection", - "m_pt": 2.1910018920898438, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 25, - 51 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": 0.5 - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 28, - 56 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25 - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 243.83924865722656, - "char": "(", - "font": "DeVinne", - "size": 14.0, - "pdfium_x0": 245.07119750976562, - "delta_pt": -1.2319488525390625, - "delta_px_300": -5.133120218912761, - "found": true - }, - "measured": true - }, - { - "document": "114-hr-2029/4", - "page": 136, - "ngid": 1252, - "kind": "section", - "m_pt": 3.7351531982421875, - "dpi_300": { - "dpi": 300, - "crop_px": [ - 45, - 43 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 13, - "geometric_x0_column_px": 12.5, - "visible_edge_offset_px": 0.5 - }, - "dpi_330": { - "dpi": 330, - "crop_px": [ - 50, - 46 - ], - "window_pt": 3.0, - "leftmost_inked_column_px": 14, - "geometric_x0_column_px": 13.75, - "visible_edge_offset_px": 0.25 - }, - "mupdf_glyph_bbox_delta": { - "mupdf_x0": 178.0, - "char": "S", - "font": "DeVinne-Italic", - "size": 14.0, - "pdfium_x0": 178.33599853515625, - "delta_pt": -0.33599853515625, - "delta_px_300": -1.399993896484375, - "found": true - }, - "measured": true - } - ], - "tests": [ - { - "test": "a vector-ink exclusion is NOT counted in the measured denominator", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the outer row once said measured=True before measure() ran, inflating n_measured by one" - }, - { - "test": "...and contributes NO value to either offset distribution", - "expected": [ - 0, - 0 - ], - "observed": [ - 0, - 0 - ], - "pass": true, - "fails_when": "" - }, - { - "test": "the denominator is exact: cleanly measured + excluded == attempted", - "expected": 16, - "observed": 16, - "pass": true, - "fails_when": "" - }, - { - "test": "every rendered crop contained ink, so the PDF->MuPDF coordinate mapping is right", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "an empty crop would mean the bottom-up/top-down y conversion is wrong and every offset below would be measuring blank paper" - }, - { - "test": "at least one worst-case target admitted a clean, unsegmented measurement", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "if all were excluded, this diagnostic would report nothing and must say so" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x20_oracle_crop_coordinates.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x20_oracle_crop_coordinates.json deleted file mode 100644 index ad131184..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x20_oracle_crop_coordinates.json +++ /dev/null @@ -1,1256 +0,0 @@ -{ - "population": "SYNTHETIC + DEVELOPMENT -- no holdout opened, nothing scored", - "contract": "A33 (crop + transform), corrected forward by A34 (renderer epsilon)", - "renderer": "MuPDF (pymupdf)", - "renderer_version": "('1.28.2', '1.28.2', None)", - "region_bbox_rule": "minimal axis-aligned union of COMMITTED neutral-line bboxes, zero padding", - "mupdf_round_eps_px": 0.001, - "frozen_inversion": "s = DPI/72; eps = 0.001 px; device_x0 = floor(bbox_x0*s + eps); pdf_x = (device_x0 + start_x_px) / s", - "frozen_width_validation": "ceil(bbox_x1*s - eps) - floor(bbox_x0*s + eps)", - "a33_stated_was": "device_x0 = floor(bbox_x0 * DPI/72) -- epsilon-free, corrected by A34", - "a30_3_sketch_was": "pdf_x = bbox_x0 + (start_x_px / image_width) * (bbox_x1 - bbox_x0)", - "metadata_insufficient": false, - "bbox": { - "synthetic_bbox": [ - 98.5, - 672.0, - 505.5, - 712.0 - ] - }, - "mupdf_mapping": { - "n_cases": 80, - "origin_rule": "pix.x == floor(bbox_x0 * DPI/72 + MUPDF_ROUND_EPS) [A34]", - "width_rule": "pix.width == ceil(bbox_x1*s - eps) - floor(bbox_x0*s + eps) [A34]", - "n_cases_where_width_differs_from_bbox_width_times_scale": 69, - "metadata_needed_beyond_bbox_and_dpi": null, - "cases": [ - { - "dpi": 300, - "x0": 100.0, - "x1": 110.0, - "pix_x": 416, - "pix_width": 43, - "predicted_origin": 416, - "predicted_width": 43 - }, - { - "dpi": 300, - "x0": 100.0, - "x1": 110.37, - "pix_x": 416, - "pix_width": 44, - "predicted_origin": 416, - "predicted_width": 44 - }, - { - "dpi": 300, - "x0": 100.0, - "x1": 112.5, - "pix_x": 416, - "pix_width": 53, - "predicted_origin": 416, - "predicted_width": 53 - }, - { - "dpi": 300, - "x0": 100.0, - "x1": 140.0, - "pix_x": 416, - "pix_width": 168, - "predicted_origin": 416, - "predicted_width": 168 - }, - { - "dpi": 300, - "x0": 100.05, - "x1": 110.05, - "pix_x": 416, - "pix_width": 43, - "predicted_origin": 416, - "predicted_width": 43 - }, - { - "dpi": 300, - "x0": 100.05, - "x1": 110.42, - "pix_x": 416, - "pix_width": 45, - "predicted_origin": 416, - "predicted_width": 45 - }, - { - "dpi": 300, - "x0": 100.05, - "x1": 112.55, - "pix_x": 416, - "pix_width": 53, - "predicted_origin": 416, - "predicted_width": 53 - }, - { - "dpi": 300, - "x0": 100.05, - "x1": 140.05, - "pix_x": 416, - "pix_width": 168, - "predicted_origin": 416, - "predicted_width": 168 - }, - { - "dpi": 300, - "x0": 100.1, - "x1": 110.1, - "pix_x": 417, - "pix_width": 42, - "predicted_origin": 417, - "predicted_width": 42 - }, - { - "dpi": 300, - "x0": 100.1, - "x1": 110.47, - "pix_x": 417, - "pix_width": 44, - "predicted_origin": 417, - "predicted_width": 44 - }, - { - "dpi": 300, - "x0": 100.1, - "x1": 112.6, - "pix_x": 417, - "pix_width": 53, - "predicted_origin": 417, - "predicted_width": 53 - }, - { - "dpi": 300, - "x0": 100.1, - "x1": 140.1, - "pix_x": 417, - "pix_width": 167, - "predicted_origin": 417, - "predicted_width": 167 - } - ] - }, - "epsilon_boundary": { - "epsilon_px": 0.001, - "origin_boundary": [ - { - "dpi": 300, - "target_device_x": 500, - "delta_px": 0.0005, - "bbox_x0": 119.99987999999999, - "x0_times_s": 499.9995, - "mupdf_pix_x": 500, - "zero_epsilon_prediction": 499, - "epsilon_aware_prediction": 500, - "inside_epsilon_band": true - }, - { - "dpi": 300, - "target_device_x": 500, - "delta_px": 0.001, - "bbox_x0": 119.99976, - "x0_times_s": 499.999, - "mupdf_pix_x": 500, - "zero_epsilon_prediction": 499, - "epsilon_aware_prediction": 500, - "inside_epsilon_band": true - }, - { - "dpi": 300, - "target_device_x": 500, - "delta_px": 0.0011, - "bbox_x0": 119.99973599999998, - "x0_times_s": 499.9989, - "mupdf_pix_x": 499, - "zero_epsilon_prediction": 499, - "epsilon_aware_prediction": 499, - "inside_epsilon_band": false - }, - { - "dpi": 300, - "target_device_x": 500, - "delta_px": 0.002, - "bbox_x0": 119.99951999999999, - "x0_times_s": 499.998, - "mupdf_pix_x": 499, - "zero_epsilon_prediction": 499, - "epsilon_aware_prediction": 499, - "inside_epsilon_band": false - }, - { - "dpi": 330, - "target_device_x": 500, - "delta_px": 0.0005, - "bbox_x0": 109.09080000000002, - "x0_times_s": 499.9995, - "mupdf_pix_x": 500, - "zero_epsilon_prediction": 499, - "epsilon_aware_prediction": 500, - "inside_epsilon_band": true - }, - { - "dpi": 330, - "target_device_x": 500, - "delta_px": 0.001, - "bbox_x0": 109.09069090909092, - "x0_times_s": 499.999, - "mupdf_pix_x": 500, - "zero_epsilon_prediction": 499, - "epsilon_aware_prediction": 500, - "inside_epsilon_band": true - }, - { - "dpi": 330, - "target_device_x": 500, - "delta_px": 0.0011, - "bbox_x0": 109.0906690909091, - "x0_times_s": 499.99890000000005, - "mupdf_pix_x": 499, - "zero_epsilon_prediction": 499, - "epsilon_aware_prediction": 499, - "inside_epsilon_band": false - }, - { - "dpi": 330, - "target_device_x": 500, - "delta_px": 0.002, - "bbox_x0": 109.09047272727273, - "x0_times_s": 499.99799999999993, - "mupdf_pix_x": 499, - "zero_epsilon_prediction": 499, - "epsilon_aware_prediction": 499, - "inside_epsilon_band": false - } - ], - "right_edge_boundary": [ - { - "dpi": 300, - "target_device_x1": 600, - "delta_px": 0.0005, - "bbox_x1": 144.00011999999998, - "x1_times_s": 600.0005, - "mupdf_right_edge": 600, - "bbox_x0": 119.99999999999999, - "mupdf_width": 100, - "zero_epsilon_prediction": 601, - "epsilon_aware_prediction": 600, - "inside_epsilon_band": true - }, - { - "dpi": 300, - "target_device_x1": 600, - "delta_px": 0.001, - "bbox_x1": 144.00024, - "x1_times_s": 600.001, - "mupdf_right_edge": 600, - "bbox_x0": 119.99999999999999, - "mupdf_width": 100, - "zero_epsilon_prediction": 601, - "epsilon_aware_prediction": 600, - "inside_epsilon_band": true - }, - { - "dpi": 300, - "target_device_x1": 600, - "delta_px": 0.0011, - "bbox_x1": 144.000264, - "x1_times_s": 600.0011, - "mupdf_right_edge": 601, - "bbox_x0": 119.99999999999999, - "mupdf_width": 101, - "zero_epsilon_prediction": 601, - "epsilon_aware_prediction": 601, - "inside_epsilon_band": false - }, - { - "dpi": 300, - "target_device_x1": 600, - "delta_px": 0.002, - "bbox_x1": 144.00047999999998, - "x1_times_s": 600.002, - "mupdf_right_edge": 601, - "bbox_x0": 119.99999999999999, - "mupdf_width": 101, - "zero_epsilon_prediction": 601, - "epsilon_aware_prediction": 601, - "inside_epsilon_band": false - }, - { - "dpi": 330, - "target_device_x1": 600, - "delta_px": 0.0005, - "bbox_x1": 130.9092, - "x1_times_s": 600.0005, - "mupdf_right_edge": 600, - "bbox_x0": 109.0909090909091, - "mupdf_width": 100, - "zero_epsilon_prediction": 601, - "epsilon_aware_prediction": 600, - "inside_epsilon_band": true - }, - { - "dpi": 330, - "target_device_x1": 600, - "delta_px": 0.001, - "bbox_x1": 130.9093090909091, - "x1_times_s": 600.001, - "mupdf_right_edge": 600, - "bbox_x0": 109.0909090909091, - "mupdf_width": 100, - "zero_epsilon_prediction": 601, - "epsilon_aware_prediction": 600, - "inside_epsilon_band": true - }, - { - "dpi": 330, - "target_device_x1": 600, - "delta_px": 0.0011, - "bbox_x1": 130.9093309090909, - "x1_times_s": 600.0011, - "mupdf_right_edge": 601, - "bbox_x0": 109.0909090909091, - "mupdf_width": 101, - "zero_epsilon_prediction": 601, - "epsilon_aware_prediction": 601, - "inside_epsilon_band": false - }, - { - "dpi": 330, - "target_device_x1": 600, - "delta_px": 0.002, - "bbox_x1": 130.90952727272727, - "x1_times_s": 600.002, - "mupdf_right_edge": 601, - "bbox_x0": 109.0909090909091, - "mupdf_width": 101, - "zero_epsilon_prediction": 601, - "epsilon_aware_prediction": 601, - "inside_epsilon_band": false - } - ], - "zero_epsilon_origin_matches": 0 - }, - "roundtrip": { - "n_identity_crossings_under_wrong_transform": 48, - "identity_crossing_examples": [ - { - "dpi": 300, - "bbox_width": 200.0, - "phase": 0.0, - "col": 0, - "drift_pt": 0.1600000000000108, - "drift_px": 0.6666666666667117, - "separation_pt": 0.25600000000001727, - "correct_transform_ngid": 1, - "wrong_transform_ngid": 2, - "wrong_refusal": null - }, - { - "dpi": 300, - "bbox_width": 200.0, - "phase": 0.11, - "col": 833, - "drift_pt": -0.12980815347719954, - "drift_px": -0.5408673061549981, - "separation_pt": 0.20769304556351928, - "correct_transform_ngid": 1, - "wrong_transform_ngid": 2, - "wrong_refusal": null - }, - { - "dpi": 300, - "bbox_width": 200.0, - "phase": 0.23, - "col": 0, - "drift_pt": 0.15000000000000568, - "drift_px": 0.6250000000000238, - "separation_pt": 0.2400000000000091, - "correct_transform_ngid": 1, - "wrong_transform_ngid": 2, - "wrong_refusal": null - }, - { - "dpi": 300, - "bbox_width": 200.0, - "phase": 0.37, - "col": 833, - "drift_pt": -0.10980815347716089, - "drift_px": -0.4575339728215037, - "separation_pt": 0.17569304556345744, - "correct_transform_ngid": 1, - "wrong_transform_ngid": 2, - "wrong_refusal": null - }, - { - "dpi": 300, - "bbox_width": 200.0, - "phase": 0.5, - "col": 834, - "drift_pt": -0.21952095808381955, - "drift_px": -0.9146706586825815, - "separation_pt": 0.3512335329341113, - "correct_transform_ngid": 1, - "wrong_transform_ngid": 2, - "wrong_refusal": null - } - ], - "n_arithmetic_cases": 60, - "frozen_inverse_ok": 60, - "sketch_ok": 49, - "worst_sketch_error_pt": 0.3523809523809689, - "worst_sketch_error_px_300": 1.468253968254037, - "renderer_agreement": [ - { - "dpi": 300, - "bbox_x0": 100.0, - "mark_x": 117.3, - "leftmost_inked_col": 72, - "predicted_col": 72, - "delta_col": 0 - }, - { - "dpi": 300, - "bbox_x0": 100.29, - "mark_x": 117.59, - "leftmost_inked_col": 72, - "predicted_col": 72, - "delta_col": 0 - }, - { - "dpi": 300, - "bbox_x0": 100.5, - "mark_x": 117.8, - "leftmost_inked_col": 72, - "predicted_col": 72, - "delta_col": 0 - }, - { - "dpi": 300, - "bbox_x0": 100.77, - "mark_x": 118.07, - "leftmost_inked_col": 72, - "predicted_col": 72, - "delta_col": 0 - }, - { - "dpi": 330, - "bbox_x0": 100.0, - "mark_x": 117.3, - "leftmost_inked_col": 79, - "predicted_col": 79, - "delta_col": 0 - }, - { - "dpi": 330, - "bbox_x0": 100.29, - "mark_x": 117.59, - "leftmost_inked_col": 79, - "predicted_col": 79, - "delta_col": 0 - }, - { - "dpi": 330, - "bbox_x0": 100.5, - "mark_x": 117.8, - "leftmost_inked_col": 79, - "predicted_col": 79, - "delta_col": 0 - }, - { - "dpi": 330, - "bbox_x0": 100.77, - "mark_x": 118.07, - "leftmost_inked_col": 80, - "predicted_col": 80, - "delta_col": 0 - } - ], - "cases": [ - { - "dpi": 300, - "bbox_x0": 100.0, - "mark_x": 100.0, - "col": 0, - "frozen_inverse": 99.83999999999999, - "frozen_err_pt": -0.1600000000000108, - "a30_3_sketch_inverse": 100.0, - "sketch_err_pt": 0.0 - }, - { - "dpi": 300, - "bbox_x0": 100.0, - "mark_x": 117.3, - "col": 72, - "frozen_inverse": 117.11999999999999, - "frozen_err_pt": -0.18000000000000682, - "a30_3_sketch_inverse": 117.14285714285714, - "sketch_err_pt": -0.15714285714285836 - }, - { - "dpi": 300, - "bbox_x0": 100.0, - "mark_x": 139.4, - "col": 164, - "frozen_inverse": 139.2, - "frozen_err_pt": -0.20000000000001705, - "a30_3_sketch_inverse": 139.04761904761904, - "sketch_err_pt": -0.3523809523809689 - }, - { - "dpi": 300, - "bbox_x0": 100.07, - "mark_x": 100.07, - "col": 0, - "frozen_inverse": 99.83999999999999, - "frozen_err_pt": -0.23000000000000398, - "a30_3_sketch_inverse": 100.07, - "sketch_err_pt": 0.0 - }, - { - "dpi": 300, - "bbox_x0": 100.07, - "mark_x": 117.36999999999999, - "col": 73, - "frozen_inverse": 117.35999999999999, - "frozen_err_pt": -0.010000000000005116, - "a30_3_sketch_inverse": 117.45095238095237, - "sketch_err_pt": 0.08095238095238244 - }, - { - "dpi": 300, - "bbox_x0": 100.07, - "mark_x": 139.47, - "col": 165, - "frozen_inverse": 139.44, - "frozen_err_pt": -0.030000000000001137, - "a30_3_sketch_inverse": 139.35571428571427, - "sketch_err_pt": -0.11428571428572809 - }, - { - "dpi": 300, - "bbox_x0": 100.13, - "mark_x": 100.13, - "col": 0, - "frozen_inverse": 100.08, - "frozen_err_pt": -0.04999999999999716, - "a30_3_sketch_inverse": 100.13, - "sketch_err_pt": 0.0 - }, - { - "dpi": 300, - "bbox_x0": 100.13, - "mark_x": 117.42999999999999, - "col": 72, - "frozen_inverse": 117.35999999999999, - "frozen_err_pt": -0.07000000000000739, - "a30_3_sketch_inverse": 117.37550898203592, - "sketch_err_pt": -0.05449101796406808 - }, - { - "dpi": 300, - "bbox_x0": 100.13, - "mark_x": 139.53, - "col": 164, - "frozen_inverse": 139.44, - "frozen_err_pt": -0.09000000000000341, - "a30_3_sketch_inverse": 139.4114371257485, - "sketch_err_pt": -0.11856287425149503 - }, - { - "dpi": 300, - "bbox_x0": 100.29, - "mark_x": 100.29, - "col": 0, - "frozen_inverse": 100.08, - "frozen_err_pt": -0.21000000000000796, - "a30_3_sketch_inverse": 100.29, - "sketch_err_pt": 0.0 - } - ] - }, - "rotation": { - "synthetic": [ - { - "rotation": 0, - "naive_clip_has_ink": true, - "rotated_clip_has_ink": true, - "image_x_axis_is_pdf_x": true, - "image_x_axis_direction": "same" - }, - { - "rotation": 90, - "naive_clip_has_ink": false, - "rotated_clip_has_ink": true, - "image_x_axis_is_pdf_x": false, - "image_x_axis_direction": "pdf_y" - }, - { - "rotation": 180, - "naive_clip_has_ink": false, - "rotated_clip_has_ink": true, - "image_x_axis_is_pdf_x": true, - "image_x_axis_direction": "mirrored" - }, - { - "rotation": 270, - "naive_clip_has_ink": false, - "rotated_clip_has_ink": true, - "image_x_axis_is_pdf_x": false, - "image_x_axis_direction": "pdf_y" - } - ], - "ruling": "RATIFIED fail-closed NONZERO_PAGE_ROTATION -- ABORTS oracle construction and may never skip a page/region, drop a stimulus or reduce a denominator. The clip carries exactly, but at 90/270 the image x axis is the PDF y axis and at 180 it is mirrored, so start_x_px stops corresponding to a neutral glyph x0" - }, - "development": { - "n_regions": 1164, - "n_regions_rendered": 1164, - "render_sample_rule": "none -- all regions rendered", - "n_short_trailing": 282, - "n_invalid_bbox": 0, - "n_out_of_page_bbox": 0, - "n_empty_render": 0, - "n_render_determinism_failures": 0, - "n_width_derivation_failures": 0, - "page_rotation_census": { - "0": 285 - }, - "end_to_end_recovery": { - "C_300": { - "recovered": 4, - "n": 4 - }, - "C_330": { - "recovered": 4, - "n": 4 - }, - "H_300": { - "recovered": 541, - "n": 541 - }, - "H_330": { - "recovered": 541, - "n": 541 - } - }, - "end_to_end_failures": [], - "bbox_width_pt": { - "min": 52.83600000000001, - "median": 358.995, - "max": 461.7408 - }, - "bbox_height_pt": { - "min": 6.008000000000003, - "median": 192.16414999999995, - "max": 721.5965 - }, - "clipped_lines": [], - "sampled_renders": [ - { - "document": "114-hr-2029/4", - "page": 1, - "region": 0, - "bbox": [ - 127.008, - 479.2221, - 485.81, - 717.62 - ], - "w300": 1496, - "w330": 1645 - }, - { - "document": "114-hr-2029/4", - "page": 1, - "region": 1, - "bbox": [ - 126.266, - 263.8594, - 484.9894, - 472.9356 - ], - "w300": 1495, - "w330": 1645 - }, - { - "document": "114-hr-2029/4", - "page": 1, - "region": 2, - "bbox": [ - 150.0702, - 244.027, - 202.9062, - 252.889 - ], - "w300": 221, - "w330": 243 - }, - { - "document": "114-hr-2029/4", - "page": 2, - "region": 0, - "bbox": [ - 133.1679, - 548.0601, - 486.5101, - 740.212 - ], - "w300": 1474, - "w330": 1620 - }, - { - "document": "114-hr-2029/4", - "page": 2, - "region": 1, - "bbox": [ - 127.5533, - 340.0601, - 486.5283, - 534.4641 - ], - "w300": 1497, - "w330": 1646 - }, - { - "document": "114-hr-2029/4", - "page": 2, - "region": 2, - "bbox": [ - 126.4198, - 132.0601, - 485.8984, - 326.5761 - ], - "w300": 1499, - "w330": 1649 - }, - { - "document": "114-hr-2029/4", - "page": 2, - "region": 3, - "bbox": [ - 126.4201, - 36.888, - 485.3437, - 118.4641 - ], - "w300": 1497, - "w330": 1646 - }, - { - "document": "114-hr-2029/4", - "page": 3, - "region": 0, - "bbox": [ - 133.1678, - 548.0601, - 486.2344, - 740.212 - ], - "w300": 1472, - "w330": 1619 - }, - { - "document": "114-hr-2029/4", - "page": 3, - "region": 1, - "bbox": [ - 127.5533, - 340.0601, - 486.7357, - 534.5901 - ], - "w300": 1498, - "w330": 1647 - }, - { - "document": "114-hr-2029/4", - "page": 3, - "region": 2, - "bbox": [ - 126.4198, - 132.0601, - 486.5283, - 326.5901 - ], - "w300": 1502, - "w330": 1651 - }, - { - "document": "114-hr-2029/4", - "page": 3, - "region": 3, - "bbox": [ - 126.4195, - 36.888, - 486.0058, - 118.6041 - ], - "w300": 1500, - "w330": 1649 - }, - { - "document": "114-hr-2029/4", - "page": 4, - "region": 0, - "bbox": [ - 133.1681, - 548.2281, - 486.4961, - 740.24 - ], - "w300": 1474, - "w330": 1620 - } - ] - }, - "stop_conditions": [], - "tests": [ - { - "test": "1. bbox is exactly the min/max union of the committed line bboxes", - "expected": [ - 98.5, - 672.0, - 505.5, - 712.0 - ], - "observed": [ - 98.5, - 672.0, - 505.5, - 712.0 - ], - "pass": true, - "fails_when": "any other rectangle means padding, a column, or a re-derivation crept in" - }, - { - "test": "2. input line ORDER cannot change the bbox", - "expected": [ - 98.5, - 672.0, - 505.5, - 712.0 - ], - "observed": [ - 98.5, - 672.0, - 505.5, - 712.0 - ], - "pass": true, - "fails_when": "" - }, - { - "test": "3+4. H/X text and anchor content cannot change the bbox", - "expected": [ - 98.5, - 672.0, - 505.5, - 712.0 - ], - "observed": [ - 98.5, - 672.0, - 505.5, - 712.0 - ], - "pass": true, - "fails_when": "if either moved it, the renderer would be reading architecture output" - }, - { - "test": "5. no padding is added on any side", - "expected": [ - 0.0, - 0.0, - 0.0, - 0.0 - ], - "observed": [ - 0.0, - 0.0, - 0.0, - 0.0 - ], - "pass": true, - "fails_when": "a non-zero delta is a free parameter the protocol never froze" - }, - { - "test": "6. every region line bbox is contained by the region bbox", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "" - }, - { - "test": "7. a neighbouring line outside the region does not expand the crop", - "expected": [ - 98.5, - 672.0, - 505.5, - 712.0 - ], - "observed": [ - 98.5, - 672.0, - 505.5, - 712.0 - ], - "pass": true, - "fails_when": "the crop would otherwise show content the frozen region does not claim" - }, - { - "test": "8. a short trailing region follows the identical rule", - "expected": [ - 100.0, - 700.0, - 500.0, - 712.0 - ], - "observed": [ - 100.0, - 700.0, - 500.0, - 712.0 - ], - "pass": true, - "fails_when": "" - }, - { - "test": "9a. non-finite committed geometry ABORTS", - "expected": "NON_FINITE_LINE_BBOX", - "observed": "NON_FINITE_LINE_BBOX", - "pass": true, - "fails_when": "" - }, - { - "test": "9b. a missing line bbox ABORTS", - "expected": "MISSING_LINE_BBOX", - "observed": "MISSING_LINE_BBOX", - "pass": true, - "fails_when": "" - }, - { - "test": "9c. a degenerate sole line ABORTS via the stricter per-line check", - "expected": "NON_POSITIVE_LINE_BBOX", - "observed": "NON_POSITIVE_LINE_BBOX", - "pass": true, - "fails_when": "the region-union guard alone would not have caught a single degenerate line" - }, - { - "test": "9d. ...while the clean frame does NOT abort, so 9a-9c prove something", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "" - }, - { - "test": "9e. one degenerate line among eight valid ones ABORTS (union alone would pass)", - "expected": "NON_POSITIVE_LINE_BBOX", - "observed": "NON_POSITIVE_LINE_BBOX", - "pass": true, - "fails_when": "a union-only check passes here, so the bad line would be rendered and never reported" - }, - { - "test": "9f. ...and a zero-HEIGHT line aborts the same way", - "expected": "NON_POSITIVE_LINE_BBOX", - "observed": "NON_POSITIVE_LINE_BBOX", - "pass": true, - "fails_when": "" - }, - { - "test": "9g. ...while the eight valid lines alone do NOT abort", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "" - }, - { - "test": "a region bbox past the left page edge is REFUSED, not clipped", - "expected": "REGION_BBOX_OUTSIDE_PAGE", - "observed": "REGION_BBOX_OUTSIDE_PAGE", - "pass": true, - "fails_when": "clipping to the page would hand the adjudicator a stimulus that is not the region the frame committed to" - }, - { - "test": "a region bbox past the bottom page edge is REFUSED, not clipped", - "expected": "REGION_BBOX_OUTSIDE_PAGE", - "observed": "REGION_BBOX_OUTSIDE_PAGE", - "pass": true, - "fails_when": "clipping to the page would hand the adjudicator a stimulus that is not the region the frame committed to" - }, - { - "test": "a region bbox past the right page edge is REFUSED, not clipped", - "expected": "REGION_BBOX_OUTSIDE_PAGE", - "observed": "REGION_BBOX_OUTSIDE_PAGE", - "pass": true, - "fails_when": "clipping to the page would hand the adjudicator a stimulus that is not the region the frame committed to" - }, - { - "test": "a region bbox past the top page edge is REFUSED, not clipped", - "expected": "REGION_BBOX_OUTSIDE_PAGE", - "observed": "REGION_BBOX_OUTSIDE_PAGE", - "pass": true, - "fails_when": "clipping to the page would hand the adjudicator a stimulus that is not the region the frame committed to" - }, - { - "test": "...while a bbox inside the page passes, so the four refusals prove something", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "" - }, - { - "test": "image column 0 is floor(bbox_x0 * DPI/72 + eps), on EVERY fractional phase and both scales", - "expected": [ - 80, - true - ], - "observed": [ - 80, - true - ], - "pass": true, - "fails_when": "A30.3 sketched column 0 == bbox_x0; if that were right this control would fail" - }, - { - "test": "the returned width is ceil(x1*s - eps) - floor(x0*s + eps), so it needs no extra metadata", - "expected": 80, - "observed": 80, - "pass": true, - "fails_when": "" - }, - { - "test": "INSIDE the epsilon band the renderer matches the EPSILON-AWARE origin, not floor", - "expected": [ - 4, - 0 - ], - "observed": [ - 4, - 0 - ], - "pass": true, - "fails_when": "if the renderer agreed with plain floor here, MUPDF_ROUND_EPS would be unfounded and device_origin_px must be reverted" - }, - { - "test": "...and the two predictions genuinely DIFFER there, so the case is discriminating", - "expected": 4, - "observed": 4, - "pass": true, - "fails_when": "" - }, - { - "test": "OUTSIDE the band both predictions agree and match the renderer (boundary bracketed)", - "expected": [ - 4, - 4 - ], - "observed": [ - 4, - 4 - ], - "pass": true, - "fails_when": "without the outside bracket this would show one favourable coordinate, not a threshold" - }, - { - "test": "RIGHT edge: inside the band the renderer matches the epsilon-aware ceil, not ceil", - "expected": [ - 4, - 0 - ], - "observed": [ - 4, - 0 - ], - "pass": true, - "fails_when": "the code claims the epsilon applies symmetrically; this is that claim's evidence" - }, - { - "test": "RIGHT edge: outside the band both agree, bracketing the threshold", - "expected": [ - 4, - 4 - ], - "observed": [ - 4, - 4 - ], - "pass": true, - "fails_when": "" - }, - { - "test": "with MUPDF_ROUND_EPS = 0 the origin boundary control FAILS on every band case", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "if it still matched, the constant would not be load-bearing and should be dropped" - }, - { - "test": "with MUPDF_ROUND_EPS = 0 the WIDTH derivation also stops matching in the band", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "this is the symmetry claim's negative control, not an argument from the code comment" - }, - { - "test": "...and the epsilon is restored afterwards, so the injection did not leak", - "expected": 0.001, - "observed": 0.001, - "pass": true, - "fails_when": "" - }, - { - "test": "12. the frozen inverse recovers the source position at every fractional origin", - "expected": [ - 60, - true - ], - "observed": [ - 60, - true - ], - "pass": true, - "fails_when": "a failure means the encoded transform is not the renderer's actual mapping" - }, - { - "test": "13. the deliberately wrong (linear-across-bbox) convention does NOT always recover it", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "if the wrong convention also passed, control 12 would prove nothing" - }, - { - "test": "the RENDERER's leftmost inked column matches the forward map within 1 px", - "expected": [ - 8, - true - ], - "observed": [ - 8, - true - ], - "pass": true, - "fails_when": "a larger gap would mean the device origin rule is wrong, not merely antialiased" - }, - { - "test": "the wrong transform can cross a nearest-x boundary and change the IDENTITY", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "without this the wrong-transform control only shows arithmetic drift, never that the resolver would return a different ngid" - }, - { - "test": "a clip in UNROTATED pdf space renders no ink on a rotated page", - "expected": [ - true, - false, - false, - false - ], - "observed": [ - true, - false, - false, - false - ], - "pass": true, - "fails_when": "if it did render, rotation would be a non-issue and the refusal unnecessary" - }, - { - "test": "...the rotation matrix carries the clip exactly, at every rotation", - "expected": [ - true, - true, - true, - true - ], - "observed": [ - true, - true, - true, - true - ], - "pass": true, - "fails_when": "" - }, - { - "test": "14. a non-zero rotation is REFUSED, fail-closed", - "expected": "NONZERO_PAGE_ROTATION", - "observed": "NONZERO_PAGE_ROTATION", - "pass": true, - "fails_when": "" - }, - { - "test": "14b. ...and rotation 0 is not refused", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "" - }, - { - "test": "end-to-end: every H/C start recovers its own ngid through the A33 transform + A30 resolver", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a failure means the frozen transform and the frozen resolver do not compose" - }, - { - "test": "...and the end-to-end population is non-empty at both scales, so it is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "" - }, - { - "test": "10. every rendered region matches the frozen width derivation at BOTH scales", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "" - }, - { - "test": "11. re-rendering the same bbox/renderer/DPI reproduces the PNG hash", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "" - }, - { - "test": "no committed neutral line is clipped by the zero-padding union", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "if the union clipped committed content, A33.1 would need review -- padding is NOT tuned" - }, - { - "test": "no DEVELOPMENT region produced an empty render", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "" - }, - { - "test": "every region enumerated was also rendered, so render denominators are not a sample", - "expected": 1164, - "observed": 1164, - "pass": true, - "fails_when": "" - }, - { - "test": "every DEVELOPMENT page consumed has rotation 0", - "expected": [ - 0 - ], - "observed": [ - 0 - ], - "pass": true, - "fails_when": "" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x21_build_oracle.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x21_build_oracle.json deleted file mode 100644 index 000e39f9..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x21_build_oracle.json +++ /dev/null @@ -1,2028 +0,0 @@ -{ - "population": "SYNTHETIC + DEVELOPMENT -- no holdout opened, nothing adjudicated, nothing scored", - "contract": "A35 (adjudicator prompt + build_oracle) as amended by A36 (C/D overlap semantics + M5 coarsening), implementing A19-A34 and HARNESS-PLAN 3-4", - "renderer": "MuPDF (pymupdf)", - "renderer_version": "('1.28.2', '1.28.2', None)", - "artifacts_created": "NONE of frames.json, oracle_key.json, oracle_adjudicated.json, metrics.json, scores.json, EXECUTION-START.json", - "a35_5_stop_resolved_by": "A36 -- REGION_IN_BOTH_FRAMES is no longer raised; the overlap is BUILT as one stimulus carrying both memberships and both routes", - "prompt": { - "prompt_sha256": "2067821ddb0464059bdb9d0b7c0555458446b77c0af94029afbbfe620384a406", - "prompt_chars": 5027, - "forbidden_patterns_checked": [ - "amount_attribution", - "architecture", - "arm_letter_H", - "arm_letter_X", - "control_status", - "deltatrack", - "extended_glyph", - "frame", - "hybrid", - "pdfium", - "renderer_scale", - "repeat_status", - "stratum" - ], - "required_instructions_checked": [ - "a33_5_ignore_marks", - "a33_5_not_bounding_box", - "a33_5_visible_ink", - "exact_printed_text", - "identity_annotation_only", - "immediate_parent", - "independent_adjudication", - "role_codebook", - "start_physical_line", - "start_x_px" - ], - "injections_caught": { - "architecture name": true, - "arm letter": true, - "frame label": true, - "stratum": true, - "control status": true, - "repeat status": true, - "render scale": true, - "engine name": true - } - }, - "blinding": { - "n_stimuli": 22, - "n_r1_repeats": 2, - "blind_id_scheme_swap_changed_selection": false, - "blind_id_scheme_swap_changed_order": false - }, - "render": { - "synthetic_bbox": [ - 100.0, - 529.0, - 300.0, - 681.0 - ], - "expected_width_300": 834, - "expected_width_330": 917, - "n_r1_pairs_checked": 2 - }, - "fail_closed": { - "n_clean_stimuli": 22, - "rotation_refusal": "NONZERO_PAGE_ROTATION", - "rotation_build_returned": null, - "refusal_classes": [ - "MISSING_LINE_BBOX", - "NON_FINITE_LINE_BBOX", - "NON_POSITIVE_LINE_BBOX", - "REGION_BBOX_OUTSIDE_PAGE", - "NONZERO_PAGE_ROTATION" - ] - }, - "leakage_and_join": { - "n_stimuli": 22, - "blind_allowed_keys": [ - "id", - "image", - "question" - ], - "injected_leaks_caught": { - "document_sha256": true, - "canonical_identity": true, - "png_sha256": true, - "structural_frame_and_stratum": true, - "forbidden_text": true - }, - "shuffled_key_defects": 22 - }, - "start_line_bijection": { - "region_lines": 8, - "bijection": [ - [ - 1, - 0 - ], - [ - 1, - 1 - ], - [ - 1, - 2 - ], - [ - 1, - 3 - ], - [ - 1, - 4 - ], - [ - 1, - 5 - ], - [ - 1, - 6 - ], - [ - 1, - 7 - ] - ] - }, - "overlap_semantics": { - "frame_counts": { - "c_frame": 19, - "d_frame": 2, - "c_and_d_overlap": 1, - "union_reported_for_information_only": 20, - "c_audit_selected": 19, - "human_tasks": 21, - "ai_route": 21, - "human_route": 21 - }, - "synthetic_overlap_regions": 1, - "audit_size_used": 5, - "n_audit_overlap_items": 2, - "n_non_audit_overlap_items": 8, - "n_r1_repeats": 2 - }, - "occurrence_join": { - "blind_id": "cc4f046a87fd2a07", - "adjudicated_heading": { - "text": "SYNTHETIC ACCOUNT", - "start_physical_line": 3, - "start_x_px": 0 - }, - "resolved": { - "matched": true, - "neutral_line_key": [ - 1, - 2 - ], - "target_pdf_x": 99.83999999999999, - "start_ngid": 20, - "occurrence_key": [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ] - }, - "emitted_occurrence_key": [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ], - "boundary_crossed_ngid": 21, - "tie_refused": true - }, - "adjudicated_encoding": { - "schema": "oracle_adjudicated/1", - "heading_fields": [ - "text", - "role", - "parent", - "start_physical_line", - "start_x_px" - ] - }, - "m5_coarsening": { - "production_anchor_kinds": [ - "account", - "agency", - "grouping", - "major", - "preamble", - "section", - "subsection", - "title" - ], - "oracle_role_to_m5": { - "account": "LEAF", - "section": "LEAF", - "agency": "CONTAINER", - "grouping": "CONTAINER", - "title": "CONTAINER", - "division": "CONTAINER", - "other": "UNSCORABLE" - }, - "emitted_kind_to_m5": { - "account": "LEAF", - "section": "LEAF", - "major": "CONTAINER", - "agency": "CONTAINER", - "grouping": "CONTAINER", - "title": "CONTAINER", - "subsection": "UNSCORABLE", - "preamble": "UNSCORABLE" - }, - "m5_status": "CORROBORATION ONLY -- may never affect the architecture decision" - }, - "development": { - "documents": [ - { - "document": "114-hr-2029/4", - "pages_consumed": 20, - "regions": 78, - "c_frame": 8, - "d_frame": 60, - "c_and_d": 8 - }, - { - "document": "118-hr-8752/1", - "pages_consumed": 20, - "regions": 79, - "c_frame": 8, - "d_frame": 56, - "c_and_d": 7 - }, - { - "document": "119-hr-1/1", - "pages_consumed": 20, - "regions": 107, - "c_frame": 8, - "d_frame": 54, - "c_and_d": 3 - } - ], - "page_limit": 20, - "note": "PAGE_LIMIT is a machinery demonstration window, not a census", - "n_stimuli": 193, - "n_r1_repeats": 17, - "n_images": 193, - "image_width_px_min": 934, - "image_width_px_max": 2110, - "frame_counts": { - "c_frame": 24, - "d_frame": 170, - "c_and_d_overlap": 18, - "union_reported_for_information_only": 176, - "c_audit_selected": 24, - "human_tasks": 192, - "ai_route": 27, - "human_route": 192 - }, - "c_and_d_overlap_regions": 18, - "overlap_instances": [ - { - "document": "114-hr-2029/4", - "page_number": 1, - "region_ordinal": 0 - }, - { - "document": "114-hr-2029/4", - "page_number": 1, - "region_ordinal": 1 - }, - { - "document": "114-hr-2029/4", - "page_number": 10, - "region_ordinal": 1 - }, - { - "document": "114-hr-2029/4", - "page_number": 12, - "region_ordinal": 1 - }, - { - "document": "114-hr-2029/4", - "page_number": 14, - "region_ordinal": 2 - }, - { - "document": "114-hr-2029/4", - "page_number": 18, - "region_ordinal": 0 - }, - { - "document": "114-hr-2029/4", - "page_number": 19, - "region_ordinal": 0 - }, - { - "document": "114-hr-2029/4", - "page_number": 20, - "region_ordinal": 0 - }, - { - "document": "118-hr-8752/1", - "page_number": 4, - "region_ordinal": 1 - }, - { - "document": "118-hr-8752/1", - "page_number": 10, - "region_ordinal": 2 - }, - { - "document": "118-hr-8752/1", - "page_number": 11, - "region_ordinal": 3 - }, - { - "document": "118-hr-8752/1", - "page_number": 14, - "region_ordinal": 3 - }, - { - "document": "118-hr-8752/1", - "page_number": 18, - "region_ordinal": 0 - }, - { - "document": "118-hr-8752/1", - "page_number": 19, - "region_ordinal": 0 - }, - { - "document": "118-hr-8752/1", - "page_number": 20, - "region_ordinal": 0 - }, - { - "document": "119-hr-1/1", - "page_number": 3, - "region_ordinal": 0 - }, - { - "document": "119-hr-1/1", - "page_number": 11, - "region_ordinal": 5 - }, - { - "document": "119-hr-1/1", - "page_number": 15, - "region_ordinal": 3 - } - ] - }, - "stop_conditions": [], - "tests": [ - { - "test": "18. the prompt asks NO M6 / amount-attribution question", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "an amount-to-account or total-the-figures question appears; M6 is deferred by A20" - }, - { - "test": "19. the prompt carries the A33.5 visible-character instruction", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the A33.5 wording is missing, so start_x_px would be collected under a different definition and nothing downstream could tell" - }, - { - "test": "the prompt forbids a text-box / bounding-box edge (A33.5)", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "an adjudicator could mark a bounding-box edge instead of the character's own ink" - }, - { - "test": "the prompt says start_x_px is an identity annotation only", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "start_x_px could be read as evidence about text, parent or role" - }, - { - "test": "the prompt requires text, parent and role to be judged independently", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "one field could be adjusted to agree with another" - }, - { - "test": "the prompt leaks no forbidden architecture / frame / stratum / control token", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "any of H/X, hybrid, extended glyph, architecture, C/D-frame, stratum, control status, repeat status or DPI appears in the adjudicator's instructions" - }, - { - "test": "the prompt carries every required instruction", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a required question or instruction is absent" - }, - { - "test": "the committed prompt LOADS -- the gate accepts it", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the committed prompt is refused by its own gate" - }, - { - "test": "10a. NEGATIVE -- every injected forbidden disclosure is CAUGHT by the prompt gate", - "expected": { - "architecture name": true, - "arm letter": true, - "frame label": true, - "stratum": true, - "control status": true, - "repeat status": true, - "render scale": true, - "engine name": true - }, - "observed": { - "architecture name": true, - "arm letter": true, - "frame label": true, - "stratum": true, - "control status": true, - "repeat status": true, - "render scale": true, - "engine name": true - }, - "pass": true, - "fails_when": "an injected architecture / frame / stratum / control / scale disclosure passes the gate, which would mean the gate cannot see the leak it exists to stop" - }, - { - "test": "10b. NEGATIVE -- an injected M6 question is CAUGHT", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "an amount-attribution question could be added without the gate firing" - }, - { - "test": "10c. NEGATIVE -- removing the A33.5 wording makes the completeness gate FAIL", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the required-instruction gate cannot detect the A33.5 instruction going missing" - }, - { - "test": "...and a prompt carrying an injected leak is REFUSED, not merely reported", - "expected": "PROMPT_LEAKS", - "observed": "PROMPT_LEAKS", - "pass": true, - "fails_when": "load_prompt returns a leaking prompt instead of refusing it" - }, - { - "test": "6. changing the blind-ID scheme changes NO selected stimulus", - "expected": [ - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",2,0]", - "[\"region\",\"synthsha0123456789\",3,0]", - "[\"region\",\"synthsha0123456789\",4,0]", - "[\"region\",\"synthsha0123456789\",5,0]", - "[\"region\",\"synthsha0123456789\",6,0]", - "[\"region\",\"synthsha0123456789\",7,0]", - "[\"region\",\"synthsha0123456789\",8,0]", - "[\"region\",\"synthsha0123456789\",9,0]", - "[\"region\",\"synthsha0123456789\",10,0]", - "[\"region\",\"synthsha0123456789\",11,0]", - "[\"region\",\"synthsha0123456789\",12,0]", - "[\"region\",\"synthsha0123456789\",13,0]", - "[\"region\",\"synthsha0123456789\",14,0]", - "[\"region\",\"synthsha0123456789\",15,0]", - "[\"region\",\"synthsha0123456789\",16,0]", - "[\"region\",\"synthsha0123456789\",17,0]", - "[\"region\",\"synthsha0123456789\",18,0]", - "[\"region\",\"synthsha0123456789\",19,0]", - "[\"region\",\"synthsha0123456789\",20,0]", - "[\"r1-repeat\",[\"region\",\"synthsha0123456789\",3,0]]", - "[\"r1-repeat\",[\"region\",\"synthsha0123456789\",7,0]]" - ], - "observed": [ - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",2,0]", - "[\"region\",\"synthsha0123456789\",3,0]", - "[\"region\",\"synthsha0123456789\",4,0]", - "[\"region\",\"synthsha0123456789\",5,0]", - "[\"region\",\"synthsha0123456789\",6,0]", - "[\"region\",\"synthsha0123456789\",7,0]", - "[\"region\",\"synthsha0123456789\",8,0]", - "[\"region\",\"synthsha0123456789\",9,0]", - "[\"region\",\"synthsha0123456789\",10,0]", - "[\"region\",\"synthsha0123456789\",11,0]", - "[\"region\",\"synthsha0123456789\",12,0]", - "[\"region\",\"synthsha0123456789\",13,0]", - "[\"region\",\"synthsha0123456789\",14,0]", - "[\"region\",\"synthsha0123456789\",15,0]", - "[\"region\",\"synthsha0123456789\",16,0]", - "[\"region\",\"synthsha0123456789\",17,0]", - "[\"region\",\"synthsha0123456789\",18,0]", - "[\"region\",\"synthsha0123456789\",19,0]", - "[\"region\",\"synthsha0123456789\",20,0]", - "[\"r1-repeat\",[\"region\",\"synthsha0123456789\",3,0]]", - "[\"r1-repeat\",[\"region\",\"synthsha0123456789\",7,0]]" - ], - "pass": true, - "fails_when": "sampling consumes a blind id, so the alias scheme could move the sample (A28.3)" - }, - { - "test": "7. ...and changes NO presentation rank", - "expected": [ - "[\"region\",\"synthsha0123456789\",9,0]", - "[\"region\",\"synthsha0123456789\",19,0]", - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",5,0]", - "[\"region\",\"synthsha0123456789\",16,0]", - "[\"region\",\"synthsha0123456789\",17,0]", - "[\"region\",\"synthsha0123456789\",6,0]", - "[\"region\",\"synthsha0123456789\",12,0]", - "[\"region\",\"synthsha0123456789\",18,0]", - "[\"region\",\"synthsha0123456789\",2,0]", - "[\"r1-repeat\",[\"region\",\"synthsha0123456789\",7,0]]", - "[\"region\",\"synthsha0123456789\",14,0]", - "[\"region\",\"synthsha0123456789\",3,0]", - "[\"r1-repeat\",[\"region\",\"synthsha0123456789\",3,0]]", - "[\"region\",\"synthsha0123456789\",8,0]", - "[\"region\",\"synthsha0123456789\",4,0]", - "[\"region\",\"synthsha0123456789\",13,0]", - "[\"region\",\"synthsha0123456789\",7,0]", - "[\"region\",\"synthsha0123456789\",11,0]", - "[\"region\",\"synthsha0123456789\",10,0]", - "[\"region\",\"synthsha0123456789\",20,0]", - "[\"region\",\"synthsha0123456789\",15,0]" - ], - "observed": [ - "[\"region\",\"synthsha0123456789\",9,0]", - "[\"region\",\"synthsha0123456789\",19,0]", - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",5,0]", - "[\"region\",\"synthsha0123456789\",16,0]", - "[\"region\",\"synthsha0123456789\",17,0]", - "[\"region\",\"synthsha0123456789\",6,0]", - "[\"region\",\"synthsha0123456789\",12,0]", - "[\"region\",\"synthsha0123456789\",18,0]", - "[\"region\",\"synthsha0123456789\",2,0]", - "[\"r1-repeat\",[\"region\",\"synthsha0123456789\",7,0]]", - "[\"region\",\"synthsha0123456789\",14,0]", - "[\"region\",\"synthsha0123456789\",3,0]", - "[\"r1-repeat\",[\"region\",\"synthsha0123456789\",3,0]]", - "[\"region\",\"synthsha0123456789\",8,0]", - "[\"region\",\"synthsha0123456789\",4,0]", - "[\"region\",\"synthsha0123456789\",13,0]", - "[\"region\",\"synthsha0123456789\",7,0]", - "[\"region\",\"synthsha0123456789\",11,0]", - "[\"region\",\"synthsha0123456789\",10,0]", - "[\"region\",\"synthsha0123456789\",20,0]", - "[\"region\",\"synthsha0123456789\",15,0]" - ], - "pass": true, - "fails_when": "blind-order ranks blind ids rather than canonical final identities" - }, - { - "test": "...and the alias itself DID change, so the control is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the substituted scheme returns the same alias, proving nothing" - }, - { - "test": "8. NEGATIVE -- an injected blind-ID collision ABORTS the build", - "expected": "BlindIdCollision", - "observed": "BlindIdCollision", - "pass": true, - "fails_when": "the build completes with a collision, silently overwriting or merging two stimuli (I14)" - }, - { - "test": "20. realized blind-ID uniqueness holds over the COMPLETE set, repeats included", - "expected": 22, - "observed": 22, - "pass": true, - "fails_when": "two realized stimuli share an adjudicator-facing id" - }, - { - "test": "...and the uniqueness assertion is made over instances that INCLUDE R1 repeats", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no repeat exists in the realized set, so control 20 never exercised the repeat case" - }, - { - "test": "1. same bbox + renderer + DPI re-renders to an IDENTICAL PNG hash", - "expected": "910e42afcff3d204f4d112eb7dad76ee9f772f381691d6dd25101161ca21e753", - "observed": "910e42afcff3d204f4d112eb7dad76ee9f772f381691d6dd25101161ca21e753", - "pass": true, - "fails_when": "the renderer is not deterministic, so a stored png_sha256 could not detect a re-render" - }, - { - "test": "4. the renderer consumes PDF geometry only -- bytes match a frame-free direct render", - "expected": "910e42afcff3d204f4d112eb7dad76ee9f772f381691d6dd25101161ca21e753", - "observed": "910e42afcff3d204f4d112eb7dad76ee9f772f381691d6dd25101161ca21e753", - "pass": true, - "fails_when": "the frame contributes something to the pixels, i.e. an arm's output could reach the image (I6)" - }, - { - "test": "17. the PNG hash CHANGES when the rendered stimulus really changes", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "two genuinely different renders hash the same, so the hash cannot detect a substitution" - }, - { - "test": "5. changing H/X text cannot change ANY rendered PNG", - "expected": [ - "0393c3c15058a7731d0891e594c166be0afc87f262c87f33c88500df95cd385f", - "0614bcc4e3217d3b7bfaa2d41ec4fdfa2412194c4672a525857067042ebd15c3", - "0d36093aa058c89a6f51cf2c971c2ae23cb6a768b88bda2d52939e9221bf3ce5", - "0e41c40661c94c29ff2d116ad624822b891fdf3112599c2034faa6848b71c96c", - "2195c25922784fbea4f897eaa5a9497161a98ea87364045447ce4f4b11d46b38", - "2b3288194f435a2ef8b6d1578fe41e9200d6124a28ea2ca730148653d08dc4e8", - "3682953cc91d9cdc67ef110947315ebb0ea20af0904ea701f82cfba21831ba68", - "4596e5e78e706e9c0bb94e21fee823dac1d0a2e68e7f0dad36a021afb51b1155", - "523e76b2dcc43b0339269ad853ddf7cb233035b11b6f2972c16e84c01d60210f", - "55b5c24f8bbde123a7761c23c69252b4a56d0a8af0f2f59d819851b33a0c4d0c", - "564afbde1c31bcf4a853c026f30d0043c8cfd7395f9dff6af4c6c14863974ee0", - "5787d162f1cc2b05461b08f7788961bc722a825750b37310543670eb121fcba7", - "59993342398a90ed8c6928eecc169a6e31fd055a4d12663fb7ae2ffcfc8521ed", - "64d9ab7852adf4bed4c745e97bdfa41d065587f80a38f80716af542d2369073e", - "77fb29c43c1a62ff27bc94d7a0eb61d6303bf707510ec65c350f8cec880d05dc", - "83d9e4e005f8dc78f00a7c84dfaa05870f67cfc46d0371de8114194969265fd3", - "901a178efbf6b463699e4dd5e5880771353407eeb09fb6545f8c268773c3b7f1", - "910e42afcff3d204f4d112eb7dad76ee9f772f381691d6dd25101161ca21e753", - "b465399f344653daf00f130dfd8c5c5739c019c5366b6c454161344d53f43af2", - "cb65c619132bd2e46033116332991afd2634b1f9f4832648ab5214005779904f", - "dc8501c025ea68920ff6567fcec53decc88e548bc1644ba4702d3a71af625660", - "f8158e1d2e7ba8ea9bc3e9cefcada264295baedd2c2e863284f7148f0c2e5d90" - ], - "observed": [ - "0393c3c15058a7731d0891e594c166be0afc87f262c87f33c88500df95cd385f", - "0614bcc4e3217d3b7bfaa2d41ec4fdfa2412194c4672a525857067042ebd15c3", - "0d36093aa058c89a6f51cf2c971c2ae23cb6a768b88bda2d52939e9221bf3ce5", - "0e41c40661c94c29ff2d116ad624822b891fdf3112599c2034faa6848b71c96c", - "2195c25922784fbea4f897eaa5a9497161a98ea87364045447ce4f4b11d46b38", - "2b3288194f435a2ef8b6d1578fe41e9200d6124a28ea2ca730148653d08dc4e8", - "3682953cc91d9cdc67ef110947315ebb0ea20af0904ea701f82cfba21831ba68", - "4596e5e78e706e9c0bb94e21fee823dac1d0a2e68e7f0dad36a021afb51b1155", - "523e76b2dcc43b0339269ad853ddf7cb233035b11b6f2972c16e84c01d60210f", - "55b5c24f8bbde123a7761c23c69252b4a56d0a8af0f2f59d819851b33a0c4d0c", - "564afbde1c31bcf4a853c026f30d0043c8cfd7395f9dff6af4c6c14863974ee0", - "5787d162f1cc2b05461b08f7788961bc722a825750b37310543670eb121fcba7", - "59993342398a90ed8c6928eecc169a6e31fd055a4d12663fb7ae2ffcfc8521ed", - "64d9ab7852adf4bed4c745e97bdfa41d065587f80a38f80716af542d2369073e", - "77fb29c43c1a62ff27bc94d7a0eb61d6303bf707510ec65c350f8cec880d05dc", - "83d9e4e005f8dc78f00a7c84dfaa05870f67cfc46d0371de8114194969265fd3", - "901a178efbf6b463699e4dd5e5880771353407eeb09fb6545f8c268773c3b7f1", - "910e42afcff3d204f4d112eb7dad76ee9f772f381691d6dd25101161ca21e753", - "b465399f344653daf00f130dfd8c5c5739c019c5366b6c454161344d53f43af2", - "cb65c619132bd2e46033116332991afd2634b1f9f4832648ab5214005779904f", - "dc8501c025ea68920ff6567fcec53decc88e548bc1644ba4702d3a71af625660", - "f8158e1d2e7ba8ea9bc3e9cefcada264295baedd2c2e863284f7148f0c2e5d90" - ], - "pass": true, - "fails_when": "an arm's text reaches the renderer or the crop, which would let the thing under test choose its own stimulus" - }, - { - "test": "2. every R1 repeat renders the EXACT same PDF bbox as its primary", - "expected": [ - 2, - true - ], - "observed": [ - 2, - true - ], - "pass": true, - "fails_when": "a repeat is cropped differently, so a text disagreement could be a crop artifact (A28.4)" - }, - { - "test": "3. ...and differs from it ONLY by raster scale (300 -> 330)", - "expected": [ - 2, - true - ], - "observed": [ - 2, - true - ], - "pass": true, - "fails_when": "a repeat differs in source region, page or document, or is not 330 DPI" - }, - { - "test": "13. the crop equals the committed-frame minimal union EXACTLY, zero padding", - "expected": [ - 100.0, - 529.0, - 300.0, - 681.0 - ], - "observed": [ - 100.0, - 529.0, - 300.0, - 681.0 - ], - "pass": true, - "fails_when": "the builder padded, expanded to column or page, re-clustered from the PDF, or repaired the bbox -- each would show the adjudicator something other than the committed region" - }, - { - "test": "14. NEGATIVE -- a region bbox extending past the page ABORTS", - "expected": "REGION_BBOX_OUTSIDE_PAGE", - "observed": "REGION_BBOX_OUTSIDE_PAGE", - "pass": true, - "fails_when": "an out-of-page bbox is clipped, intersected or padded instead of refused (A33.1)" - }, - { - "test": "15. NEGATIVE -- a non-positive committed line bbox ABORTS", - "expected": "NON_POSITIVE_LINE_BBOX", - "observed": "NON_POSITIVE_LINE_BBOX", - "pass": true, - "fails_when": "a degenerate line passes because its siblings make the union positive, so defective committed geometry would render and never be seen" - }, - { - "test": "NEGATIVE -- a missing committed line bbox ABORTS", - "expected": "MISSING_LINE_BBOX", - "observed": "MISSING_LINE_BBOX", - "pass": true, - "fails_when": "a line with no geometry is skipped, silently shrinking the crop" - }, - { - "test": "NEGATIVE -- a non-finite committed line bbox ABORTS", - "expected": "NON_FINITE_LINE_BBOX", - "observed": "NON_FINITE_LINE_BBOX", - "pass": true, - "fails_when": "a NaN coordinate propagates into a clip rectangle" - }, - { - "test": "16. NEGATIVE -- a non-zero page rotation ABORTS oracle construction", - "expected": "NONZERO_PAGE_ROTATION", - "observed": "NONZERO_PAGE_ROTATION", - "pass": true, - "fails_when": "a rotated page renders anyway, though start_x_px would no longer be a PDF x offset (A33.4)" - }, - { - "test": "16b. ...and the rotation refusal CANNOT become a skipped stimulus", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "the build returns a result with the rotated page's regions dropped, turning an unrepresentable condition into a quietly smaller study with no visible loss" - }, - { - "test": "...and a clean build of the same shape DOES produce stimuli, so 16b is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the clean case produces nothing either, so 'None' proved nothing about rotation" - }, - { - "test": "NEGATIVE -- a confirmatory holdout member is REFUSED while the boundary is not VALID", - "expected": "HOLDOUT_BEFORE_EXECUTION_BOUNDARY", - "observed": "HOLDOUT_BEFORE_EXECUTION_BOUNDARY", - "pass": true, - "fails_when": "a holdout document can be opened before execution is authorised" - }, - { - "test": "the boundary ACCESS MATRIX permits a holdout only at VALID", - "expected": { - "ABSENT": "HOLDOUT_BEFORE_EXECUTION_BOUNDARY", - "UNCOMMITTED": "HOLDOUT_BEFORE_EXECUTION_BOUNDARY", - "MUTATED": "HOLDOUT_BEFORE_EXECUTION_BOUNDARY", - "VALID": null - }, - "observed": { - "ABSENT": "HOLDOUT_BEFORE_EXECUTION_BOUNDARY", - "UNCOMMITTED": "HOLDOUT_BEFORE_EXECUTION_BOUNDARY", - "MUTATED": "HOLDOUT_BEFORE_EXECUTION_BOUNDARY", - "VALID": null - }, - "pass": true, - "fails_when": "an ABSENT, UNCOMMITTED or MUTATED marker unlocks the holdout -- a stray or edited EXECUTION-START.json would then authorize the confirmatory run" - }, - { - "test": "...and every CANONICAL confirmatory write obeys the same VALID-only rule", - "expected": { - "ABSENT": "CONFIRMATORY_WRITE_BEFORE_EXECUTION", - "UNCOMMITTED": "CONFIRMATORY_WRITE_BEFORE_EXECUTION", - "MUTATED": "CONFIRMATORY_WRITE_BEFORE_EXECUTION", - "VALID": null - }, - "observed": { - "ABSENT": "CONFIRMATORY_WRITE_BEFORE_EXECUTION", - "UNCOMMITTED": "CONFIRMATORY_WRITE_BEFORE_EXECUTION", - "MUTATED": "CONFIRMATORY_WRITE_BEFORE_EXECUTION", - "VALID": null - }, - "pass": true, - "fails_when": "an artifact can be created under a weaker condition than the material it describes" - }, - { - "test": "the S1 and cross-engine canonical paths are inside the same guarded set", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "an S1 or cross-engine confirmatory artifact could be written with no boundary at all" - }, - { - "test": "...and a scratch path is NOT refused, so the write guard is not blanket-refusing", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "the guard refuses everything, which would make the controls above meaningless" - }, - { - "test": "no real execution marker was created by any of this", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "the boundary controls created the very marker they exist to test" - }, - { - "test": "9. the adjudicator-facing artifact carries ONLY id, image and question", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a field beyond the blinding allowlist reaches the adjudicator" - }, - { - "test": "9b. ...and no private VALUE appears anywhere in its serialization", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "a document sha, canonical identity or png hash is smuggled inside an allowed field" - }, - { - "test": "9c. ...and no forbidden architecture / frame / stratum token appears in it", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the blind file names an architecture, frame, stratum, control or repeat" - }, - { - "test": "10. NEGATIVE -- every injected leak makes the leakage gate FAIL", - "expected": { - "document_sha256": true, - "canonical_identity": true, - "png_sha256": true, - "structural_frame_and_stratum": true, - "forbidden_text": true - }, - "observed": { - "document_sha256": true, - "canonical_identity": true, - "png_sha256": true, - "structural_frame_and_stratum": true, - "forbidden_text": true - }, - "pass": true, - "fails_when": "an injected private value, an extra structural field, or a forbidden token passes the gate -- which would mean a green leakage report proves nothing" - }, - { - "test": "...and write_artifacts REFUSES a leaking blind file rather than writing it", - "expected": "BLIND_ARTIFACT_LEAKS", - "observed": "BLIND_ARTIFACT_LEAKS", - "pass": true, - "fails_when": "a leaking artifact is written and the leak survives to the adjudicator" - }, - { - "test": "11. the private key carries every field the downstream join needs", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a join field is missing, so an adjudication could not be bound back to its region" - }, - { - "test": "12. NEGATIVE -- a misassociated private key BREAKS the join, proving it load-bearing", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "adjudications still align after the key is shuffled, which would mean the join is fake and the key decorative" - }, - { - "test": "...and the shuffle was real: at least two distinct stimuli existed to permute", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "fewer than two stimuli, so the shuffle was a no-op and control 12 was vacuous" - }, - { - "test": "the bijection covers EXACTLY the region's committed neutral lines", - "expected": [ - [ - 1, - 0 - ], - [ - 1, - 1 - ], - [ - 1, - 2 - ], - [ - 1, - 3 - ], - [ - 1, - 4 - ], - [ - 1, - 5 - ], - [ - 1, - 6 - ], - [ - 1, - 7 - ] - ], - "observed": [ - [ - 1, - 0 - ], - [ - 1, - 1 - ], - [ - 1, - 2 - ], - [ - 1, - 3 - ], - [ - 1, - 4 - ], - [ - 1, - 5 - ], - [ - 1, - 6 - ], - [ - 1, - 7 - ] - ], - "pass": true, - "fails_when": "a printed line maps to a line outside the region, or a region line is unreachable" - }, - { - "test": "...and it is a bijection, not a multimap", - "expected": 8, - "observed": 8, - "pass": true, - "fails_when": "two printed-line indices resolve to the same committed line" - }, - { - "test": "printed order is committed-ordinal order, top to bottom", - "expected": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7 - ], - "observed": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7 - ], - "pass": true, - "fails_when": "the bijection is not sorted by ordinal, so line 1 is not the topmost printed line" - }, - { - "test": "start_physical_line 1 resolves to the topmost committed line", - "expected": [ - 1, - 0 - ], - "observed": [ - 1, - 0 - ], - "pass": true, - "fails_when": "the 1-based index is off by one" - }, - { - "test": "NEGATIVE -- an out-of-range printed-line index REFUSES rather than guessing", - "expected": "START_LINE_OUT_OF_RANGE", - "observed": "START_LINE_OUT_OF_RANGE", - "pass": true, - "fails_when": "an impossible line index resolves to some line anyway, which would hand the nearest-x resolver a confident and wrong start_ngid" - }, - { - "test": "NEGATIVE -- a zero or negative index REFUSES", - "expected": "START_LINE_OUT_OF_RANGE", - "observed": "START_LINE_OUT_OF_RANGE", - "pass": true, - "fails_when": "0 is accepted, silently reading as the last line under Python indexing" - }, - { - "test": "1. a C-only region yields ONE stimulus with frames == ['C']", - "expected": [ - 1, - [ - "C" - ] - ], - "observed": [ - 1, - [ - "C" - ] - ], - "pass": true, - "fails_when": "a C-only region is mislabelled or emitted more than once" - }, - { - "test": "2. a D-only region yields ONE stimulus with frames == ['D']", - "expected": [ - "D" - ], - "observed": [ - "D" - ], - "pass": true, - "fails_when": "a D-only region is mislabelled or emitted more than once" - }, - { - "test": "3. an OVERLAP region yields ONE stimulus with frames == ['C','D']", - "expected": [ - "C", - "D" - ], - "observed": [ - "C", - "D" - ], - "pass": true, - "fails_when": "the overlap is projected onto a single frame, losing one membership (A36.2)" - }, - { - "test": "4. the overlap creates EXACTLY ONE primary blind id", - "expected": 1, - "observed": 1, - "pass": true, - "fails_when": "two blind ids exist for one physical region, so it would be rendered and adjudicated twice and A30.5's identity uniqueness would have had to be weakened to permit it" - }, - { - "test": "17. ...and is rendered exactly once at primary DPI", - "expected": 1, - "observed": 1, - "pass": true, - "fails_when": "the same primary region is rasterised twice, which would also produce two PNG hashes for one committed bbox" - }, - { - "test": "5. adding D membership to a C item cannot change its BASE IDENTITY", - "expected": [ - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",2,0]", - "[\"region\",\"synthsha0123456789\",3,0]" - ], - "observed": [ - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",2,0]", - "[\"region\",\"synthsha0123456789\",3,0]" - ], - "pass": true, - "fails_when": "frame membership leaked into A28.3's identity, which would re-key every selection and every blind id the moment a region became discordant" - }, - { - "test": "8. adding C membership to a D item cannot remove it from the D census", - "expected": [ - true, - true, - true - ], - "observed": [ - true, - true, - true - ], - "pass": true, - "fails_when": "a region drawn into C is dropped from the discordance census, shrinking the only evidence that can satisfy Rule 1" - }, - { - "test": "...and the D-only items are in D too, so control 8 is not comparing against nothing", - "expected": [ - true, - true, - true - ], - "observed": [ - true, - true, - true - ], - "pass": true, - "fails_when": "the D-only fixture is not actually in D" - }, - { - "test": "9. frame membership ORDER cannot change canonical output", - "expected": [ - "C", - "D" - ], - "observed": [ - "C", - "D" - ], - "pass": true, - "fails_when": "membership is emitted in input order, so the same stimulus could serialize two ways" - }, - { - "test": "6. adding D membership cannot change C selection", - "expected": [ - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",2,0]", - "[\"region\",\"synthsha0123456789\",3,0]", - "[\"region\",\"synthsha0123456789\",4,0]", - "[\"region\",\"synthsha0123456789\",5,0]", - "[\"region\",\"synthsha0123456789\",6,0]", - "[\"region\",\"synthsha0123456789\",7,0]", - "[\"region\",\"synthsha0123456789\",8,0]", - "[\"region\",\"synthsha0123456789\",9,0]", - "[\"region\",\"synthsha0123456789\",10,0]", - "[\"region\",\"synthsha0123456789\",11,0]", - "[\"region\",\"synthsha0123456789\",12,0]", - "[\"region\",\"synthsha0123456789\",13,0]", - "[\"region\",\"synthsha0123456789\",14,0]", - "[\"region\",\"synthsha0123456789\",15,0]", - "[\"region\",\"synthsha0123456789\",16,0]", - "[\"region\",\"synthsha0123456789\",17,0]", - "[\"region\",\"synthsha0123456789\",18,0]", - "[\"region\",\"synthsha0123456789\",19,0]", - "[\"region\",\"synthsha0123456789\",20,0]" - ], - "observed": [ - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",2,0]", - "[\"region\",\"synthsha0123456789\",3,0]", - "[\"region\",\"synthsha0123456789\",4,0]", - "[\"region\",\"synthsha0123456789\",5,0]", - "[\"region\",\"synthsha0123456789\",6,0]", - "[\"region\",\"synthsha0123456789\",7,0]", - "[\"region\",\"synthsha0123456789\",8,0]", - "[\"region\",\"synthsha0123456789\",9,0]", - "[\"region\",\"synthsha0123456789\",10,0]", - "[\"region\",\"synthsha0123456789\",11,0]", - "[\"region\",\"synthsha0123456789\",12,0]", - "[\"region\",\"synthsha0123456789\",13,0]", - "[\"region\",\"synthsha0123456789\",14,0]", - "[\"region\",\"synthsha0123456789\",15,0]", - "[\"region\",\"synthsha0123456789\",16,0]", - "[\"region\",\"synthsha0123456789\",17,0]", - "[\"region\",\"synthsha0123456789\",18,0]", - "[\"region\",\"synthsha0123456789\",19,0]", - "[\"region\",\"synthsha0123456789\",20,0]" - ], - "pass": true, - "fails_when": "the C membership set moved when D membership changed, i.e. the draw consumed the thing it is supposed to be independent of" - }, - { - "test": "7. adding D membership cannot change C AUDIT selection or its denominator", - "expected": [ - [ - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",3,0]", - "[\"region\",\"synthsha0123456789\",12,0]", - "[\"region\",\"synthsha0123456789\",15,0]", - "[\"region\",\"synthsha0123456789\",20,0]" - ], - 5 - ], - "observed": [ - [ - "[\"region\",\"synthsha0123456789\",1,0]", - "[\"region\",\"synthsha0123456789\",3,0]", - "[\"region\",\"synthsha0123456789\",12,0]", - "[\"region\",\"synthsha0123456789\",15,0]", - "[\"region\",\"synthsha0123456789\",20,0]" - ], - 5 - ], - "pass": true, - "fails_when": "the 25-item audit sample or size moved with D membership, which would make the audit a function of architecture disagreement (A36.5)" - }, - { - "test": "...and the D-heavy fixture really did gain D members, so control 7 is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no region gained D membership, so nothing was perturbed" - }, - { - "test": "11. an overlap stimulus requires BOTH the AI and the human route", - "expected": [ - "ai", - "human" - ], - "observed": [ - "ai", - "human" - ], - "pass": true, - "fails_when": "an overlap region is answered on one route, dropping it from RQ2 metrics or from Rule 1 decision evidence (A36.4)" - }, - { - "test": "...and C-only takes AI while D-only takes human", - "expected": [ - [ - "ai" - ], - [ - "human" - ] - ], - "observed": [ - [ - "ai" - ], - [ - "human" - ] - ], - "pass": true, - "fails_when": "a frame's route is wrong, so the wrong oracle would supply its labels" - }, - { - "test": "12. the C metric route reads the AI answer EVEN WHERE a human answer exists", - "expected": "AI-ANSWER", - "observed": "AI-ANSWER", - "pass": true, - "fails_when": "human truth is substituted into C on overlap regions, making C a mixed oracle whose source is selected by H/X discordance -- the architectures choosing their own oracle on exactly the regions where they disagree" - }, - { - "test": "13. the D decision route reads the human answer EVEN WHERE an AI answer exists", - "expected": "HUMAN-ANSWER", - "observed": "HUMAN-ANSWER", - "pass": true, - "fails_when": "an AI answer decides Rule 1, which 5.5.1 reserves for human adjudication" - }, - { - "test": "...and a missing mandated answer REFUSES rather than falling back to the other route", - "expected": "ANSWER_MISSING_FOR_REQUIRED_ROUTE", - "observed": "ANSWER_MISSING_FOR_REQUIRED_ROUTE", - "pass": true, - "fails_when": "a missing AI answer silently falls back to human, which is the prohibited substitution arriving through the back door" - }, - { - "test": "the adjudication artifact schema requires two namespaced answer sets (A36.4)", - "expected": "ADJUDICATION_NOT_NAMESPACED", - "observed": "ADJUDICATION_NOT_NAMESPACED", - "pass": true, - "fails_when": "a flat one-answer-per-id artifact validates, which would collapse an overlap region's two independent answers into whichever was written last" - }, - { - "test": "...and a correctly namespaced artifact is accepted", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "the validator refuses everything, making the control above meaningless" - }, - { - "test": "14. an audit-selected C-and-D item uses ONE human task for both D and the audit", - "expected": [ - true, - [ - 1, - 1 - ] - ], - "observed": [ - true, - [ - 1, - 1 - ] - ], - "pass": true, - "fails_when": "the human is asked to answer the same blind image twice, or the item exists in no fixture so the reuse rule was never exercised" - }, - { - "test": "...and that one task carries BOTH purposes", - "expected": [ - [ - "d_decision", - "c_audit" - ], - [ - "d_decision", - "c_audit" - ] - ], - "observed": [ - [ - "d_decision", - "c_audit" - ], - [ - "d_decision", - "c_audit" - ] - ], - "pass": true, - "fails_when": "the shared answer is recorded as serving only one purpose" - }, - { - "test": "15. a NON-audit C-and-D human answer cannot enlarge the audit denominator", - "expected": [ - true, - 5 - ], - "observed": [ - true, - 5 - ], - "pass": true, - "fails_when": "a human answer that exists only because of D membership counts as an audit item, inflating the audit beyond the seeded cframe-audit draw (A36.5)" - }, - { - "test": "...and non-audit overlap items exist, so control 15 is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "every overlap item happened to be audit-selected, so nothing was tested" - }, - { - "test": "16. an R1 repeat keeps ONE identity and inherits its primary's routes", - "expected": [ - 2, - true - ], - "observed": [ - 2, - true - ], - "pass": true, - "fails_when": "a repeat carries a route-specific identity or a route its primary does not have, which would make the reliability measurement compare two different things" - }, - { - "test": "10. the blind artifact leaks no membership and no route", - "expected": [ - 0, - [], - [] - ], - "observed": [ - 0, - [], - [] - ], - "pass": true, - "fails_when": "frame membership, audit status or adjudication route reaches the adjudicator, who could then infer that a region is one the architectures disagree about" - }, - { - "test": "10b. NEGATIVE -- injected membership/route in the blind file FAILS the gate", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "membership or route can be added to the adjudicator's file without the gate firing" - }, - { - "test": "the occurrence join RESOLVES an adjudicated heading to an A30 occurrence key", - "expected": [ - true, - [ - 1, - 2 - ] - ], - "observed": [ - true, - [ - 1, - 2 - ] - ], - "pass": true, - "fails_when": "a scorer cannot get from an adjudicated (line, x) to an occurrence identity at all, which is what A35's image-binding join left unproven" - }, - { - "test": "...and it selects the NEAREST candidate, not merely some candidate", - "expected": 20, - "observed": 20, - "pass": true, - "fails_when": "the resolver returned a glyph that is not nearest the converted coordinate" - }, - { - "test": "...and the key is the A30 4-tuple (doc_sha, page, neutral_line_key, start_ngid)", - "expected": [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ], - "observed": [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ], - "pass": true, - "fails_when": "the occurrence key is not the frozen A30 shape, so it cannot join to the emitted side" - }, - { - "test": "each stimulus carries ONLY its own page's sentinel occurrence", - "expected": { - "1": [ - "SYNTHETIC ACCOUNT SENTINEL P1" - ], - "2": [ - "SYNTHETIC ACCOUNT SENTINEL P2" - ], - "3": [ - "SYNTHETIC ACCOUNT SENTINEL P3" - ], - "4": [ - "SYNTHETIC ACCOUNT SENTINEL P4" - ] - }, - "observed": { - "1": [ - "SYNTHETIC ACCOUNT SENTINEL P1" - ], - "2": [ - "SYNTHETIC ACCOUNT SENTINEL P2" - ], - "3": [ - "SYNTHETIC ACCOUNT SENTINEL P3" - ], - "4": [ - "SYNTHETIC ACCOUNT SENTINEL P4" - ] - }, - "pass": true, - "fails_when": "a stimulus holds another page's sentinel, i.e. region_ordinal is being matched without the page and every multi-page document contaminates every region" - }, - { - "test": "...and the fixture really does reuse region_ordinal across pages, so this is not vacuous", - "expected": [ - true, - [ - 0 - ] - ], - "observed": [ - true, - [ - 0 - ] - ], - "pass": true, - "fails_when": "the fixture has one page or distinct ordinals per page, so a page-blind filter would have looked correct here" - }, - { - "test": "POSITIVE -- the adjudicated key EQUALS the emitted architecture occurrence key", - "expected": [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ], - "observed": [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ], - "pass": true, - "fails_when": "the two sides of the M1-M5 join do not meet, so no matched-heading denominator exists" - }, - { - "test": "NEGATIVE -- crossing the nearest-glyph boundary CHANGES the resolved key", - "expected": [ - true, - 21 - ], - "observed": [ - true, - 21 - ], - "pass": true, - "fails_when": "moving the annotation onto the other glyph resolves to the same ngid, which would mean the resolver is not actually reading the coordinate" - }, - { - "test": "NEGATIVE -- an EXACT geometric tie refuses rather than picking one", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "a tie silently resolves to a glyph, inventing an identity A30.3 says does not exist" - }, - { - "test": "NEGATIVE -- removing the identity candidates makes the join REFUSE", - "expected": [ - false, - "MISSING_IDENTITY_CANDIDATES" - ], - "observed": [ - false, - "MISSING_IDENTITY_CANDIDATES" - ], - "pass": true, - "fails_when": "the join returns a match without candidate geometry, i.e. it is not reading it" - }, - { - "test": "NEGATIVE -- an out-of-range start_physical_line REFUSES", - "expected": "START_LINE_OUT_OF_RANGE", - "observed": "START_LINE_OUT_OF_RANGE", - "pass": true, - "fails_when": "an impossible line resolves anyway, handing the resolver the wrong candidate set" - }, - { - "test": "NEGATIVE -- changing heading TEXT does not move the occurrence identity", - "expected": [ - [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 2, - [ - 2, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 3, - [ - 3, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 4, - [ - 4, - 2 - ], - 20 - ] - ], - "observed": [ - [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 2, - [ - 2, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 3, - [ - 3, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 4, - [ - 4, - 2 - ], - 20 - ] - ], - "pass": true, - "fails_when": "identity consumes text, so a word-boundary defect would silently renumber the very occurrences the study exists to compare (A30.1)" - }, - { - "test": "NEGATIVE -- changing anchor KIND does not move the occurrence identity", - "expected": [ - [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 2, - [ - 2, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 3, - [ - 3, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 4, - [ - 4, - 2 - ], - 20 - ] - ], - "observed": [ - [ - "synthsha0123456789", - 1, - [ - 1, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 2, - [ - 2, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 3, - [ - 3, - 2 - ], - 20 - ], - [ - "synthsha0123456789", - 4, - [ - 4, - 2 - ], - 20 - ] - ], - "pass": true, - "fails_when": "identity consumes anchor kind rather than absolute source position" - }, - { - "test": "NEGATIVE -- an UNMATCHED occurrence is RETAINED and explicitly flagged", - "expected": [ - "UNMATCHED", - "SYNTHETIC_REFUSAL" - ], - "observed": [ - "UNMATCHED", - "SYNTHETIC_REFUSAL" - ], - "pass": true, - "fails_when": "an occurrence whose A30 resolution refused is dropped, shrinking an M1-M5 denominator invisibly instead of being reported as a refusal" - }, - { - "test": "REQUIRED_JOIN_FIELDS now includes the occurrence-join facts", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the completeness gate still checks only image/region binding, so a key missing the occurrence facts would pass and the defect would resurface in score_metrics" - }, - { - "test": "NEGATIVE -- a key missing its architecture occurrences FAILS the completeness gate", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "verify_join stays green without the occurrence records, which is exactly the false sufficiency A35's join had" - }, - { - "test": "a well-formed adjudicated artifact VALIDATES", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "the encoding rejects a conforming artifact" - }, - { - "test": "the heading fields are EXACTLY the six the prompt asks for", - "expected": [ - "text", - "role", - "parent", - "start_physical_line", - "start_x_px" - ], - "observed": [ - "text", - "role", - "parent", - "start_physical_line", - "start_x_px" - ], - "pass": true, - "fails_when": "the encoding asks for a field the prompt never collected, or drops one it did" - }, - { - "test": "NEGATIVE -- an unknown blind id REFUSES", - "expected": "UNKNOWN_BLIND_ID", - "observed": "UNKNOWN_BLIND_ID", - "pass": true, - "fails_when": "an answer for a stimulus that does not exist is accepted and would join to nothing" - }, - { - "test": "NEGATIVE -- an answer whose id differs from its namespace key REFUSES", - "expected": "ADJUDICATION_ID_MISMATCH", - "observed": "ADJUDICATION_ID_MISMATCH", - "pass": true, - "fails_when": "an answer can be filed under one id while claiming another, silently misjoining it" - }, - { - "test": "NEGATIVE -- a stimulus missing a REQUIRED route REFUSES, with no fallback", - "expected": "ADJUDICATION_ROUTE_MISSING", - "observed": "ADJUDICATION_ROUTE_MISSING", - "pass": true, - "fails_when": "a missing AI answer falls back to the human one, which is the A36.4 prohibition" - }, - { - "test": "UNREADABLE stays representable per field", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "an UNREADABLE field is rejected, forcing the adjudicator to guess a value instead" - }, - { - "test": "the emitted map is COMPLETE against production's AnchorKind", - "expected": [ - "account", - "agency", - "grouping", - "major", - "preamble", - "section", - "subsection", - "title" - ], - "observed": [ - "account", - "agency", - "grouping", - "major", - "preamble", - "section", - "subsection", - "title" - ], - "pass": true, - "fails_when": "production emits a kind the M5 map does not cover, so that kind would arrive as an unmapped role -- this control is what makes a future kind FAIL instead of slipping in" - }, - { - "test": "every oracle role maps exactly as A36.7 freezes it", - "expected": { - "account": "LEAF", - "section": "LEAF", - "agency": "CONTAINER", - "grouping": "CONTAINER", - "title": "CONTAINER", - "division": "CONTAINER", - "other": "UNSCORABLE" - }, - "observed": { - "account": "LEAF", - "section": "LEAF", - "agency": "CONTAINER", - "grouping": "CONTAINER", - "title": "CONTAINER", - "division": "CONTAINER", - "other": "UNSCORABLE" - }, - "pass": true, - "fails_when": "a role coarsens differently from the frozen table" - }, - { - "test": "every emitted kind maps exactly as A36.7 freezes it", - "expected": { - "account": "LEAF", - "section": "LEAF", - "major": "CONTAINER", - "agency": "CONTAINER", - "grouping": "CONTAINER", - "title": "CONTAINER", - "subsection": "UNSCORABLE", - "preamble": "UNSCORABLE" - }, - "observed": { - "account": "LEAF", - "section": "LEAF", - "major": "CONTAINER", - "agency": "CONTAINER", - "grouping": "CONTAINER", - "title": "CONTAINER", - "subsection": "UNSCORABLE", - "preamble": "UNSCORABLE" - }, - "pass": true, - "fails_when": "an emitted kind coarsens differently from the frozen table" - }, - { - "test": "a LEAF/LEAF pair is scorable and agrees", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "an account-to-account match is excluded or read as disagreement" - }, - { - "test": "a LEAF/CONTAINER pair is scorable and DISAGREES", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "a genuine role disagreement is hidden" - }, - { - "test": "an UNSCORABLE oracle role is EXCLUDED, not counted as disagreement", - "expected": [ - false, - null - ], - "observed": [ - false, - null - ], - "pass": true, - "fails_when": "`other` enters the M5 denominator, penalising the architecture for a role M5 was never licensed to score" - }, - { - "test": "an UNSCORABLE emitted kind is EXCLUDED on the other side too", - "expected": [ - false, - null - ], - "observed": [ - false, - null - ], - "pass": true, - "fails_when": "`subsection` or `preamble` enters the denominator" - }, - { - "test": "NEGATIVE -- an unknown oracle role REFUSES rather than becoming UNSCORABLE", - "expected": "UnknownRole", - "observed": "UnknownRole", - "pass": true, - "fails_when": "an unmapped role silently becomes UNSCORABLE, quietly SHRINKING the denominator -- and a smaller denominator reads as a cleaner result rather than as a defect" - }, - { - "test": "NEGATIVE -- an unknown emitted kind REFUSES", - "expected": "UnknownRole", - "observed": "UnknownRole", - "pass": true, - "fails_when": "an unmapped emitted kind is silently excluded" - }, - { - "test": "18. raw C, raw D and the C-and-D overlap are reported SEPARATELY on real material", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "a frame size or the overlap count is missing or disagrees with the frames, so a reader could not see that an overlap region is counted once in each estimand" - }, - { - "test": "every DEVELOPMENT overlap region produced EXACTLY ONE stimulus (A36.2)", - "expected": 18, - "observed": 18, - "pass": true, - "fails_when": "an overlap region was emitted twice or dropped, so |C| + |D| would not reconcile with the realized stimulus set" - }, - { - "test": "...and every one of them carries BOTH required routes", - "expected": [ - 18, - true - ], - "observed": [ - 18, - true - ], - "pass": true, - "fails_when": "an overlap region is answered on one route only, silently dropping it from either the RQ2 metrics or the Rule 1 decision evidence" - }, - { - "test": "the DEVELOPMENT overlap population is non-empty, so the two controls above are not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no overlap region exists on this material, so nothing about overlap was exercised" - }, - { - "test": "every DEVELOPMENT stimulus renders at the frozen width for its own DPI", - "expected": [ - 193, - true - ], - "observed": [ - 193, - true - ], - "pass": true, - "fails_when": "a rendered image is not the width the A34 derivation predicts for its bbox and scale" - }, - { - "test": "every planned DEVELOPMENT stimulus was rendered, so no denominator is a sample", - "expected": 193, - "observed": 193, - "pass": true, - "fails_when": "a stimulus was planned but not rendered, which would shrink the study invisibly" - }, - { - "test": "the DEVELOPMENT blind file leaks nothing", - "expected": [ - 0, - [], - [] - ], - "observed": [ - 0, - [], - [] - ], - "pass": true, - "fails_when": "a private value or forbidden token reaches the adjudicator on real material" - }, - { - "test": "the DEVELOPMENT join is complete", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a real stimulus cannot be bound back to its region" - }, - { - "test": "the DEVELOPMENT population is non-empty, so these controls are not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no document or no stimulus was produced, so every check above passed on nothing" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x22_score_input_contract.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x22_score_input_contract.json deleted file mode 100644 index d0098139..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x22_score_input_contract.json +++ /dev/null @@ -1,785 +0,0 @@ -{ - "population": "SYNTHETIC + DEVELOPMENT -- no holdout opened, nothing adjudicated, nothing scored", - "contract": "A38 (the score_metrics input contract)", - "question": "can every score_metrics obligation be satisfied from committed artifacts, with no PDF access, no runner access, no text matching, and no implementation-time choice?", - "document": "118-hr-8752/1", - "page_limit": 16, - "note": "PAGE_LIMIT is a machinery demonstration window, not a census", - "artifacts_created": "NONE of frames.json, oracle_key.json, oracle_blind.json, oracle_adjudicated.json, metrics.json, scores.json, EXECUTION-START.json", - "identity_candidates": { - "n_neutral_lines": 433, - "n_with_candidates": 433, - "n_candidates": 17842 - }, - "bilateral_gate": { - "limit": "one skeleton by A19 -- both runners call run_hybrid.neutral_skeleton; this gate catches divergence in what each arm RETURNS, not two independent derivations" - }, - "architecture_occurrences": { - "H": { - "total": 33, - "MATCHABLE": 33, - "UNMATCHED": 0 - }, - "X": { - "total": 33, - "MATCHABLE": 33, - "UNMATCHED": 0 - } - }, - "occurrence_scope": { - "ordinals_reused_across_pages": { - "0": [ - 2, - 3, - 4, - 5, - 6, - 11, - 13, - 15, - 16 - ], - "1": [ - 2, - 3, - 4, - 5 - ], - "2": [ - 3, - 8, - 12, - 14, - 15 - ] - }, - "n_stimuli": 51, - "n_occurrence_rows_in_key": 50, - "counterfactual_contaminated_rows": 425 - }, - "genuine_unmatched": { - "lever": "merged-line text perturbation -> MERGE_RECONSTRUCTION_MISMATCH; placement reads print_lines/emitted and is unaffected", - "page": 2, - "region_ordinal": 0, - "unmatched_reason": "MERGE_RECONSTRUCTION_MISMATCH", - "stimuli_carrying_it": 1 - }, - "immediate_parent": { - "checked": 33, - "agree": 33, - "kinds": { - "title": { - "n": 2, - "with_parent": 0 - }, - "major": { - "n": 2, - "with_parent": 2 - }, - "agency": { - "n": 8, - "with_parent": 8 - }, - "account": { - "n": 14, - "with_parent": 14 - }, - "grouping": { - "n": 1, - "with_parent": 1 - }, - "section": { - "n": 6, - "with_parent": 6 - } - }, - "n_root": 2 - }, - "m9_raw": { - "H": { - "derive_size_bands_returns_a_band": true, - "coverage": 1.0, - "coverage_floor": 0.85, - "coverage_meets_floor": true, - "n_lines_total": 293, - "n_margin_numbered_lines": 277, - "n_margin_numbered_with_glyph_size": 277, - "rule0_comparison": "RAW FACTS ONLY -- Rule 0 margin-line interpretation frozen by A39.1 (count of Page.lines where line_number is not None; any positive deficit fires); application belongs to the later architecture decision" - }, - "X": { - "derive_size_bands_returns_a_band": true, - "coverage": 1.0, - "coverage_floor": 0.85, - "coverage_meets_floor": true, - "n_lines_total": 387, - "n_margin_numbered_lines": 370, - "n_margin_numbered_with_glyph_size": 370, - "rule0_comparison": "RAW FACTS ONLY -- Rule 0 margin-line interpretation frozen by A39.1 (count of Page.lines where line_number is not None; any positive deficit fires); application belongs to the later architecture decision" - } - }, - "s1": { - "schema": "s1_control/1", - "advance_scale": 1.25, - "sabotaged_arm": "X", - "primary": { - "risk_set_lines": 387, - "text_discordant_lines": 100, - "m0a_rate": 0.25839793281653745 - }, - "sabotaged": { - "risk_set_lines": 387, - "text_discordant_lines": 125, - "m0a_rate": 0.32299741602067183 - }, - "fires": true, - "reason": null - }, - "cross_engine": { - "document": "118-hr-8752/1", - "limited_page_count": 16, - "whole_document_page_count": 98, - "sampled_pages": [ - 13, - 15 - ], - "matched": 55, - "denominator": 55, - "gate": { - "matched_fraction": 1.0, - "worst_page": 13, - "worst_page_fraction": 1.0, - "document_threshold": 0.95, - "page_threshold": 0.75, - "pass": true, - "failed_on": [] - }, - "qualification": null, - "oversegmentation": { - "pdfium_over_120": 0.8333, - "pymupdf_over_120": 0.8333 - }, - "whole_document_gate": { - "matched_fraction": 1.0, - "worst_page": 13, - "worst_page_fraction": 1.0, - "document_threshold": 0.95, - "page_threshold": 0.75, - "pass": true, - "failed_on": [] - } - }, - "reachability": { - "reachable": { - "M0a text discordance": true, - "M0b segmentation discordance": true, - "M0c anchor discordance": true, - "both_absent": true, - "M1-M3 emitted occurrences": true, - "M4 immediate parent": true, - "M5 emitted kind": true, - "M7 emitted text": true, - "M9 raw facts": true, - "occurrence join candidates": true, - "occurrence join geometry": true, - "adjudication routing": true - } - }, - "oracle_key_schema": "oracle_key/3", - "forward_ambiguities": [], - "resolved_ambiguities": [ - { - "was": "RULE0_MARGIN_LINE_QUANTITY", - "status": "RESOLVED_BY_A39_1", - "ruling": "margin_lines_recovered = count of Page.lines where line_number is not None; any strictly positive per-document deficit fires; no tolerance. The glyph-size count and per-line keys remain diagnostics and do not determine the clause." - } - ], - "tests": [ - { - "test": "every committed neutral line's candidates are EXACTLY its gids", - "expected": [ - 433, - true - ], - "observed": [ - 433, - true - ], - "pass": true, - "fails_when": "a gid has no candidate, or a candidate names a gid outside the line -- either way the A30.3 resolver would search the wrong glyph set" - }, - { - "test": "the candidate population is non-empty, so the check above is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no line carries candidates, so 'all lines agree' passed on nothing" - }, - { - "test": "no candidate exists outside the committed gid sets (U+0020 among them)", - "expected": "set()", - "observed": "set()", - "pass": true, - "fails_when": "a candidate was invented from a source glyph the skeleton rejected, e.g. a word space" - }, - { - "test": "candidate x0 is UNROUNDED source geometry", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "every x0 lands on a 4-decimal grid, i.e. it was rounded for serialization -- rounding here can move a nearest-glyph decision" - }, - { - "test": "candidates serialize in ngid order (determinism ONLY, never reading order)", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "serialization order varies between runs, so the committed frame is not reproducible" - }, - { - "test": "NEGATIVE -- mutating ONE arm's candidate x0 ABORTS frame construction", - "expected": "NEUTRAL_SKELETON_MISMATCH", - "observed": "NEUTRAL_SKELETON_MISMATCH", - "pass": true, - "fails_when": "the arms' candidate geometry can diverge without the frame noticing, so the committed candidates would silently be whichever arm was read first" - }, - { - "test": "...and the UNMUTATED pair still builds, so the gate is not refusing everything", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "construction refuses even on clean input, making the control above meaningless" - }, - { - "test": "both arms produced occurrence records", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "an arm produced no occurrences, so M1-M5 would have an empty emitted side" - }, - { - "test": "every record is MATCHABLE or UNMATCHED and none is silently absent", - "expected": [ - 33, - 33 - ], - "observed": [ - 33, - 33 - ], - "pass": true, - "fails_when": "a record carries neither status, i.e. an occurrence was dropped rather than reported" - }, - { - "test": "a MATCHABLE record carries the A30 4-component occurrence key", - "expected": [ - 4 - ], - "observed": [ - 4 - ], - "pass": true, - "fails_when": "the key is not the frozen (doc_sha, page, neutral_line_key, start_ngid) shape" - }, - { - "test": "records are in DOCUMENT order, never ngid order", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "occurrences are ordered by ngid, which A30.1 forbids -- ngid order disagreed with printed order on 10 of 33,602 emitted lines" - }, - { - "test": "the DEVELOPMENT material really does reuse region ordinals across pages", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no ordinal repeats across pages here, so this material could not have exposed the contamination and the controls below would be vacuous" - }, - { - "test": "EVERY private-key occurrence satisfies anchor.page == stimulus.page AND region == region", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a stimulus carries an occurrence printed on another page or in another region, which would put headings the adjudicator never saw on the emitted side of the M1-M5 join" - }, - { - "test": "NEGATIVE -- the retired region-ordinal-only filter DOES contaminate this material", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the old filter happens to be equivalent here, so the repair is unproven and the control above passes for the wrong reason" - }, - { - "test": "no architecture occurrence is duplicated into multiple PRIMARY stimuli", - "expected": {}, - "observed": {}, - "pass": true, - "fails_when": "one emitted heading is counted in two regions, inflating the emitted set twice over" - }, - { - "test": "a MATCHABLE occurrence's A30 key page AGREES with its stimulus page", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the identity key names a different page from the stimulus carrying it, so the join would bind an adjudication to a heading printed elsewhere" - }, - { - "test": "a GENUINE A30 refusal is UNMATCHED with an explicit reason", - "expected": [ - "UNMATCHED", - null, - true - ], - "observed": [ - "UNMATCHED", - null, - true - ], - "pass": true, - "fails_when": "the perturbation did not actually induce an identity refusal, so this control proves nothing about how refusals are handled" - }, - { - "test": "...and it KEEPS its A28.5 physical page and region", - "expected": [ - 2, - 0 - ], - "observed": [ - 2, - 0 - ], - "pass": true, - "fails_when": "an identity refusal erased the physical placement, so the occurrence would vanish from every region-scoped record and shrink an M1-M5 denominator invisibly" - }, - { - "test": "the UNMATCHED occurrence SURVIVES into its exact stimulus", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the refused occurrence reaches no private key at all, i.e. it was silently dropped" - }, - { - "test": "...and appears in NO other page/region", - "expected": [ - "0edf34ebb764cd39" - ], - "observed": [ - "0edf34ebb764cd39" - ], - "pass": true, - "fails_when": "the refused occurrence also leaked into a stimulus for a different page or region" - }, - { - "test": "every committed immediate_parent EQUALS production breadcrumb_for's penultimate element", - "expected": [ - 33, - 33 - ], - "observed": [ - 33, - 33 - ], - "pass": true, - "fails_when": "the parent was derived by a NEW hierarchy walk that disagrees with production, so M4 would score the emitted side against a hierarchy production never produced" - }, - { - "test": "the parent comparison is non-empty", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no occurrence matched a production anchor, so the equality above compared nothing" - }, - { - "test": "a root/no-parent case exists and is represented as null", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no one-element breadcrumb appears, so the no-parent branch was never exercised" - }, - { - "test": "M9 raw facts are committed for BOTH arms", - "expected": [ - [ - "coverage", - "coverage_floor", - "coverage_meets_floor", - "derive_size_bands_returns_a_band", - "margin_numbered_line_keys", - "n_lines_total", - "n_margin_numbered_lines", - "n_margin_numbered_with_glyph_size" - ], - [ - "coverage", - "coverage_floor", - "coverage_meets_floor", - "derive_size_bands_returns_a_band", - "margin_numbered_line_keys", - "n_lines_total", - "n_margin_numbered_lines", - "n_margin_numbered_with_glyph_size" - ] - ], - "observed": [ - [ - "coverage", - "coverage_floor", - "coverage_meets_floor", - "derive_size_bands_returns_a_band", - "margin_numbered_line_keys", - "n_lines_total", - "n_margin_numbered_lines", - "n_margin_numbered_with_glyph_size" - ], - [ - "coverage", - "coverage_floor", - "coverage_meets_floor", - "derive_size_bands_returns_a_band", - "margin_numbered_line_keys", - "n_lines_total", - "n_margin_numbered_lines", - "n_margin_numbered_with_glyph_size" - ] - ], - "pass": true, - "fails_when": "a raw M9 quantity is missing, so score_metrics would have to rerun production anchor extraction to discover it" - }, - { - "test": "the frozen coverage floor is production's own 0.85", - "expected": [ - 0.85, - 0.85 - ], - "observed": [ - 0.85, - 0.85 - ], - "pass": true, - "fails_when": "the floor drifted from production's _COVERAGE_MIN, i.e. a threshold was invented here" - }, - { - "test": "per-line margin keys are recorded, so a SET comparison stays possible", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "only counts are recorded, foreclosing the set-difference reading of Rule 0 before it has been ruled on" - }, - { - "test": "this stage still records FACTS and does not APPLY Rule 0", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "frame construction applies the Rule 0 comparison, which belongs to the later architecture decision" - }, - { - "test": "RULE0_MARGIN_LINE_QUANTITY is RESOLVED_BY_A39_1, and the ruling is implemented", - "expected": { - "H_loses": [ - "H", - true, - 1 - ], - "X_loses": [ - "X", - true, - 1 - ], - "equal": [ - null, - false, - 0 - ] - }, - "observed": { - "H_loses": [ - "H", - true, - 1 - ], - "X_loses": [ - "X", - true, - 1 - ], - "equal": [ - null, - false, - 0 - ] - }, - "pass": true, - "fails_when": "the frozen A39.1 clause -- count of line_number is not None, ANY positive deficit fires, no tolerance -- is not what margin_line_loss implements, so the ledger would be claiming a resolution the code does not deliver" - }, - { - "test": "the S1 sabotage scale is exactly the frozen 1.25", - "expected": 1.25, - "observed": 1.25, - "pass": true, - "fails_when": "the scale drifted from the frozen value, or became a tunable dial" - }, - { - "test": "S1 FIRES on DEVELOPMENT -- sabotage raises M0", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "advances x 1.25 does not raise the discordance count, which would mean M0 cannot detect a change it certainly should -- the comparator is not live" - }, - { - "test": "...and the sabotage really changed the input, so firing is not an artifact", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the two runs report the same discordance, so nothing was actually sabotaged" - }, - { - "test": "primary and sabotaged M0 are reported SEPARATELY", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "only one number is reported, so a reader cannot see what the control established" - }, - { - "test": "the risk set is unchanged by sabotage, so only the TEXT moved", - "expected": 387, - "observed": 387, - "pass": true, - "fails_when": "the sabotage moved the neutral skeleton, i.e. it changed the frame rather than the seam decision under test" - }, - { - "test": "NEGATIVE -- a DEAD comparator (sabotage changes nothing) reports S1 FAIL", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "a comparator that cannot move still reports S1 firing, which is precisely the phase-1 defect S1 exists to catch" - }, - { - "test": "the producer RUNS on a real DEVELOPMENT PDF without raising", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the producer cannot execute at all -- which is how a nonexistent row key and the wrong denominator survived: nothing ever ran it on real measurements" - }, - { - "test": "sampled pages equal the independently derived A39.2 sample", - "expected": [ - 13, - 15 - ], - "observed": [ - 13, - 15 - ], - "pass": true, - "fails_when": "the producer measures a different page set from the frozen sample" - }, - { - "test": "matched count and DENOMINATOR equal the direct x09 computation", - "expected": [ - 55, - 55 - ], - "observed": [ - 55, - 55 - ], - "pass": true, - "fails_when": "the producer counts a different numerator or denominator than the frozen rule -- the exact defect: matched/pdfium instead of matched/max(pdfium, pymupdf)" - }, - { - "test": "the whole GATE VERDICT equals X09.gate on the same rows", - "expected": { - "matched_fraction": 1.0, - "worst_page": 13, - "worst_page_fraction": 1.0, - "document_threshold": 0.95, - "page_threshold": 0.75, - "pass": true, - "failed_on": [] - }, - "observed": { - "matched_fraction": 1.0, - "worst_page": 13, - "worst_page_fraction": 1.0, - "document_threshold": 0.95, - "page_threshold": 0.75, - "pass": true, - "failed_on": [] - }, - "pass": true, - "fails_when": "the producer's verdict differs from the frozen rule applied directly, i.e. it is recomputing the rule rather than calling it" - }, - { - "test": "document fraction, worst page and pass all agree", - "expected": [ - 1.0, - 13, - true - ], - "observed": [ - 1.0, - 13, - true - ], - "pass": true, - "fails_when": "a reported field disagrees with the gate that produced it" - }, - { - "test": "cross-engine failure is NEVER decision-blocking", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "a cross-engine failure blocks the architecture decision, which A27.6 forbids" - }, - { - "test": "PyMuPDF over-segmentation scores 100/120, not 100/100", - "expected": 0.8333, - "observed": 0.8333, - "pass": true, - "fails_when": "the denominator is the PDFium count, so over-segmentation by the SECOND engine is invisible -- the control would report perfect agreement while the engines disagree" - }, - { - "test": "PDFium over-segmentation scores 100/120 too -- the gate is SYMMETRIC", - "expected": 0.8333, - "observed": 0.8333, - "pass": true, - "fails_when": "the gate favours one engine, so which engine over-segments changes the verdict" - }, - { - "test": "...and both are capable of FAILING the frozen thresholds", - "expected": [ - false, - false - ], - "observed": [ - false, - false - ], - "pass": true, - "fails_when": "an 0.833 agreement passes, so the thresholds cannot see over-segmentation at all" - }, - { - "test": "NEGATIVE -- a wrong document SHA REFUSES before any sample or measurement", - "expected": "SOURCE_SHA256_MISMATCH", - "observed": "SOURCE_SHA256_MISMATCH", - "pass": true, - "fails_when": "a caller-supplied SHA that does not match the bytes selects a different page sample for the same document, silently and reproducibly" - }, - { - "test": "...and the correct SHA is accepted", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "verification refuses the real document, making the control above meaningless" - }, - { - "test": "mutating a NON-sampled page changes nothing", - "expected": { - "matched_fraction": 1.0, - "worst_page": 13, - "worst_page_fraction": 1.0, - "document_threshold": 0.95, - "page_threshold": 0.75, - "pass": true, - "failed_on": [] - }, - "observed": { - "matched_fraction": 1.0, - "worst_page": 13, - "worst_page_fraction": 1.0, - "document_threshold": 0.95, - "page_threshold": 0.75, - "pass": true, - "failed_on": [] - }, - "pass": true, - "fails_when": "an unsampled page reaches the gate, so this is not a 10 % sampled control at all" - }, - { - "test": "...while mutating a SAMPLED page DOES change the verdict", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "a sampled page cannot move the verdict, so sample membership is dead and the control above passed for the wrong reason" - }, - { - "test": "an unsampled page exists, so the membership control is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "every page is sampled here, so 'unsampled changes nothing' was trivially true" - }, - { - "test": "limit=None is a valid WHOLE-DOCUMENT call", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the canonical writer's default path raises or silently measures a prefix -- a truncated confirmatory measurement would qualify a frame on a fraction of its pages" - }, - { - "test": "...and it measures MORE pages than the limited call, so None is not read as a prefix", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "limit=None produced the same page count as the limited run, i.e. it was reinterpreted" - }, - { - "test": "EVERY listed score input is reachable from committed JSON, with no PDF or runner", - "expected": { - "M0a text discordance": true, - "M0b segmentation discordance": true, - "M0c anchor discordance": true, - "both_absent": true, - "M1-M3 emitted occurrences": true, - "M4 immediate parent": true, - "M5 emitted kind": true, - "M7 emitted text": true, - "M9 raw facts": true, - "occurrence join candidates": true, - "occurrence join geometry": true, - "adjudication routing": true - }, - "observed": { - "M0a text discordance": true, - "M0b segmentation discordance": true, - "M0c anchor discordance": true, - "both_absent": true, - "M1-M3 emitted occurrences": true, - "M4 immediate parent": true, - "M5 emitted kind": true, - "M7 emitted text": true, - "M9 raw facts": true, - "occurrence join candidates": true, - "occurrence join geometry": true, - "adjudication routing": true - }, - "pass": true, - "fails_when": "a fact score_metrics needs survives only in a live Python object, so the scorer would have to reopen a PDF or rerun an architecture to get it" - }, - { - "test": "NEGATIVE -- removing the occurrence records makes the reachability answer FALSE", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "the reachability check cannot detect a missing producer, so it would report success on an artifact chain that cannot actually feed the scorer" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x23_control_fixtures.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x23_control_fixtures.json deleted file mode 100644 index 4336b7db..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x23_control_fixtures.json +++ /dev/null @@ -1,1280 +0,0 @@ -{ - "population": "DEVELOPMENT + SYNTHETIC -- no confirmatory holdout material anywhere", - "contract": "A39.3 (control sources) as amended by A40 (live boundary control)", - "manifest": "results/control_fixtures.json", - "mutation_classes": { - "before": "SALARIES AND EXPENSES", - "delete": "AND EXPENSES", - "weld": "SALARIESAND EXPENSES", - "split": "SALA RIES AND EXPENSES" - }, - "realized": { - "counts": { - "N-A": 8, - "N-B": 8, - "N-C": 4 - }, - "allocation": { - "DELETE_ONE_WORD": 3, - "WELD_TWO_WORDS": 3, - "SPLIT_ONE_WORD": 2 - }, - "eligible_population": { - "admitted_account_sources": 96, - "na_eligible": 83, - "nb_eligible": 96, - "per_document": { - "114-hr-2029/4": { - "xml_account_records": 105, - "independent_anchors": 85, - "anchor_inversions": 0, - "bridge_paired": 55, - "bridge_refusals": { - "UNDISCRIMINATED_GROUP": 2, - "NO_PHYSICAL_OCCURRENCE": 3 - }, - "admitted_sources": 55 - }, - "118-hr-8752/1": { - "xml_account_records": 41, - "independent_anchors": 111, - "anchor_inversions": 0, - "bridge_paired": 41, - "bridge_refusals": {}, - "admitted_sources": 41 - } - } - }, - "na": [ - { - "index": 0, - "variant": "DELETE_ONE_WORD", - "source": "118-hr-8752/1", - "page": 20, - "before": "RESEARCH AND DEVELOPMENT", - "after": "RESEARCH AND", - "boundary_transitions": [], - "non_space_unchanged": false, - "generated_sha256": "cd1460a092dc44fc" - }, - { - "index": 1, - "variant": "WELD_TWO_WORDS", - "source": "114-hr-2029/4", - "page": 126, - "before": "(RESCISSIONS OF FUNDS)", - "after": "(RESCISSIONSOF FUNDS)", - "boundary_transitions": [ - [ - 1, - 0 - ] - ], - "non_space_unchanged": true, - "generated_sha256": "104f85729ac095d0" - }, - { - "index": 2, - "variant": "SPLIT_ONE_WORD", - "source": "114-hr-2029/4", - "page": 21, - "before": "(RESCISSION OF FUNDS)", - "after": "(RESCI SSION OF FUNDS)", - "boundary_transitions": [ - [ - 0, - 1 - ] - ], - "non_space_unchanged": true, - "generated_sha256": "61c9178fc8c51ab4" - }, - { - "index": 3, - "variant": "DELETE_ONE_WORD", - "source": "118-hr-8752/1", - "page": 58, - "before": "OPERATIONS AND SUPPORT", - "after": "AND SUPPORT", - "boundary_transitions": [], - "non_space_unchanged": false, - "generated_sha256": "b3f30e24b50424c3" - }, - { - "index": 4, - "variant": "WELD_TWO_WORDS", - "source": "114-hr-2029/4", - "page": 25, - "before": "VETERANS INSURANCE AND INDEMNITIES", - "after": "VETERANSINSURANCE AND INDEMNITIES", - "boundary_transitions": [ - [ - 1, - 0 - ] - ], - "non_space_unchanged": true, - "generated_sha256": "d5f67bb44f8490ce" - }, - { - "index": 5, - "variant": "SPLIT_ONE_WORD", - "source": "118-hr-8752/1", - "page": 56, - "before": "OPERATIONS AND SUPPORT", - "after": "OPERA TIONS AND SUPPORT", - "boundary_transitions": [ - [ - 0, - 1 - ] - ], - "non_space_unchanged": true, - "generated_sha256": "1bc08acb57b17f12" - }, - { - "index": 6, - "variant": "DELETE_ONE_WORD", - "source": "114-hr-2029/4", - "page": 23, - "before": "COMPENSATION AND PENSIONS", - "after": "AND PENSIONS", - "boundary_transitions": [], - "non_space_unchanged": false, - "generated_sha256": "7f0fddf896985ff9" - }, - { - "index": 7, - "variant": "WELD_TWO_WORDS", - "source": "118-hr-8752/1", - "page": 44, - "before": "OPERATIONS AND SUPPORT", - "after": "OPERATIONSAND SUPPORT", - "boundary_transitions": [ - [ - 1, - 0 - ] - ], - "non_space_unchanged": true, - "generated_sha256": "ce75db58bfc020fb" - } - ], - "nb": [ - { - "index": 0, - "source": "114-hr-2029/4", - "page": 57, - "expected_text": "(RESCISSIONS OF FUNDS)", - "xml_path": "tests/corpus/114-hr-2029/4_reported-in-senate.xml", - "xml_sha256": "68e98a6b0b204cfe" - }, - { - "index": 1, - "source": "118-hr-8752/1", - "page": 49, - "expected_text": "NATIONAL FLOOD INSURANCE FUND", - "xml_path": "tests/corpus/118-hr-8752/1_reported-in-house.xml", - "xml_sha256": "e4bc6b90eeff87d0" - }, - { - "index": 2, - "source": "114-hr-2029/4", - "page": 60, - "expected_text": "FOREIGN CURRENCY FLUCTUATIONS ACCOUNT", - "xml_path": "tests/corpus/114-hr-2029/4_reported-in-senate.xml", - "xml_sha256": "68e98a6b0b204cfe" - }, - { - "index": 3, - "source": "114-hr-2029/4", - "page": 21, - "expected_text": "(RESCISSION OF FUNDS)", - "xml_path": "tests/corpus/114-hr-2029/4_reported-in-senate.xml", - "xml_sha256": "68e98a6b0b204cfe" - }, - { - "index": 4, - "source": "118-hr-8752/1", - "page": 57, - "expected_text": "PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS", - "xml_path": "tests/corpus/118-hr-8752/1_reported-in-house.xml", - "xml_sha256": "e4bc6b90eeff87d0" - }, - { - "index": 5, - "source": "118-hr-8752/1", - "page": 15, - "expected_text": "PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS", - "xml_path": "tests/corpus/118-hr-8752/1_reported-in-house.xml", - "xml_sha256": "e4bc6b90eeff87d0" - }, - { - "index": 6, - "source": "114-hr-2029/4", - "page": 94, - "expected_text": "VETERANS HOUSING BENEFIT PROGRAM FUND", - "xml_path": "tests/corpus/114-hr-2029/4_reported-in-senate.xml", - "xml_sha256": "68e98a6b0b204cfe" - }, - { - "index": 7, - "source": "118-hr-8752/1", - "page": 11, - "expected_text": "(INCLUDING TRANSFER OF FUNDS)", - "xml_path": "tests/corpus/118-hr-8752/1_reported-in-house.xml", - "xml_sha256": "e4bc6b90eeff87d0" - } - ], - "nc": [ - { - "index": 0, - "generated_sha256": "89c88afa4c4e941e", - "expected": [] - }, - { - "index": 1, - "generated_sha256": "1a43384798a0d059", - "expected": [] - }, - { - "index": 2, - "generated_sha256": "14a0d4bc036147c7", - "expected": [] - }, - { - "index": 3, - "generated_sha256": "e9a091786d020f59", - "expected": [] - } - ] - }, - "pdf_id": { - "ids": { - "na_00_delete_one_word.pdf": "[<3326DFA53C91ABC448E4436423FB1E2C><3326DFA53C91ABC448E4436423FB1E2C>]", - "na_01_weld_two_words.pdf": "[]", - "na_02_split_one_word.pdf": "[<218DD759D47A3A1CECA6EE25C3E76F00><218DD759D47A3A1CECA6EE25C3E76F00>]", - "na_03_delete_one_word.pdf": "[]", - "na_04_weld_two_words.pdf": "[]", - "na_05_split_one_word.pdf": "[]", - "na_06_delete_one_word.pdf": "[<81D9ED22EAE74C1BF3808CE6DE961A80><81D9ED22EAE74C1BF3808CE6DE961A80>]", - "na_07_weld_two_words.pdf": "[<8DA8ECEDB14A4EAB24AAF7B09887595E><8DA8ECEDB14A4EAB24AAF7B09887595E>]", - "nc_00_heading_free.pdf": "[<46990A4C37F9A6DF4AAD1EC8313C5EDC><46990A4C37F9A6DF4AAD1EC8313C5EDC>]", - "nc_01_heading_free.pdf": "[]", - "nc_02_heading_free.pdf": "[]", - "nc_03_heading_free.pdf": "[]" - }, - "selection_stable_under_rename": true - }, - "f3_f4_replay": { - "population": 96, - "na_eligible": 83, - "nb_eligible": 96, - "expectations": [ - { - "schedule_index": 0, - "variant": "DELETE_ONE_WORD", - "expected_before": "RESEARCH AND DEVELOPMENT", - "expected_after": "RESEARCH AND" - }, - { - "schedule_index": 1, - "variant": "WELD_TWO_WORDS", - "expected_before": "(RESCISSIONS OF FUNDS)", - "expected_after": "(RESCISSIONSOF FUNDS)" - }, - { - "schedule_index": 2, - "variant": "SPLIT_ONE_WORD", - "expected_before": "(RESCISSION OF FUNDS)", - "expected_after": "(RESCI SSION OF FUNDS)" - }, - { - "schedule_index": 3, - "variant": "DELETE_ONE_WORD", - "expected_before": "OPERATIONS AND SUPPORT", - "expected_after": "AND SUPPORT" - }, - { - "schedule_index": 4, - "variant": "WELD_TWO_WORDS", - "expected_before": "VETERANS INSURANCE AND INDEMNITIES", - "expected_after": "VETERANSINSURANCE AND INDEMNITIES" - }, - { - "schedule_index": 5, - "variant": "SPLIT_ONE_WORD", - "expected_before": "OPERATIONS AND SUPPORT", - "expected_after": "OPERA TIONS AND SUPPORT" - }, - { - "schedule_index": 6, - "variant": "DELETE_ONE_WORD", - "expected_before": "COMPENSATION AND PENSIONS", - "expected_after": "AND PENSIONS" - }, - { - "schedule_index": 7, - "variant": "WELD_TWO_WORDS", - "expected_before": "OPERATIONS AND SUPPORT", - "expected_after": "OPERATIONSAND SUPPORT" - } - ], - "f3_negatives": { - "A selected N-A replaced by another ELIGIBLE N-A": true, - "B selected N-B replaced by another ELIGIBLE N-B": true, - "C same text, wrong XML structural identity": true, - "D same text, wrong physical occurrence": true, - "E 8/8 counts kept, one source substituted": true, - "F right members, deterministic order broken": true - }, - "f4_alternative_live": { - "DELETE_ONE_WORD": true, - "WELD_TWO_WORDS": true, - "SPLIT_ONE_WORD": true - }, - "f4_rejected": { - "DELETE_ONE_WORD": true, - "WELD_TWO_WORDS": true, - "SPLIT_ONE_WORD": true - } - }, - "g6_falsifications": { - "sabotaged_g6": { - "defects": [], - "allocation": { - "DELETE_ONE_WORD": 3, - "WELD_TWO_WORDS": 3, - "SPLIT_ONE_WORD": 2 - } - }, - "sabotage_live": { - "run_hybrid.run": true, - "run_extended.run": true, - "pdf_anchors.extract_anchors": true - }, - "stale_x26": [ - "MUTATION_INPUT_MISMATCH", - "ORACLE_EVIDENCE_STALE" - ], - "fresh_x26_wrong_truth": [ - "MUTATION_INPUT_MISMATCH" - ] - }, - "nc_rendered_distinctness": { - "rendered_png_sha256": [ - "ce7aefae40978180bfe7e81310238808657ec42b8845b0d6c81160fdcc309596", - "ebd2572ef7032604dd88ab13cc8aef6364517ff9c97f7a1062172d37f107e2d3", - "bb14d96a3c6ddc9a40a7b88bac2f4323b8da4c187bacd335d6f960e36b243b23", - "d62985083b9d7b5757d52bb1750b0db40aeb0afd402ad4865c8393fcc974faa2" - ], - "container_sha256": [ - "89c88afa4c4e941e", - "1a43384798a0d059", - "14a0d4bc036147c7", - "e9a091786d020f59" - ], - "distinct_rendered": 4 - }, - "reproducibility": { - "n_generated": 12 - }, - "holdout_identity_guard": { - "n_holdout_source_sha256": 17, - "counterfactual_reasons": [ - "HOLDOUT_SOURCE_IDENTITY", - "ORACLE_EVIDENCE_STALE", - "STALE_SHA256" - ] - }, - "g6": { - "negatives": { - "N-A count wrong": { - "expected": "WRONG_CONTROL_COUNT", - "fired": true - }, - "N-B count wrong": { - "expected": "WRONG_CONTROL_COUNT", - "fired": true - }, - "N-C count wrong": { - "expected": "WRONG_CONTROL_COUNT", - "fired": true - }, - "N-A allocation wrong": { - "expected": "WRONG_NA_ALLOCATION", - "fired": true - }, - "RETIRED size variant reintroduced": { - "expected": "RETIRED_VARIANT_PRESENT", - "fired": true - }, - "N-A mutation before == after": { - "expected": "DEAD_OR_MISCLASSIFIED_MUTATION", - "fired": true - }, - "WELD changes non-space characters": { - "expected": "DEAD_OR_MISCLASSIFIED_MUTATION", - "fired": true - }, - "SPLIT changes non-space characters": { - "expected": "DEAD_OR_MISCLASSIFIED_MUTATION", - "fired": true - }, - "WELD moves more than one boundary": { - "expected": "DEAD_OR_MISCLASSIFIED_MUTATION", - "fired": true - }, - "SPLIT moves a boundary the wrong way": { - "expected": "DEAD_OR_MISCLASSIFIED_MUTATION", - "fired": true - }, - "stale source SHA": { - "expected": "STALE_SHA256", - "fired": true - }, - "stale generated PDF SHA": { - "expected": "STALE_SHA256", - "fired": true - }, - "duplicate control identity": { - "expected": "DUPLICATE_CONTROL_IDENTITY", - "fired": true - }, - "missing expected truth": { - "expected": "MISSING_EXPECTED_TRUTH", - "fired": true - }, - "N-B missing XML corroboration": { - "expected": "MISSING_XML_CORROBORATION", - "fired": true - }, - "N-C expected headings non-empty": { - "expected": "NC_EXPECTED_HEADINGS_NOT_EMPTY", - "fired": true - }, - "holdout provenance in the chain": { - "expected": "HOLDOUT_PROVENANCE", - "fired": true - }, - "mutation recipe disagrees with variant": { - "expected": "MUTATION_RECIPE_VARIANT_MISMATCH", - "fired": true - } - } - }, - "tests": [ - { - "test": "DELETE changes the NON-SPACE character sequence", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "deleting a word leaves the non-space sequence intact, i.e. nothing was removed" - }, - { - "test": "WELD leaves non-space identical and removes exactly ONE boundary (1 -> 0)", - "expected": [ - true, - true, - [ - [ - 1, - 0 - ] - ] - ], - "observed": [ - true, - true, - [ - [ - 1, - 0 - ] - ] - ], - "pass": true, - "fails_when": "weld changed a character, or moved more than one boundary -- it would then be a content corruption rather than the missing-boundary class M3 distinguishes" - }, - { - "test": "SPLIT leaves non-space identical and inserts exactly ONE boundary (0 -> 1)", - "expected": [ - true, - true, - [ - [ - 0, - 1 - ] - ] - ], - "observed": [ - true, - true, - [ - [ - 0, - 1 - ] - ] - ], - "pass": true, - "fails_when": "split changed a character, or moved more than one boundary" - }, - { - "test": "SPLIT is the exact complement of WELD", - "expected": [ - [ - [ - 0, - 1 - ] - ], - [ - [ - 1, - 0 - ] - ] - ], - "observed": [ - [ - [ - 0, - 1 - ] - ], - [ - [ - 1, - 0 - ] - ] - ], - "pass": true, - "fails_when": "the two classes are not complementary, so N-A no longer spans both M3 directions" - }, - { - "test": "the SPLIT example matches the A40.2 illustration", - "expected": "SALA RIES AND EXPENSES", - "observed": "SALA RIES AND EXPENSES", - "pass": true, - "fails_when": "the frozen floor(len/2) target produces a different string than A40.2 specifies" - }, - { - "test": "NEGATIVE -- a DEAD mutation (before == after) is not live", - "expected": [ - false, - false - ], - "observed": [ - false, - false - ], - "pass": true, - "fails_when": "an unchanged string is accepted as a mutation, which is exactly the A39.3 failure the size control was retired for" - }, - { - "test": "NEGATIVE -- the RETIRED size variant can never be live", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "PULL_HEADING_TO_BODY_SIZE is still treated as a valid mutation class" - }, - { - "test": "both SPLIT pieces are at least 3 characters", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "a split leaves a 1-2 character fragment, which is a different failure mode" - }, - { - "test": "the manifest holds exactly 8 N-A, 8 N-B, 4 N-C", - "expected": { - "N-A": 8, - "N-B": 8, - "N-C": 4 - }, - "observed": { - "N-A": 8, - "N-B": 8, - "N-C": 4 - }, - "pass": true, - "fails_when": "a control class is under- or over-populated, so a Rule 3 blocker is not satisfied" - }, - { - "test": "N-A realizes the frozen 3 / 3 / 2 allocation", - "expected": { - "DELETE_ONE_WORD": 3, - "WELD_TWO_WORDS": 3, - "SPLIT_ONE_WORD": 2 - }, - "observed": { - "DELETE_ONE_WORD": 3, - "WELD_TWO_WORDS": 3, - "SPLIT_ONE_WORD": 2 - }, - "pass": true, - "fails_when": "the index-mod-3 schedule was not applied, so the three M3 directions are not balanced as A40.1 freezes them" - }, - { - "test": "every N-A mutation is LIVE on the realized fixtures", - "expected": [ - 8, - true - ], - "observed": [ - 8, - true - ], - "pass": true, - "fails_when": "a realized control cannot fire, so it reports coverage the study does not have" - }, - { - "test": "the RETIRED variant appears nowhere in the realized set", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "PULL_HEADING_TO_BODY_SIZE survived the A40 retirement" - }, - { - "test": "every N-C expects EXACTLY no headings", - "expected": [ - [], - [], - [], - [] - ], - "observed": [ - [], - [], - [], - [] - ], - "pass": true, - "fails_when": "a constructionally heading-free region expects a heading, so the over-triggering control could never fail" - }, - { - "test": "every N-A and N-B carries paired GPO XML corroboration", - "expected": [ - 16, - 16 - ], - "observed": [ - 16, - 16 - ], - "pass": true, - "fails_when": "a control's truth rests on something other than the independent source -- in the worst case on H or X's own output" - }, - { - "test": "all 20 control identities are unique", - "expected": 20, - "observed": 20, - "pass": true, - "fails_when": "two controls share an identity, so one would silently overwrite the other" - }, - { - "test": "every generated N-A prints its mutation in the TARGET REGION exactly once", - "expected": [ - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1 - ], - "observed": [ - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1 - ], - "pass": true, - "fails_when": "the mutation did not reach the adjudicated region, so the stimulus would not contain the alteration the control exists to test" - }, - { - "test": "...and the ORIGINAL heading is absent from that TARGET REGION", - "expected": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ], - "observed": [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ], - "pass": true, - "fails_when": "the unmutated heading survives inside the crop, so the region shows both and the control cannot distinguish an oracle that saw the alteration" - }, - { - "test": "DIAGNOSTIC -- some N-A pages legitimately repeat the original heading OUTSIDE the crop", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no fixture exercises the duplicate-elsewhere case, so the occurrence-scoped rule is indistinguishable from the page-scoped one it replaced" - }, - { - "test": "NEGATIVE A -- an original heading INSIDE the target region is detected", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "an unmutated heading inside the crop goes unnoticed, so the region rule cannot fail" - }, - { - "test": "NEGATIVE B -- a mutation absent from the target region is detected", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "the mutation counts even when it lies outside the crop, so the rule would pass a stimulus that shows the adjudicator nothing" - }, - { - "test": "POSITIVE -- a fixture whose original repeats ELSEWHERE on the page still passes", - "expected": [ - 1, - 0, - true - ], - "observed": [ - 1, - 0, - true - ], - "pass": true, - "fails_when": "the rule still gates on an unrelated duplicate outside the selected region, which is the scope error A40.13 corrects" - }, - { - "test": "every generated fixture carries the id its own basename determines", - "expected": { - "na_00_delete_one_word.pdf": "[<3326DFA53C91ABC448E4436423FB1E2C><3326DFA53C91ABC448E4436423FB1E2C>]", - "na_01_weld_two_words.pdf": "[]", - "na_02_split_one_word.pdf": "[<218DD759D47A3A1CECA6EE25C3E76F00><218DD759D47A3A1CECA6EE25C3E76F00>]", - "na_03_delete_one_word.pdf": "[]", - "na_04_weld_two_words.pdf": "[]", - "na_05_split_one_word.pdf": "[]", - "na_06_delete_one_word.pdf": "[<81D9ED22EAE74C1BF3808CE6DE961A80><81D9ED22EAE74C1BF3808CE6DE961A80>]", - "na_07_weld_two_words.pdf": "[<8DA8ECEDB14A4EAB24AAF7B09887595E><8DA8ECEDB14A4EAB24AAF7B09887595E>]", - "nc_00_heading_free.pdf": "[<46990A4C37F9A6DF4AAD1EC8313C5EDC><46990A4C37F9A6DF4AAD1EC8313C5EDC>]", - "nc_01_heading_free.pdf": "[]", - "nc_02_heading_free.pdf": "[]", - "nc_03_heading_free.pdf": "[]" - }, - "observed": { - "na_00_delete_one_word.pdf": "[<3326DFA53C91ABC448E4436423FB1E2C><3326DFA53C91ABC448E4436423FB1E2C>]", - "na_01_weld_two_words.pdf": "[]", - "na_02_split_one_word.pdf": "[<218DD759D47A3A1CECA6EE25C3E76F00><218DD759D47A3A1CECA6EE25C3E76F00>]", - "na_03_delete_one_word.pdf": "[]", - "na_04_weld_two_words.pdf": "[]", - "na_05_split_one_word.pdf": "[]", - "na_06_delete_one_word.pdf": "[<81D9ED22EAE74C1BF3808CE6DE961A80><81D9ED22EAE74C1BF3808CE6DE961A80>]", - "na_07_weld_two_words.pdf": "[<8DA8ECEDB14A4EAB24AAF7B09887595E><8DA8ECEDB14A4EAB24AAF7B09887595E>]", - "nc_00_heading_free.pdf": "[<46990A4C37F9A6DF4AAD1EC8313C5EDC><46990A4C37F9A6DF4AAD1EC8313C5EDC>]", - "nc_01_heading_free.pdf": "[]", - "nc_02_heading_free.pdf": "[]", - "nc_03_heading_free.pdf": "[]" - }, - "pass": true, - "fails_when": "a generated fixture's trailer id is not the deterministic one, so its bytes are not reproducible from the committed inputs" - }, - { - "test": "all generated trailer ids are DISTINCT", - "expected": 12, - "observed": 12, - "pass": true, - "fails_when": "two fixtures share a PDF identity" - }, - { - "test": "a renamed copy would carry a DIFFERENT id (so the counterfactual is real)", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the rename does not change the id, so this proves nothing about /ID being ignored" - }, - { - "test": "...and neither source selection nor control identity moves", - "expected": [ - [ - "[\"xml-account\",\"e4bc6b90eeff87d05eaa5ba61792c8a3b24aebcb86e5d6c98bb03de839be536d\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"68e98a6b0b204cfec7b0d79fdd8e42184fd72cc2872ce93facf6907c1bcbb1a8\",\"id7B6E2DF0F81A4AFDAAFD9EE47559AA49\",96,126]", - "[\"xml-account\",\"68e98a6b0b204cfec7b0d79fdd8e42184fd72cc2872ce93facf6907c1bcbb1a8\",\"id6F67AFBC04BD41B4BB87288874CC1F2D\",5,21]", - "[\"xml-account\",\"e4bc6b90eeff87d05eaa5ba61792c8a3b24aebcb86e5d6c98bb03de839be536d\",\"H7BD885E459AC4C37A68D0646B2B1789D\",35,58]", - "[\"xml-account\",\"68e98a6b0b204cfec7b0d79fdd8e42184fd72cc2872ce93facf6907c1bcbb1a8\",\"H8F1342C5C10F4399BBE581A2248B993A\",11,25]", - "[\"xml-account\",\"e4bc6b90eeff87d05eaa5ba61792c8a3b24aebcb86e5d6c98bb03de839be536d\",\"HC308222694524A38859E317746AEB753\",29,56]", - "[\"xml-account\",\"68e98a6b0b204cfec7b0d79fdd8e42184fd72cc2872ce93facf6907c1bcbb1a8\",\"H28C2B4A56282401B89ABB1F6D6EF0496\",8,23]", - "[\"xml-account\",\"e4bc6b90eeff87d05eaa5ba61792c8a3b24aebcb86e5d6c98bb03de839be536d\",\"H3AAFBF6B6E5E4300A3FA053B7EB13F1A\",23,44]" - ], - [ - "[\"control\",\"N-A\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",126,1,\"WELD_TWO_WORDS\"]", - "[\"control\",\"N-A\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",21,2,\"SPLIT_ONE_WORD\"]", - "[\"control\",\"N-A\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",23,6,\"DELETE_ONE_WORD\"]", - "[\"control\",\"N-A\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",25,4,\"WELD_TWO_WORDS\"]", - "[\"control\",\"N-A\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",20,0,\"DELETE_ONE_WORD\"]", - "[\"control\",\"N-A\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",44,7,\"WELD_TWO_WORDS\"]", - "[\"control\",\"N-A\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",56,5,\"SPLIT_ONE_WORD\"]", - "[\"control\",\"N-A\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",58,3,\"DELETE_ONE_WORD\"]", - "[\"control\",\"N-B\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",21,3,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",57,0,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",60,2,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",94,6,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",11,7,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",15,5,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",49,1,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",57,4,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-C\",\"14a0d4bc036147c7113f9ca2d5b8b77868112b2db725b6a456355f8b1b0abe17\",1,2,\"SYNTHETIC_HEADING_FREE\"]", - "[\"control\",\"N-C\",\"1a43384798a0d0595cefb1628bf092c867efa109da82c7d87c48fa3ef45f501d\",1,1,\"SYNTHETIC_HEADING_FREE\"]", - "[\"control\",\"N-C\",\"89c88afa4c4e941e2f99eab249212fc607b348ff34326debe668614f05678e24\",1,0,\"SYNTHETIC_HEADING_FREE\"]", - "[\"control\",\"N-C\",\"e9a091786d020f59edf0c959a79e4c0ddae3fa59dee139b54e77693571108c94\",1,3,\"SYNTHETIC_HEADING_FREE\"]" - ] - ], - "observed": [ - [ - "[\"xml-account\",\"e4bc6b90eeff87d05eaa5ba61792c8a3b24aebcb86e5d6c98bb03de839be536d\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"68e98a6b0b204cfec7b0d79fdd8e42184fd72cc2872ce93facf6907c1bcbb1a8\",\"id7B6E2DF0F81A4AFDAAFD9EE47559AA49\",96,126]", - "[\"xml-account\",\"68e98a6b0b204cfec7b0d79fdd8e42184fd72cc2872ce93facf6907c1bcbb1a8\",\"id6F67AFBC04BD41B4BB87288874CC1F2D\",5,21]", - "[\"xml-account\",\"e4bc6b90eeff87d05eaa5ba61792c8a3b24aebcb86e5d6c98bb03de839be536d\",\"H7BD885E459AC4C37A68D0646B2B1789D\",35,58]", - "[\"xml-account\",\"68e98a6b0b204cfec7b0d79fdd8e42184fd72cc2872ce93facf6907c1bcbb1a8\",\"H8F1342C5C10F4399BBE581A2248B993A\",11,25]", - "[\"xml-account\",\"e4bc6b90eeff87d05eaa5ba61792c8a3b24aebcb86e5d6c98bb03de839be536d\",\"HC308222694524A38859E317746AEB753\",29,56]", - "[\"xml-account\",\"68e98a6b0b204cfec7b0d79fdd8e42184fd72cc2872ce93facf6907c1bcbb1a8\",\"H28C2B4A56282401B89ABB1F6D6EF0496\",8,23]", - "[\"xml-account\",\"e4bc6b90eeff87d05eaa5ba61792c8a3b24aebcb86e5d6c98bb03de839be536d\",\"H3AAFBF6B6E5E4300A3FA053B7EB13F1A\",23,44]" - ], - [ - "[\"control\",\"N-A\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",126,1,\"WELD_TWO_WORDS\"]", - "[\"control\",\"N-A\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",21,2,\"SPLIT_ONE_WORD\"]", - "[\"control\",\"N-A\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",23,6,\"DELETE_ONE_WORD\"]", - "[\"control\",\"N-A\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",25,4,\"WELD_TWO_WORDS\"]", - "[\"control\",\"N-A\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",20,0,\"DELETE_ONE_WORD\"]", - "[\"control\",\"N-A\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",44,7,\"WELD_TWO_WORDS\"]", - "[\"control\",\"N-A\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",56,5,\"SPLIT_ONE_WORD\"]", - "[\"control\",\"N-A\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",58,3,\"DELETE_ONE_WORD\"]", - "[\"control\",\"N-B\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",21,3,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",57,0,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",60,2,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"7e3786d7abb1c54c030903d5ea58471e7827877f4d3c92cae858a79972b56df2\",94,6,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",11,7,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",15,5,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",49,1,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-B\",\"d0751bfbe0e53cd89a1906fb5a720035eb4cb6fa69c0bbfbab024f801bc24ae0\",57,4,\"XML_CORROBORATED_HEADING\"]", - "[\"control\",\"N-C\",\"14a0d4bc036147c7113f9ca2d5b8b77868112b2db725b6a456355f8b1b0abe17\",1,2,\"SYNTHETIC_HEADING_FREE\"]", - "[\"control\",\"N-C\",\"1a43384798a0d0595cefb1628bf092c867efa109da82c7d87c48fa3ef45f501d\",1,1,\"SYNTHETIC_HEADING_FREE\"]", - "[\"control\",\"N-C\",\"89c88afa4c4e941e2f99eab249212fc607b348ff34326debe668614f05678e24\",1,0,\"SYNTHETIC_HEADING_FREE\"]", - "[\"control\",\"N-C\",\"e9a091786d020f59edf0c959a79e4c0ddae3fa59dee139b54e77693571108c94\",1,3,\"SYNTHETIC_HEADING_FREE\"]" - ] - ], - "pass": true, - "fails_when": "a PDF trailer id reached source selection or control identity, which would make provenance depend on a filename rather than on committed source/SHA identity" - }, - { - "test": "F3 -- the committed manifest AGREES with a full independent source replay", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the committed selection cannot be reproduced from the committed XML/PDF and the frozen constants, so either the manifest or the deterministic chain is wrong" - }, - { - "test": "F4 -- every N-A variant is derived from the frozen schedule, not read from the manifest", - "expected": [ - "DELETE_ONE_WORD", - "WELD_TWO_WORDS", - "SPLIT_ONE_WORD", - "DELETE_ONE_WORD", - "WELD_TWO_WORDS", - "SPLIT_ONE_WORD", - "DELETE_ONE_WORD", - "WELD_TWO_WORDS" - ], - "observed": [ - "DELETE_ONE_WORD", - "WELD_TWO_WORDS", - "SPLIT_ONE_WORD", - "DELETE_ONE_WORD", - "WELD_TWO_WORDS", - "SPLIT_ONE_WORD", - "DELETE_ONE_WORD", - "WELD_TWO_WORDS" - ], - "pass": true, - "fails_when": "the replay takes the variant from the record it is checking, so a re-scheduled manifest would validate against itself" - }, - { - "test": "F3 NEGATIVES -- every deterministically wrong manifest FAILS with its own reason", - "expected": { - "A selected N-A replaced by another ELIGIBLE N-A": true, - "B selected N-B replaced by another ELIGIBLE N-B": true, - "C same text, wrong XML structural identity": true, - "D same text, wrong physical occurrence": true, - "E 8/8 counts kept, one source substituted": true, - "F right members, deterministic order broken": true - }, - "observed": { - "A selected N-A replaced by another ELIGIBLE N-A": true, - "B selected N-B replaced by another ELIGIBLE N-B": true, - "C same text, wrong XML structural identity": true, - "D same text, wrong physical occurrence": true, - "E 8/8 counts kept, one source substituted": true, - "F right members, deterministic order broken": true - }, - "pass": true, - "fails_when": "a coherent but deterministically wrong manifest validates, which is exactly the false green F3 exists to remove" - }, - { - "test": "each ALTERNATIVE mutation is itself LIVE (else the negative proves nothing)", - "expected": { - "DELETE_ONE_WORD": true, - "WELD_TWO_WORDS": true, - "SPLIT_ONE_WORD": true - }, - "observed": { - "DELETE_ONE_WORD": true, - "WELD_TWO_WORDS": true, - "SPLIT_ONE_WORD": true - }, - "pass": true, - "fails_when": "the alternative is dead, so its rejection shows only that dead mutations fail -- which is a different control" - }, - { - "test": "F4 NEGATIVES -- a live but non-deterministic target is REJECTED", - "expected": { - "DELETE_ONE_WORD": true, - "WELD_TWO_WORDS": true, - "SPLIT_ONE_WORD": true - }, - "observed": { - "DELETE_ONE_WORD": true, - "WELD_TWO_WORDS": true, - "SPLIT_ONE_WORD": true - }, - "pass": true, - "fails_when": "a different-but-live mutation passes, so the deterministic target rule is not enforced" - }, - { - "test": "F4 NEGATIVE -- recipe metadata altered while expected_after stays correct is REJECTED", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the recipe is not compared, so the committed target rule is unverifiable" - }, - { - "test": "F4 NEGATIVE -- a SELF-CONSISTENT rewrite of before/after/recipe is REJECTED", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the replay takes expected_before from the manifest, so a record that rewrites its own input and recomputes everything from it validates against itself -- F4 would be self-validating rather than independent" - }, - { - "test": "a WARM replay cache would mask the sabotage entirely", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the cache does not short-circuit the replay, so this control no longer explains why the cache must be invalidated -- re-derive the sabotage control before trusting it" - }, - { - "test": "G6 validates with run_hybrid/run_extended/extract_anchors ALL raising", - "expected": [ - [], - 96, - 83, - 96, - 8, - 8, - { - "DELETE_ONE_WORD": 3, - "WELD_TWO_WORDS": 3, - "SPLIT_ONE_WORD": 2 - } - ], - "observed": [ - [], - 96, - 83, - 96, - 8, - 8, - { - "DELETE_ONE_WORD": 3, - "WELD_TWO_WORDS": 3, - "SPLIT_ONE_WORD": 2 - } - ], - "pass": true, - "fails_when": "an H, X or pdf_anchors dependency has entered the result-bearing G6 replay path, so control source truth would be derived from the architectures under test" - }, - { - "test": "every sabotaged entrypoint really raises when called", - "expected": { - "run_hybrid.run": true, - "run_extended.run": true, - "pdf_anchors.extract_anchors": true - }, - "observed": { - "run_hybrid.run": true, - "run_extended.run": true, - "pdf_anchors.extract_anchors": true - }, - "pass": true, - "fails_when": "a monkeypatch did not take, so the sabotage control passed because nothing was disabled" - }, - { - "test": "the mutated control state is still WELL-FORMED (so the negatives cannot pass for the wrong reason)", - "expected": [ - true, - true, - true - ], - "observed": [ - true, - true, - true - ], - "pass": true, - "fails_when": "the mutation broke a record's structure, so a rejection would demonstrate malformedness rather than the binding invariant under test" - }, - { - "test": "a valid-but-different control state is REJECTED while the old x26 evidence still says failures==[]", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "x26 evidence certifies a control/oracle state it was not produced for, which is the exact false green A40.16 closed -- the evidence would only be certifying itself" - }, - { - "test": "...and the untouched x26 artifact really did still report success", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the committed evidence already reported failures, so the staleness negative above is not isolating the input binding" - }, - { - "test": "even CURRENT x26 evidence cannot certify wrong N-B truth", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "N-B expected truth is not independently replayed from source, so a manifest and its evidence can agree with each other while both disagree with the actual source" - }, - { - "test": "the x26 evidence file is byte-restored", - "expected": "cc0fd95b152e1dd0b3c5ffd5f8415a514dbc9274597b376087018c82fdc09aee", - "observed": "cc0fd95b152e1dd0b3c5ffd5f8415a514dbc9274597b376087018c82fdc09aee", - "pass": true, - "fails_when": "the regression left the committed evidence modified, which would dirty the tree" - }, - { - "test": "and the authoritative state still validates cleanly", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the regression did not restore the state it mutated" - }, - { - "test": "the four N-C controls render to FOUR DISTINCT PNG hashes", - "expected": 4, - "observed": 4, - "pass": true, - "fails_when": "two heading-free controls are the same image, so the four-item N-C denominator is really fewer -- and container SHA uniqueness would have hidden it" - }, - { - "test": "...through the REAL build_oracle.render_region at the frozen primary DPI", - "expected": [ - 4, - true - ], - "observed": [ - 4, - true - ], - "pass": true, - "fails_when": "the distinctness was measured by some other path than the one that will render the actual stimulus" - }, - { - "test": "container SHAs are distinct too, but are NOT the evidence", - "expected": 4, - "observed": 4, - "pass": true, - "fails_when": "the generated files are byte-identical, which is a different defect" - }, - { - "test": "a clean rebuild reproduces IDENTICAL bytes for every generated fixture", - "expected": { - "na_00_delete_one_word.pdf": "cd1460a092dc44fc4cf2f7d33b4fd9ece0d21adf2f7f5d0fd1b83cbf037a522f", - "na_01_weld_two_words.pdf": "104f85729ac095d06e2a56e3d48bfcc30abe9f298b6c99bf98469c3c8d67c061", - "na_02_split_one_word.pdf": "61c9178fc8c51ab4b81fc0c4a7b34303eaabbde27b9db39f59e8c8584642d7bd", - "na_03_delete_one_word.pdf": "b3f30e24b50424c3e8f634ddfc501f1029352d7a361fad26a1c71831ec167469", - "na_04_weld_two_words.pdf": "d5f67bb44f8490cea2e9d3330cad0b373b7a0253d5f4f8179ff3892d820202ac", - "na_05_split_one_word.pdf": "1bc08acb57b17f1220ae35046e931fac73647e46f41bf7da14f1428c9e9da45e", - "na_06_delete_one_word.pdf": "7f0fddf896985ff90a9a9fea34593b9cda3022c4cb86d5f8bdbf3740ca73eb4c", - "na_07_weld_two_words.pdf": "ce75db58bfc020fb02e20f74eb827f77a8f2f90580ccb3f7900eb31e6f298b28", - "nc_00_heading_free.pdf": "89c88afa4c4e941e2f99eab249212fc607b348ff34326debe668614f05678e24", - "nc_01_heading_free.pdf": "1a43384798a0d0595cefb1628bf092c867efa109da82c7d87c48fa3ef45f501d", - "nc_02_heading_free.pdf": "14a0d4bc036147c7113f9ca2d5b8b77868112b2db725b6a456355f8b1b0abe17", - "nc_03_heading_free.pdf": "e9a091786d020f59edf0c959a79e4c0ddae3fa59dee139b54e77693571108c94" - }, - "observed": { - "na_00_delete_one_word.pdf": "cd1460a092dc44fc4cf2f7d33b4fd9ece0d21adf2f7f5d0fd1b83cbf037a522f", - "na_01_weld_two_words.pdf": "104f85729ac095d06e2a56e3d48bfcc30abe9f298b6c99bf98469c3c8d67c061", - "na_02_split_one_word.pdf": "61c9178fc8c51ab4b81fc0c4a7b34303eaabbde27b9db39f59e8c8584642d7bd", - "na_03_delete_one_word.pdf": "b3f30e24b50424c3e8f634ddfc501f1029352d7a361fad26a1c71831ec167469", - "na_04_weld_two_words.pdf": "d5f67bb44f8490cea2e9d3330cad0b373b7a0253d5f4f8179ff3892d820202ac", - "na_05_split_one_word.pdf": "1bc08acb57b17f1220ae35046e931fac73647e46f41bf7da14f1428c9e9da45e", - "na_06_delete_one_word.pdf": "7f0fddf896985ff90a9a9fea34593b9cda3022c4cb86d5f8bdbf3740ca73eb4c", - "na_07_weld_two_words.pdf": "ce75db58bfc020fb02e20f74eb827f77a8f2f90580ccb3f7900eb31e6f298b28", - "nc_00_heading_free.pdf": "89c88afa4c4e941e2f99eab249212fc607b348ff34326debe668614f05678e24", - "nc_01_heading_free.pdf": "1a43384798a0d0595cefb1628bf092c867efa109da82c7d87c48fa3ef45f501d", - "nc_02_heading_free.pdf": "14a0d4bc036147c7113f9ca2d5b8b77868112b2db725b6a456355f8b1b0abe17", - "nc_03_heading_free.pdf": "e9a091786d020f59edf0c959a79e4c0ddae3fa59dee139b54e77693571108c94" - }, - "pass": true, - "fails_when": "a rebuild from identical inputs produces different bytes, so every committed hash goes stale on the next run and G6 cannot stay green" - }, - { - "test": "the fixtures actually carry a canonicalised PDF /ID", - "expected": [ - 12, - true - ], - "observed": [ - 12, - true - ], - "pass": true, - "fails_when": "a generated PDF has no /ID array, so the canonicalisation silently did nothing and reproducibility would depend on luck" - }, - { - "test": "the authoritative holdout SHA set is non-empty", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the membership manifest yielded no source hashes, so the identity guard would be vacuous and every control would pass it trivially" - }, - { - "test": "no committed control carries a holdout source identity", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a control's bytes are a confirmatory source" - }, - { - "test": "NEGATIVE -- a DEVELOPMENT-looking record carrying a holdout SHA is REJECTED", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "confirmatory bytes copied in under an innocuous document id and path pass the gate, because nothing about the bytes has to change to defeat a name-based scan" - }, - { - "test": "...and the name-based scan alone would NOT have caught it", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "the name scan happens to catch this case, so the identity check is not what is being demonstrated and the control proves less than it claims" - }, - { - "test": "the realized manifest carries NO defect except the declared-outstanding replays", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "either the validator rejects a correctly built fixture set, or a real defect is hiding among the outstanding-replay markers" - }, - { - "test": "G6 is green ONLY because both replays are actually implemented and pass", - "expected": [ - true, - true, - true - ], - "observed": [ - true, - true, - true - ], - "pass": true, - "fails_when": "either a replay is switched off while G6 still reports PASS -- the semantic false green A40.12 removed -- or the manifest no longer satisfies a replay that is switched on" - }, - { - "test": "EVERY injected defect makes G6 FAIL with its own reason", - "expected": { - "N-A count wrong": true, - "N-B count wrong": true, - "N-C count wrong": true, - "N-A allocation wrong": true, - "RETIRED size variant reintroduced": true, - "N-A mutation before == after": true, - "WELD changes non-space characters": true, - "SPLIT changes non-space characters": true, - "WELD moves more than one boundary": true, - "SPLIT moves a boundary the wrong way": true, - "stale source SHA": true, - "stale generated PDF SHA": true, - "duplicate control identity": true, - "missing expected truth": true, - "N-B missing XML corroboration": true, - "N-C expected headings non-empty": true, - "holdout provenance in the chain": true, - "mutation recipe disagrees with variant": true - }, - "observed": { - "N-A count wrong": true, - "N-B count wrong": true, - "N-C count wrong": true, - "N-A allocation wrong": true, - "RETIRED size variant reintroduced": true, - "N-A mutation before == after": true, - "WELD changes non-space characters": true, - "SPLIT changes non-space characters": true, - "WELD moves more than one boundary": true, - "SPLIT moves a boundary the wrong way": true, - "stale source SHA": true, - "stale generated PDF SHA": true, - "duplicate control identity": true, - "missing expected truth": true, - "N-B missing XML corroboration": true, - "N-C expected headings non-empty": true, - "holdout provenance in the chain": true, - "mutation recipe disagrees with variant": true - }, - "pass": true, - "fails_when": "an injected defect passes validation -- a green G6 would then certify a fixture set that cannot satisfy its Rule 3 blockers" - }, - { - "test": "NEGATIVE -- changing the generated PDF BYTES with a stale manifest hash FAILS", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the validator compares recorded strings rather than recomputing from the file, so a silently edited fixture would keep certifying" - }, - { - "test": "...and restoring the bytes clears the byte defect (only the outstanding replays remain)", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the byte-level control left the fixture set permanently invalid" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x24_xml_source_bridge.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x24_xml_source_bridge.json deleted file mode 100644 index 52a57327..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x24_xml_source_bridge.json +++ /dev/null @@ -1,790 +0,0 @@ -{ - "population": "DEVELOPMENT only -- no holdout opened, nothing adjudicated or scored", - "contract": "A40 F1/F2 -- independent account source truth and the physical bridge", - "schema": { - "schemas": { - "114-hr-2029/4": { - "doctype_root": "bill", - "public_id": "-//US Congress//DTDs/bill.dtd//EN", - "system_id": "bill.dtd", - "stylesheet": "billres.xsl" - }, - "118-hr-8752/1": { - "doctype_root": "bill", - "public_id": "-//US Congress//DTDs/bill.dtd//EN", - "system_id": "bill.dtd", - "stylesheet": "billres.xsl" - } - }, - "element_counts": { - "114-hr-2029/4": { - "account_elements": 0, - "appropriations_small": 107, - "header_attr": 0 - }, - "118-hr-8752/1": { - "account_elements": 0, - "appropriations_small": 44, - "header_attr": 0 - } - } - }, - "bridge": { - "per_document": { - "114-hr-2029/4": { - "xml_records": 105, - "paired": 55, - "order_agreement": { - "checks": [ - { - "scope": "unique_only", - "n": 1, - "n_inversions": 0, - "at": [] - }, - { - "scope": "all_paired", - "n": 55, - "n_inversions": 0, - "at": [] - } - ], - "n_unique": 1, - "n_paired": 55 - }, - "anchor_monotonicity": { - "n_anchors": 85, - "n_adjacent_pairs": 84, - "n_inversions": 0, - "at": [] - }, - "n_independent_anchors": 85, - "na_eligible": 45 - }, - "118-hr-8752/1": { - "xml_records": 41, - "paired": 41, - "order_agreement": { - "checks": [ - { - "scope": "unique_only", - "n": 8, - "n_inversions": 0, - "at": [] - }, - { - "scope": "all_paired", - "n": 41, - "n_inversions": 0, - "at": [] - } - ], - "n_unique": 8, - "n_paired": 41 - }, - "anchor_monotonicity": { - "n_anchors": 111, - "n_adjacent_pairs": 110, - "n_inversions": 0, - "at": [] - }, - "n_independent_anchors": 111, - "na_eligible": 38 - } - }, - "refusals": { - "114-hr-2029/4": [ - { - "reason": "UNDISCRIMINATED_GROUP", - "text": "(INCLUDING TRANSFER OF FUNDS)", - "xml_count": 38 - }, - { - "reason": "NO_PHYSICAL_OCCURRENCE", - "text": "GENERAL OPERATING EXPENSES, VETERANS BENEFITS ADMINISTRATION", - "xml_count": 2 - }, - { - "reason": "NO_PHYSICAL_OCCURRENCE", - "text": "GRANTS FOR CONSTRUCTION OF STATE EXTENDED CARE FACILITIES", - "xml_count": 2 - }, - { - "reason": "NO_PHYSICAL_OCCURRENCE", - "text": "NATIVE AMERICAN VETERAN HOUSING LOAN PROGRAM ACCOUNT", - "xml_count": 2 - }, - { - "reason": "UNDISCRIMINATED_GROUP", - "text": "SALARIES AND EXPENSES", - "xml_count": 6 - } - ], - "118-hr-8752/1": [] - } - }, - "architecture_disabled": { - "n_sources": 96, - "n_na_eligible": 83 - }, - "populations": { - "n_paired": 96, - "n_na_eligible": 83, - "na_selected": [ - { - "document": "114-hr-2029/4", - "element_id": "idF8D001406A6243A8838D80C7241CCBDF", - "page": 126, - "printed_text": "(RESCISSIONS OF FUNDS)" - }, - { - "document": "114-hr-2029/4", - "element_id": "idD50F9E5AE38746D68732969A822E264A", - "page": 127, - "printed_text": "(RESCISSIONS OF FUNDS)" - }, - { - "document": "114-hr-2029/4", - "element_id": "id391D86C18A674027A4F24ACA5B72BF63", - "page": 32, - "printed_text": "INFORMATION TECHNOLOGY SYSTEMS" - }, - { - "document": "118-hr-8752/1", - "element_id": "H678E045EEF614CC397A326B1375E18E1", - "page": 20, - "printed_text": "RESEARCH AND DEVELOPMENT" - }, - { - "document": "114-hr-2029/4", - "element_id": "H008CC077F54D4254A1DFF78AE4A1255C", - "page": 35, - "printed_text": "OFFICE OF INSPECTOR GENERAL" - }, - { - "document": "118-hr-8752/1", - "element_id": "HF913816B59C34861BBEC58B3ABC546D6", - "page": 17, - "printed_text": "PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS" - }, - { - "document": "118-hr-8752/1", - "element_id": "H84D3DE10957043D8AC503C0E976A7FD4", - "page": 11, - "printed_text": "OPERATIONS AND SUPPORT" - }, - { - "document": "114-hr-2029/4", - "element_id": "HE56DF44D7E454E9898F01E1D6652F69C", - "page": 101, - "printed_text": "INFORMATION TECHNOLOGY SYSTEMS" - } - ], - "nb_selected": [ - { - "document": "118-hr-8752/1", - "element_id": "H9C028C40025A4625B7711A1013A4D551", - "page": 49, - "printed_text": "NATIONAL FLOOD INSURANCE FUND" - }, - { - "document": "114-hr-2029/4", - "element_id": "H62F09D480AFC43CCA4A5A9A59E5731AF", - "page": 37, - "printed_text": "CONSTRUCTION, MINOR PROJECTS" - }, - { - "document": "114-hr-2029/4", - "element_id": "id3B3816AC51A84923ADA634D768D3CA49", - "page": 127, - "printed_text": "(RESCISSIONS OF FUNDS)" - }, - { - "document": "114-hr-2029/4", - "element_id": "id1298040CDE544515AA76CA08C86563B1", - "page": 90, - "printed_text": "(RESCISSIONS OF FUNDS)" - }, - { - "document": "114-hr-2029/4", - "element_id": "HF21CCD23CFCC4D1F94D0EDF7ED2B2E2B", - "page": 28, - "printed_text": "MEDICAL SUPPORT AND COMPLIANCE" - }, - { - "document": "114-hr-2029/4", - "element_id": "HE56DF44D7E454E9898F01E1D6652F69C", - "page": 101, - "printed_text": "INFORMATION TECHNOLOGY SYSTEMS" - }, - { - "document": "118-hr-8752/1", - "element_id": "H64B67A103A1B4FF3981C2A3176C818CA", - "page": 95, - "printed_text": "(RESCISSIONS OF FUNDS)" - }, - { - "document": "118-hr-8752/1", - "element_id": "H3AAFBF6B6E5E4300A3FA053B7EB13F1A", - "page": 44, - "printed_text": "OPERATIONS AND SUPPORT" - } - ] - }, - "stop_conditions": [], - "tests": [ - { - "test": "both DEVELOPMENT sources declare the legacy US Congress bill DTD", - "expected": { - "114-hr-2029/4": "-//US Congress//DTDs/bill.dtd//EN", - "118-hr-8752/1": "-//US Congress//DTDs/bill.dtd//EN" - }, - "observed": { - "114-hr-2029/4": "-//US Congress//DTDs/bill.dtd//EN", - "118-hr-8752/1": "-//US Congress//DTDs/bill.dtd//EN" - }, - "pass": true, - "fails_when": "the corpus uses a different schema, so the structural predicate below would be asserted about elements this DTD does not define" - }, - { - "test": "this corpus has NO element -- the account level is appropriations-small", - "expected": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "observed": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "pass": true, - "fails_when": "an element exists, so the newer schema's structure should be used instead and the legacy predicate would be the wrong one" - }, - { - "test": "the header is a CHILD ELEMENT here, never the attribute the prose examples abbreviate", - "expected": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "observed": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "pass": true, - "fails_when": "some headers are attributes, so a child-only reader silently truncates the population" - }, - { - "test": "a schema mismatch REFUSES rather than being read anyway", - "expected": "UNEXPECTED_SCHEMA", - "observed": "UNEXPECTED_SCHEMA", - "pass": true, - "fails_when": "an unexpected DTD is parsed as though it were the legacy bill DTD" - }, - { - "test": "CONSISTENCY ONLY -- the paired set carries no order inversion", - "expected": { - "114-hr-2029/4": [ - 0, - 0 - ], - "118-hr-8752/1": [ - 0, - 0 - ] - }, - "observed": { - "114-hr-2029/4": [ - 0, - 0 - ], - "118-hr-8752/1": [ - 0, - 0 - ] - }, - "pass": true, - "fails_when": "the paired rows contradict themselves. NOTE this check CANNOT license the pairing rule: these rows were paired index-to-index, so a green result is guaranteed by construction. It is retained as a consistency assertion only" - }, - { - "test": "the LICENSING evidence is independent of the pairing -- anchors alone, zero inversions", - "expected": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "observed": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "pass": true, - "fails_when": "the independent anchors invert, so XML reading order does not correspond to physical print order and the ordinal pairing rule is unlicensed -- a STOP. This is the check the unique-occurrence subset (n=1 in 114-hr-2029/4) was far too small to make; x25 falsifies it in full, including the five bracketing negatives" - }, - { - "test": "NEGATIVE -- a contradicted physical ordering IS detected as an inversion", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the order check cannot go red, so its green result on the real material proves nothing" - }, - { - "test": "NEGATIVE -- an extra XML occurrence REFUSES the whole group on count mismatch", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "a count mismatch is paired anyway, so the k-to-k rule would silently mis-associate a heading with a different physical occurrence" - }, - { - "test": "...and the refusal removes the ENTIRE group, not just the surplus", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "only the extra record is dropped, leaving the rest of an unverifiable group paired. Asserted as a direct property of the paired set rather than as an arithmetic identity, which would silently absorb any other refusal into the expected count" - }, - { - "test": "the source population is BYTE-IDENTICAL with run_hybrid/run_extended/extract_anchors raising", - "expected": [ - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id2E62F0413D7047B9BDB72A5D3E8EBBAC\",4,20]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id6F67AFBC04BD41B4BB87288874CC1F2D\",5,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idAB8DD978E87E47C982A04ECE698F66E0\",6,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id56679E906A574CD2999B872083B585C8\",7,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H28C2B4A56282401B89ABB1F6D6EF0496\",8,23]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H07FB15160B4E4D139BA230212A101204\",10,24]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H8F1342C5C10F4399BBE581A2248B993A\",11,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HBE25A18341144E4F802D8580A83BE15F\",12,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCE84943256314714844B7C4206C9EEE5\",13,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HC5B6BEB95970464DADCC79D283BC0158\",15,26]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HF21CCD23CFCC4D1F94D0EDF7ED2B2E2B\",16,28]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H0099245F179749BB92D220FFD263FB4D\",17,28]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H13BC0C8B220248C8BD4DDF52FE406751\",18,29]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HF6B0DB6767D94CCDAFBCA28E60323FF4\",19,30]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H71312D9E29F2492B9345DDA6F99FFAD2\",21,30]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id391D86C18A674027A4F24ACA5B72BF63\",23,32]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H008CC077F54D4254A1DFF78AE4A1255C\",25,35]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4BDADBC74FFB479C81AFFA91CFDD2F76\",26,35]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H62F09D480AFC43CCA4A5A9A59E5731AF\",27,37]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H3CC412EC29904E5EB8F18BFC2F407552\",29,39]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H74590EC1B49A4428AA62A08A1857D2DB\",42,51]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id4C6564A2F4A94827823CC6AFC031ADF3\",43,54]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HA455255406D44BA0B7CAC5B48831100F\",46,57]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HDF3F475772C041D8B38DF22D1534D2F9\",47,57]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H14754A1F2E044DD183E15C482B43FB7C\",49,60]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H41B9065124E44D00801181944BDECF8E\",52,61]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H7F1E7E8A3597444BA02CDA4217A3BE59\",53,67]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id1298040CDE544515AA76CA08C86563B1\",58,90]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id3F0DC73C9BDA4734B039E10767E365EF\",59,91]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H406FDAD023DF45C8A71E821AA6CDDD65\",60,92]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB793FFF2BB814E11AE00B5FF65103518\",62,93]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB092A1529E86427E980461688FF15EA4\",63,94]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H50AA9DF7CAF546EBB565A12B7E32ED25\",64,94]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4512476C89E14D819E4B1CCE1F57B5D6\",65,95]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HA6965BB588BA4C83AE702B3512EBA8B9\",67,95]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCB2B0B1C3CCD4900BD43FA815951E9F8\",68,97]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4CDA12D59AF4430EA69F8B2B01F46CCA\",69,98]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HD702CA5401B349E4B92AD8CDD1F6CEAD\",70,99]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H9C7007EC424F4A35907F4C5E62077861\",71,100]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idFC412A773B43411CAC7AA8BCBA7CA19D\",73,100]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE56DF44D7E454E9898F01E1D6652F69C\",75,101]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE030FC976E27419DA53A4669DE517999\",76,105]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H327ACD92E67E49FB93183EAF25B2F05C\",77,105]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB19A7BF7DD2243B9A4BE34B229A2EC20\",78,108]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H39D1CF2F6DD04CC6BB36BD389058DBC7\",80,109]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id48E7E82AA89042BB8DE1A9F8ECB5A880\",85,115]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCE66C02D114A45CD9EA2E7A2B3D65ED3\",93,122]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H440502B7B451409093FE927A90D1A70F\",94,122]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idF8D001406A6243A8838D80C7241CCBDF\",95,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id7B6E2DF0F81A4AFDAAFD9EE47559AA49\",96,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idA6C2B7639F1D40B7913757096579EA7C\",97,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idD50F9E5AE38746D68732969A822E264A\",98,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id3B3816AC51A84923ADA634D768D3CA49\",99,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H506A309468FF4A949672831D165AC5E2\",101,132]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4BF55239BE2944269976B585A6286134\",104,133]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H83CABDD0A60844B294FF6036C9C5CF3A\",0,2]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H5A961F0C10704C7BB15820C290742502\",1,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF7492E96DF014286A8C83E8661FBD17E\",2,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H185E5D91987D4A259D17EAFA3851FF21\",3,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H32F6540DEBEB4D48B546B0F46135EEA3\",4,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6080F001A5054872962FE766EC12F560\",5,4]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H84D3DE10957043D8AC503C0E976A7FD4\",6,11]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC54243F7C7DF40AFB25964407D87BB96\",7,11]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HA698FC87624B49DE88184BC043AC28AF\",8,12]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H9F4C1B03650241FB8463F3834E89E0E2\",9,13]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H8F0E83D54B934790941808AEEC1D464C\",10,14]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H4AEC49930993478E946147D96784AF5D\",11,15]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HAF692D1DAD8A45858DCA0142AB484A68\",12,15]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HEEF1858D2D3047AD826AC348490319F2\",13,16]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF8F2FA7EE6C64C738DED07BDF48CFBA9\",14,16]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF913816B59C34861BBEC58B3ABC546D6\",15,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HA6ABB0CED01542A990738D3130890D40\",16,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HFBE37EBB40FD417B94BDC37CB0548640\",17,18]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H28885A070F614C7CB41CC2D9CD3C0F5E\",18,18]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HFA67B26D3E884932BAB1955052FFAB81\",19,20]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HCB53FDF8764F433A9D38B08EA43E0890\",21,43]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H5BFB8B5395E24CBEAD8F124C098937BF\",22,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H3AAFBF6B6E5E4300A3FA053B7EB13F1A\",23,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HDBFE999FE60A4A57A64943054E0B9738\",24,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6FE2F43D79C2428688CF3C8C77599852\",25,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6D1C524D551246108CDEE553D9EB01AD\",26,49]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H9C028C40025A4625B7711A1013A4D551\",27,49]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H49E2D7DADD4F40B9B9FBB488C4126775\",28,51]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC308222694524A38859E317746AEB753\",29,56]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HDE3E18C5FE1043DB94B3506799113DE3\",30,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H86A6F791079E4F6B9A1EBB805B81C8F9\",31,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H842FF1D33FDC40A489A55C6910F380E0\",32,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC9FF1E30713544B4A9EFDBC706429FF7\",33,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HD1B4F89AD7AB4440B85326DE0A3CD9FB\",34,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H7BD885E459AC4C37A68D0646B2B1789D\",35,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H3FD9EC7D9D28490D91467523E23886AA\",36,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HE85ADC06B8CE434FB2BAE4A632831143\",37,59]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H0C5B9BB52F06498DB7D4BB82D69D58E0\",38,59]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC79B53E84C144258A4477E1CD0311508\",39,64]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H64B67A103A1B4FF3981C2A3176C818CA\",40,95]" - ], - "observed": [ - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id2E62F0413D7047B9BDB72A5D3E8EBBAC\",4,20]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id6F67AFBC04BD41B4BB87288874CC1F2D\",5,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idAB8DD978E87E47C982A04ECE698F66E0\",6,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id56679E906A574CD2999B872083B585C8\",7,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H28C2B4A56282401B89ABB1F6D6EF0496\",8,23]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H07FB15160B4E4D139BA230212A101204\",10,24]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H8F1342C5C10F4399BBE581A2248B993A\",11,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HBE25A18341144E4F802D8580A83BE15F\",12,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCE84943256314714844B7C4206C9EEE5\",13,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HC5B6BEB95970464DADCC79D283BC0158\",15,26]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HF21CCD23CFCC4D1F94D0EDF7ED2B2E2B\",16,28]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H0099245F179749BB92D220FFD263FB4D\",17,28]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H13BC0C8B220248C8BD4DDF52FE406751\",18,29]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HF6B0DB6767D94CCDAFBCA28E60323FF4\",19,30]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H71312D9E29F2492B9345DDA6F99FFAD2\",21,30]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id391D86C18A674027A4F24ACA5B72BF63\",23,32]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H008CC077F54D4254A1DFF78AE4A1255C\",25,35]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4BDADBC74FFB479C81AFFA91CFDD2F76\",26,35]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H62F09D480AFC43CCA4A5A9A59E5731AF\",27,37]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H3CC412EC29904E5EB8F18BFC2F407552\",29,39]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H74590EC1B49A4428AA62A08A1857D2DB\",42,51]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id4C6564A2F4A94827823CC6AFC031ADF3\",43,54]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HA455255406D44BA0B7CAC5B48831100F\",46,57]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HDF3F475772C041D8B38DF22D1534D2F9\",47,57]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H14754A1F2E044DD183E15C482B43FB7C\",49,60]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H41B9065124E44D00801181944BDECF8E\",52,61]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H7F1E7E8A3597444BA02CDA4217A3BE59\",53,67]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id1298040CDE544515AA76CA08C86563B1\",58,90]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id3F0DC73C9BDA4734B039E10767E365EF\",59,91]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H406FDAD023DF45C8A71E821AA6CDDD65\",60,92]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB793FFF2BB814E11AE00B5FF65103518\",62,93]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB092A1529E86427E980461688FF15EA4\",63,94]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H50AA9DF7CAF546EBB565A12B7E32ED25\",64,94]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4512476C89E14D819E4B1CCE1F57B5D6\",65,95]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HA6965BB588BA4C83AE702B3512EBA8B9\",67,95]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCB2B0B1C3CCD4900BD43FA815951E9F8\",68,97]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4CDA12D59AF4430EA69F8B2B01F46CCA\",69,98]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HD702CA5401B349E4B92AD8CDD1F6CEAD\",70,99]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H9C7007EC424F4A35907F4C5E62077861\",71,100]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idFC412A773B43411CAC7AA8BCBA7CA19D\",73,100]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE56DF44D7E454E9898F01E1D6652F69C\",75,101]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE030FC976E27419DA53A4669DE517999\",76,105]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H327ACD92E67E49FB93183EAF25B2F05C\",77,105]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB19A7BF7DD2243B9A4BE34B229A2EC20\",78,108]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H39D1CF2F6DD04CC6BB36BD389058DBC7\",80,109]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id48E7E82AA89042BB8DE1A9F8ECB5A880\",85,115]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCE66C02D114A45CD9EA2E7A2B3D65ED3\",93,122]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H440502B7B451409093FE927A90D1A70F\",94,122]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idF8D001406A6243A8838D80C7241CCBDF\",95,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id7B6E2DF0F81A4AFDAAFD9EE47559AA49\",96,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idA6C2B7639F1D40B7913757096579EA7C\",97,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idD50F9E5AE38746D68732969A822E264A\",98,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id3B3816AC51A84923ADA634D768D3CA49\",99,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H506A309468FF4A949672831D165AC5E2\",101,132]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4BF55239BE2944269976B585A6286134\",104,133]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H83CABDD0A60844B294FF6036C9C5CF3A\",0,2]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H5A961F0C10704C7BB15820C290742502\",1,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF7492E96DF014286A8C83E8661FBD17E\",2,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H185E5D91987D4A259D17EAFA3851FF21\",3,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H32F6540DEBEB4D48B546B0F46135EEA3\",4,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6080F001A5054872962FE766EC12F560\",5,4]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H84D3DE10957043D8AC503C0E976A7FD4\",6,11]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC54243F7C7DF40AFB25964407D87BB96\",7,11]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HA698FC87624B49DE88184BC043AC28AF\",8,12]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H9F4C1B03650241FB8463F3834E89E0E2\",9,13]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H8F0E83D54B934790941808AEEC1D464C\",10,14]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H4AEC49930993478E946147D96784AF5D\",11,15]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HAF692D1DAD8A45858DCA0142AB484A68\",12,15]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HEEF1858D2D3047AD826AC348490319F2\",13,16]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF8F2FA7EE6C64C738DED07BDF48CFBA9\",14,16]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF913816B59C34861BBEC58B3ABC546D6\",15,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HA6ABB0CED01542A990738D3130890D40\",16,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HFBE37EBB40FD417B94BDC37CB0548640\",17,18]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H28885A070F614C7CB41CC2D9CD3C0F5E\",18,18]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HFA67B26D3E884932BAB1955052FFAB81\",19,20]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HCB53FDF8764F433A9D38B08EA43E0890\",21,43]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H5BFB8B5395E24CBEAD8F124C098937BF\",22,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H3AAFBF6B6E5E4300A3FA053B7EB13F1A\",23,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HDBFE999FE60A4A57A64943054E0B9738\",24,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6FE2F43D79C2428688CF3C8C77599852\",25,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6D1C524D551246108CDEE553D9EB01AD\",26,49]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H9C028C40025A4625B7711A1013A4D551\",27,49]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H49E2D7DADD4F40B9B9FBB488C4126775\",28,51]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC308222694524A38859E317746AEB753\",29,56]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HDE3E18C5FE1043DB94B3506799113DE3\",30,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H86A6F791079E4F6B9A1EBB805B81C8F9\",31,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H842FF1D33FDC40A489A55C6910F380E0\",32,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC9FF1E30713544B4A9EFDBC706429FF7\",33,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HD1B4F89AD7AB4440B85326DE0A3CD9FB\",34,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H7BD885E459AC4C37A68D0646B2B1789D\",35,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H3FD9EC7D9D28490D91467523E23886AA\",36,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HE85ADC06B8CE434FB2BAE4A632831143\",37,59]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H0C5B9BB52F06498DB7D4BB82D69D58E0\",38,59]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC79B53E84C144258A4477E1CD0311508\",39,64]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H64B67A103A1B4FF3981C2A3176C818CA\",40,95]" - ], - "pass": true, - "fails_when": "an architecture path contributes to source enumeration, which A40.3 forbids -- and would mean N-A/N-B truth is derived from the thing under test" - }, - { - "test": "...and so is N-A eligibility", - "expected": [ - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id2E62F0413D7047B9BDB72A5D3E8EBBAC\",4,20]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id6F67AFBC04BD41B4BB87288874CC1F2D\",5,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idAB8DD978E87E47C982A04ECE698F66E0\",6,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id56679E906A574CD2999B872083B585C8\",7,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H28C2B4A56282401B89ABB1F6D6EF0496\",8,23]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H8F1342C5C10F4399BBE581A2248B993A\",11,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HBE25A18341144E4F802D8580A83BE15F\",12,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCE84943256314714844B7C4206C9EEE5\",13,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HF21CCD23CFCC4D1F94D0EDF7ED2B2E2B\",16,28]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H13BC0C8B220248C8BD4DDF52FE406751\",18,29]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H71312D9E29F2492B9345DDA6F99FFAD2\",21,30]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id391D86C18A674027A4F24ACA5B72BF63\",23,32]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H008CC077F54D4254A1DFF78AE4A1255C\",25,35]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4BDADBC74FFB479C81AFFA91CFDD2F76\",26,35]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H62F09D480AFC43CCA4A5A9A59E5731AF\",27,37]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H3CC412EC29904E5EB8F18BFC2F407552\",29,39]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H74590EC1B49A4428AA62A08A1857D2DB\",42,51]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id4C6564A2F4A94827823CC6AFC031ADF3\",43,54]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HA455255406D44BA0B7CAC5B48831100F\",46,57]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HDF3F475772C041D8B38DF22D1534D2F9\",47,57]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H14754A1F2E044DD183E15C482B43FB7C\",49,60]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H7F1E7E8A3597444BA02CDA4217A3BE59\",53,67]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id1298040CDE544515AA76CA08C86563B1\",58,90]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id3F0DC73C9BDA4734B039E10767E365EF\",59,91]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H406FDAD023DF45C8A71E821AA6CDDD65\",60,92]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB092A1529E86427E980461688FF15EA4\",63,94]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H50AA9DF7CAF546EBB565A12B7E32ED25\",64,94]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4512476C89E14D819E4B1CCE1F57B5D6\",65,95]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCB2B0B1C3CCD4900BD43FA815951E9F8\",68,97]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HD702CA5401B349E4B92AD8CDD1F6CEAD\",70,99]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idFC412A773B43411CAC7AA8BCBA7CA19D\",73,100]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE56DF44D7E454E9898F01E1D6652F69C\",75,101]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE030FC976E27419DA53A4669DE517999\",76,105]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H327ACD92E67E49FB93183EAF25B2F05C\",77,105]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB19A7BF7DD2243B9A4BE34B229A2EC20\",78,108]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H39D1CF2F6DD04CC6BB36BD389058DBC7\",80,109]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id48E7E82AA89042BB8DE1A9F8ECB5A880\",85,115]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCE66C02D114A45CD9EA2E7A2B3D65ED3\",93,122]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H440502B7B451409093FE927A90D1A70F\",94,122]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idF8D001406A6243A8838D80C7241CCBDF\",95,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id7B6E2DF0F81A4AFDAAFD9EE47559AA49\",96,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idA6C2B7639F1D40B7913757096579EA7C\",97,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idD50F9E5AE38746D68732969A822E264A\",98,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id3B3816AC51A84923ADA634D768D3CA49\",99,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H506A309468FF4A949672831D165AC5E2\",101,132]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H83CABDD0A60844B294FF6036C9C5CF3A\",0,2]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H5A961F0C10704C7BB15820C290742502\",1,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF7492E96DF014286A8C83E8661FBD17E\",2,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H185E5D91987D4A259D17EAFA3851FF21\",3,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H32F6540DEBEB4D48B546B0F46135EEA3\",4,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6080F001A5054872962FE766EC12F560\",5,4]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H84D3DE10957043D8AC503C0E976A7FD4\",6,11]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC54243F7C7DF40AFB25964407D87BB96\",7,11]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HA698FC87624B49DE88184BC043AC28AF\",8,12]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H9F4C1B03650241FB8463F3834E89E0E2\",9,13]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H8F0E83D54B934790941808AEEC1D464C\",10,14]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H4AEC49930993478E946147D96784AF5D\",11,15]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HAF692D1DAD8A45858DCA0142AB484A68\",12,15]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HEEF1858D2D3047AD826AC348490319F2\",13,16]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF8F2FA7EE6C64C738DED07BDF48CFBA9\",14,16]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF913816B59C34861BBEC58B3ABC546D6\",15,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HA6ABB0CED01542A990738D3130890D40\",16,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H28885A070F614C7CB41CC2D9CD3C0F5E\",18,18]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HFA67B26D3E884932BAB1955052FFAB81\",19,20]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HCB53FDF8764F433A9D38B08EA43E0890\",21,43]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H5BFB8B5395E24CBEAD8F124C098937BF\",22,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H3AAFBF6B6E5E4300A3FA053B7EB13F1A\",23,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HDBFE999FE60A4A57A64943054E0B9738\",24,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6D1C524D551246108CDEE553D9EB01AD\",26,49]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H9C028C40025A4625B7711A1013A4D551\",27,49]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H49E2D7DADD4F40B9B9FBB488C4126775\",28,51]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC308222694524A38859E317746AEB753\",29,56]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HDE3E18C5FE1043DB94B3506799113DE3\",30,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H86A6F791079E4F6B9A1EBB805B81C8F9\",31,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H842FF1D33FDC40A489A55C6910F380E0\",32,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC9FF1E30713544B4A9EFDBC706429FF7\",33,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HD1B4F89AD7AB4440B85326DE0A3CD9FB\",34,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H7BD885E459AC4C37A68D0646B2B1789D\",35,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H3FD9EC7D9D28490D91467523E23886AA\",36,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HE85ADC06B8CE434FB2BAE4A632831143\",37,59]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC79B53E84C144258A4477E1CD0311508\",39,64]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H64B67A103A1B4FF3981C2A3176C818CA\",40,95]" - ], - "observed": [ - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id2E62F0413D7047B9BDB72A5D3E8EBBAC\",4,20]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id6F67AFBC04BD41B4BB87288874CC1F2D\",5,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idAB8DD978E87E47C982A04ECE698F66E0\",6,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id56679E906A574CD2999B872083B585C8\",7,21]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H28C2B4A56282401B89ABB1F6D6EF0496\",8,23]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H8F1342C5C10F4399BBE581A2248B993A\",11,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HBE25A18341144E4F802D8580A83BE15F\",12,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCE84943256314714844B7C4206C9EEE5\",13,25]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HF21CCD23CFCC4D1F94D0EDF7ED2B2E2B\",16,28]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H13BC0C8B220248C8BD4DDF52FE406751\",18,29]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H71312D9E29F2492B9345DDA6F99FFAD2\",21,30]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id391D86C18A674027A4F24ACA5B72BF63\",23,32]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H008CC077F54D4254A1DFF78AE4A1255C\",25,35]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4BDADBC74FFB479C81AFFA91CFDD2F76\",26,35]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H62F09D480AFC43CCA4A5A9A59E5731AF\",27,37]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H3CC412EC29904E5EB8F18BFC2F407552\",29,39]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H74590EC1B49A4428AA62A08A1857D2DB\",42,51]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id4C6564A2F4A94827823CC6AFC031ADF3\",43,54]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HA455255406D44BA0B7CAC5B48831100F\",46,57]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HDF3F475772C041D8B38DF22D1534D2F9\",47,57]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H14754A1F2E044DD183E15C482B43FB7C\",49,60]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H7F1E7E8A3597444BA02CDA4217A3BE59\",53,67]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id1298040CDE544515AA76CA08C86563B1\",58,90]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id3F0DC73C9BDA4734B039E10767E365EF\",59,91]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H406FDAD023DF45C8A71E821AA6CDDD65\",60,92]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB092A1529E86427E980461688FF15EA4\",63,94]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H50AA9DF7CAF546EBB565A12B7E32ED25\",64,94]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H4512476C89E14D819E4B1CCE1F57B5D6\",65,95]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCB2B0B1C3CCD4900BD43FA815951E9F8\",68,97]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HD702CA5401B349E4B92AD8CDD1F6CEAD\",70,99]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idFC412A773B43411CAC7AA8BCBA7CA19D\",73,100]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE56DF44D7E454E9898F01E1D6652F69C\",75,101]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE030FC976E27419DA53A4669DE517999\",76,105]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H327ACD92E67E49FB93183EAF25B2F05C\",77,105]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HB19A7BF7DD2243B9A4BE34B229A2EC20\",78,108]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H39D1CF2F6DD04CC6BB36BD389058DBC7\",80,109]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id48E7E82AA89042BB8DE1A9F8ECB5A880\",85,115]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HCE66C02D114A45CD9EA2E7A2B3D65ED3\",93,122]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H440502B7B451409093FE927A90D1A70F\",94,122]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idF8D001406A6243A8838D80C7241CCBDF\",95,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id7B6E2DF0F81A4AFDAAFD9EE47559AA49\",96,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idA6C2B7639F1D40B7913757096579EA7C\",97,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idD50F9E5AE38746D68732969A822E264A\",98,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id3B3816AC51A84923ADA634D768D3CA49\",99,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H506A309468FF4A949672831D165AC5E2\",101,132]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H83CABDD0A60844B294FF6036C9C5CF3A\",0,2]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H5A961F0C10704C7BB15820C290742502\",1,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF7492E96DF014286A8C83E8661FBD17E\",2,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H185E5D91987D4A259D17EAFA3851FF21\",3,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H32F6540DEBEB4D48B546B0F46135EEA3\",4,3]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6080F001A5054872962FE766EC12F560\",5,4]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H84D3DE10957043D8AC503C0E976A7FD4\",6,11]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC54243F7C7DF40AFB25964407D87BB96\",7,11]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HA698FC87624B49DE88184BC043AC28AF\",8,12]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H9F4C1B03650241FB8463F3834E89E0E2\",9,13]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H8F0E83D54B934790941808AEEC1D464C\",10,14]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H4AEC49930993478E946147D96784AF5D\",11,15]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HAF692D1DAD8A45858DCA0142AB484A68\",12,15]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HEEF1858D2D3047AD826AC348490319F2\",13,16]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF8F2FA7EE6C64C738DED07BDF48CFBA9\",14,16]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF913816B59C34861BBEC58B3ABC546D6\",15,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HA6ABB0CED01542A990738D3130890D40\",16,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H28885A070F614C7CB41CC2D9CD3C0F5E\",18,18]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HFA67B26D3E884932BAB1955052FFAB81\",19,20]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HCB53FDF8764F433A9D38B08EA43E0890\",21,43]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H5BFB8B5395E24CBEAD8F124C098937BF\",22,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H3AAFBF6B6E5E4300A3FA053B7EB13F1A\",23,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HDBFE999FE60A4A57A64943054E0B9738\",24,44]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H6D1C524D551246108CDEE553D9EB01AD\",26,49]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H9C028C40025A4625B7711A1013A4D551\",27,49]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H49E2D7DADD4F40B9B9FBB488C4126775\",28,51]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC308222694524A38859E317746AEB753\",29,56]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HDE3E18C5FE1043DB94B3506799113DE3\",30,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H86A6F791079E4F6B9A1EBB805B81C8F9\",31,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H842FF1D33FDC40A489A55C6910F380E0\",32,57]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC9FF1E30713544B4A9EFDBC706429FF7\",33,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HD1B4F89AD7AB4440B85326DE0A3CD9FB\",34,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H7BD885E459AC4C37A68D0646B2B1789D\",35,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H3FD9EC7D9D28490D91467523E23886AA\",36,58]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HE85ADC06B8CE434FB2BAE4A632831143\",37,59]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HC79B53E84C144258A4477E1CD0311508\",39,64]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H64B67A103A1B4FF3981C2A3176C818CA\",40,95]" - ], - "pass": true, - "fails_when": "eligibility consults H or X" - }, - { - "test": "...and so is the na-source ranking's selected 8", - "expected": [ - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idF8D001406A6243A8838D80C7241CCBDF\",95,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idD50F9E5AE38746D68732969A822E264A\",98,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id391D86C18A674027A4F24ACA5B72BF63\",23,32]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H008CC077F54D4254A1DFF78AE4A1255C\",25,35]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF913816B59C34861BBEC58B3ABC546D6\",15,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H84D3DE10957043D8AC503C0E976A7FD4\",6,11]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE56DF44D7E454E9898F01E1D6652F69C\",75,101]" - ], - "observed": [ - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idF8D001406A6243A8838D80C7241CCBDF\",95,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idD50F9E5AE38746D68732969A822E264A\",98,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id391D86C18A674027A4F24ACA5B72BF63\",23,32]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H008CC077F54D4254A1DFF78AE4A1255C\",25,35]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF913816B59C34861BBEC58B3ABC546D6\",15,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H84D3DE10957043D8AC503C0E976A7FD4\",6,11]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE56DF44D7E454E9898F01E1D6652F69C\",75,101]" - ], - "pass": true, - "fails_when": "ranking or selection consults H or X" - }, - { - "test": "the sabotage is real -- calling a disabled path raises", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the monkeypatch did not take, so the control passed because nothing was disabled" - }, - { - "test": "N-A eligible population is at least 8", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "fewer than 8 independent N-A sources survive the bridge -- a STOP, not a reason to relax eligibility" - }, - { - "test": "N-B eligible population is at least 8", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "fewer than 8 independent N-B sources survive the bridge" - }, - { - "test": "the na-source draw is reproducible from the committed identities", - "expected": [ - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idF8D001406A6243A8838D80C7241CCBDF\",95,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idD50F9E5AE38746D68732969A822E264A\",98,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id391D86C18A674027A4F24ACA5B72BF63\",23,32]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H008CC077F54D4254A1DFF78AE4A1255C\",25,35]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF913816B59C34861BBEC58B3ABC546D6\",15,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H84D3DE10957043D8AC503C0E976A7FD4\",6,11]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE56DF44D7E454E9898F01E1D6652F69C\",75,101]" - ], - "observed": [ - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idF8D001406A6243A8838D80C7241CCBDF\",95,126]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"idD50F9E5AE38746D68732969A822E264A\",98,127]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"id391D86C18A674027A4F24ACA5B72BF63\",23,32]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H678E045EEF614CC397A326B1375E18E1\",20,20]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"H008CC077F54D4254A1DFF78AE4A1255C\",25,35]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"HF913816B59C34861BBEC58B3ABC546D6\",15,17]", - "[\"xml-account\",\"tests/corpus/118-hr-8752/1_reported-in-house.xml\",\"H84D3DE10957043D8AC503C0E976A7FD4\",6,11]", - "[\"xml-account\",\"tests/corpus/114-hr-2029/4_reported-in-senate.xml\",\"HE56DF44D7E454E9898F01E1D6652F69C\",75,101]" - ], - "pass": true, - "fails_when": "the ranking is not deterministic" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x25_bridge_validation.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x25_bridge_validation.json deleted file mode 100644 index e8511f1c..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x25_bridge_validation.json +++ /dev/null @@ -1,1792 +0,0 @@ -{ - "population": "DEVELOPMENT only -- no holdout opened, nothing adjudicated or scored", - "contract": "A40 sections 1 and 2 -- independent bridge validation and the provenance split", - "anchors": { - "114-hr-2029/4": { - "n_anchors": 85, - "by_class": { - "C": 83, - "A": 2 - }, - "monotonicity": { - "n_anchors": 85, - "n_adjacent_pairs": 84, - "n_inversions": 0, - "at": [] - }, - "n_physical_lines": 7151, - "n_pages": 138 - }, - "118-hr-8752/1": { - "n_anchors": 111, - "by_class": { - "C": 89, - "A": 22 - }, - "monotonicity": { - "n_anchors": 111, - "n_adjacent_pairs": 110, - "n_inversions": 0, - "at": [] - }, - "n_physical_lines": 5907, - "n_pages": 98 - } - }, - "brackets": { - "114-hr-2029/4": { - "n_account_records": 105, - "n_groups": 28, - "n_repeated_groups": 27, - "n_paired": 55, - "refusals": [ - { - "reason": "UNDISCRIMINATED_GROUP", - "text": "(INCLUDING TRANSFER OF FUNDS)", - "xml_count": 38 - }, - { - "reason": "NO_PHYSICAL_OCCURRENCE", - "text": "GENERAL OPERATING EXPENSES, VETERANS BENEFITS ADMINISTRATION", - "xml_count": 2 - }, - { - "reason": "NO_PHYSICAL_OCCURRENCE", - "text": "GRANTS FOR CONSTRUCTION OF STATE EXTENDED CARE FACILITIES", - "xml_count": 2 - }, - { - "reason": "NO_PHYSICAL_OCCURRENCE", - "text": "NATIVE AMERICAN VETERAN HOUSING LOAN PROGRAM ACCOUNT", - "xml_count": 2 - }, - { - "reason": "UNDISCRIMINATED_GROUP", - "text": "SALARIES AND EXPENSES", - "xml_count": 6 - } - ], - "refusal_counts": { - "UNDISCRIMINATED_GROUP": 2, - "NO_PHYSICAL_OCCURRENCE": 3 - }, - "brackets": [ - { - "text": "(INCLUDING RESCISSIONS OF FUNDS)", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 63387, - "physical_line": 2638, - "page_number": 51, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 152366, - "physical_line": 6345, - "page_number": 122, - "n_anchors_before_in_xml": 78, - "n_anchors_before_in_pdf": 78, - "same_interval": true - } - ] - }, - { - "text": "(INCLUDING TRANSFER OF FUNDS)", - "xml_count": 38, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": false, - "occurrences": [ - { - "k": 0, - "xml_offset": 3584, - "physical_line": 134, - "page_number": 4, - "n_anchors_before_in_xml": 2, - "n_anchors_before_in_pdf": 2, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 18300, - "physical_line": 761, - "page_number": 16, - "n_anchors_before_in_xml": 5, - "n_anchors_before_in_pdf": 5, - "same_interval": true - }, - { - "k": 2, - "xml_offset": 19933, - "physical_line": 825, - "page_number": 17, - "n_anchors_before_in_xml": 5, - "n_anchors_before_in_pdf": 5, - "same_interval": true - }, - { - "k": 3, - "xml_offset": 21715, - "physical_line": 895, - "page_number": 18, - "n_anchors_before_in_xml": 5, - "n_anchors_before_in_pdf": 5, - "same_interval": true - }, - { - "k": 4, - "xml_offset": 26807, - "physical_line": 1119, - "page_number": 23, - "n_anchors_before_in_xml": 9, - "n_anchors_before_in_pdf": 9, - "same_interval": true - }, - { - "k": 5, - "xml_offset": 35847, - "physical_line": 1510, - "page_number": 30, - "n_anchors_before_in_xml": 11, - "n_anchors_before_in_pdf": 11, - "same_interval": true - }, - { - "k": 6, - "xml_offset": 38068, - "physical_line": 1628, - "page_number": 32, - "n_anchors_before_in_xml": 14, - "n_anchors_before_in_pdf": 14, - "same_interval": true - }, - { - "k": 7, - "xml_offset": 47827, - "physical_line": 2010, - "page_number": 39, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 8, - "xml_offset": 48362, - "physical_line": 2032, - "page_number": 39, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 9, - "xml_offset": 51503, - "physical_line": 2160, - "page_number": 42, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 10, - "xml_offset": 53186, - "physical_line": 2226, - "page_number": 43, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 11, - "xml_offset": 55427, - "physical_line": 2314, - "page_number": 45, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 12, - "xml_offset": 56311, - "physical_line": 2354, - "page_number": 46, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 13, - "xml_offset": 57751, - "physical_line": 2410, - "page_number": 47, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 14, - "xml_offset": 58547, - "physical_line": 2442, - "page_number": 47, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 15, - "xml_offset": 59783, - "physical_line": 2492, - "page_number": 48, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 16, - "xml_offset": 60992, - "physical_line": 2540, - "page_number": 49, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 17, - "xml_offset": 62065, - "physical_line": 2584, - "page_number": 50, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 18, - "xml_offset": 62962, - "physical_line": 2620, - "page_number": 51, - "n_anchors_before_in_xml": 26, - "n_anchors_before_in_pdf": 26, - "same_interval": true - }, - { - "k": 19, - "xml_offset": 68434, - "physical_line": 2852, - "page_number": 55, - "n_anchors_before_in_xml": 28, - "n_anchors_before_in_pdf": 28, - "same_interval": true - }, - { - "k": 20, - "xml_offset": 70289, - "physical_line": 2924, - "page_number": 57, - "n_anchors_before_in_xml": 28, - "n_anchors_before_in_pdf": 28, - "same_interval": true - }, - { - "k": 21, - "xml_offset": 90053, - "physical_line": 3845, - "page_number": 74, - "n_anchors_before_in_xml": 47, - "n_anchors_before_in_pdf": 47, - "same_interval": true - }, - { - "k": 22, - "xml_offset": 103632, - "physical_line": 4406, - "page_number": 85, - "n_anchors_before_in_xml": 51, - "n_anchors_before_in_pdf": 51, - "same_interval": true - }, - { - "k": 23, - "xml_offset": 105070, - "physical_line": 4460, - "page_number": 86, - "n_anchors_before_in_xml": 51, - "n_anchors_before_in_pdf": 51, - "same_interval": true - }, - { - "k": 24, - "xml_offset": 107224, - "physical_line": 4540, - "page_number": 87, - "n_anchors_before_in_xml": 51, - "n_anchors_before_in_pdf": 51, - "same_interval": true - }, - { - "k": 25, - "xml_offset": 113413, - "physical_line": 4792, - "page_number": 92, - "n_anchors_before_in_xml": 59, - "n_anchors_before_in_pdf": 59, - "same_interval": true - }, - { - "k": 26, - "xml_offset": 123297, - "physical_line": 5180, - "page_number": 100, - "n_anchors_before_in_xml": 62, - "n_anchors_before_in_pdf": 62, - "same_interval": true - }, - { - "k": 27, - "xml_offset": 136273, - "physical_line": 5700, - "page_number": 110, - "n_anchors_before_in_xml": 76, - "n_anchors_before_in_pdf": 76, - "same_interval": true - }, - { - "k": 28, - "xml_offset": 136809, - "physical_line": 5722, - "page_number": 110, - "n_anchors_before_in_xml": 76, - "n_anchors_before_in_pdf": 76, - "same_interval": true - }, - { - "k": 29, - "xml_offset": 139954, - "physical_line": 5846, - "page_number": 113, - "n_anchors_before_in_xml": 76, - "n_anchors_before_in_pdf": 76, - "same_interval": true - }, - { - "k": 30, - "xml_offset": 141638, - "physical_line": 5910, - "page_number": 114, - "n_anchors_before_in_xml": 76, - "n_anchors_before_in_pdf": 76, - "same_interval": true - }, - { - "k": 31, - "xml_offset": 143821, - "physical_line": 5996, - "page_number": 116, - "n_anchors_before_in_xml": 77, - "n_anchors_before_in_pdf": 77, - "same_interval": true - }, - { - "k": 32, - "xml_offset": 144705, - "physical_line": 6034, - "page_number": 116, - "n_anchors_before_in_xml": 77, - "n_anchors_before_in_pdf": 77, - "same_interval": true - }, - { - "k": 33, - "xml_offset": 146476, - "physical_line": 6104, - "page_number": 118, - "n_anchors_before_in_xml": 78, - "n_anchors_before_in_pdf": 78, - "same_interval": true - }, - { - "k": 34, - "xml_offset": 147273, - "physical_line": 6136, - "page_number": 118, - "n_anchors_before_in_xml": 78, - "n_anchors_before_in_pdf": 78, - "same_interval": true - }, - { - "k": 35, - "xml_offset": 148772, - "physical_line": 6196, - "page_number": 119, - "n_anchors_before_in_xml": 78, - "n_anchors_before_in_pdf": 78, - "same_interval": true - }, - { - "k": 36, - "xml_offset": 149981, - "physical_line": 6244, - "page_number": 120, - "n_anchors_before_in_xml": 78, - "n_anchors_before_in_pdf": 78, - "same_interval": true - }, - { - "k": 37, - "xml_offset": 151054, - "physical_line": 6291, - "page_number": 121, - "n_anchors_before_in_xml": 78, - "n_anchors_before_in_pdf": 78, - "same_interval": true - } - ] - }, - { - "text": "(RESCISSION OF FUNDS)", - "xml_count": 8, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 24028, - "physical_line": 995, - "page_number": 20, - "n_anchors_before_in_xml": 5, - "n_anchors_before_in_pdf": 5, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 24351, - "physical_line": 1011, - "page_number": 21, - "n_anchors_before_in_xml": 6, - "n_anchors_before_in_pdf": 6, - "same_interval": true - }, - { - "k": 2, - "xml_offset": 24679, - "physical_line": 1025, - "page_number": 21, - "n_anchors_before_in_xml": 7, - "n_anchors_before_in_pdf": 7, - "same_interval": true - }, - { - "k": 3, - "xml_offset": 25011, - "physical_line": 1039, - "page_number": 21, - "n_anchors_before_in_xml": 8, - "n_anchors_before_in_pdf": 8, - "same_interval": true - }, - { - "k": 4, - "xml_offset": 67395, - "physical_line": 2806, - "page_number": 54, - "n_anchors_before_in_xml": 27, - "n_anchors_before_in_pdf": 27, - "same_interval": true - }, - { - "k": 5, - "xml_offset": 70820, - "physical_line": 2946, - "page_number": 57, - "n_anchors_before_in_xml": 28, - "n_anchors_before_in_pdf": 28, - "same_interval": true - }, - { - "k": 6, - "xml_offset": 112075, - "physical_line": 4734, - "page_number": 91, - "n_anchors_before_in_xml": 58, - "n_anchors_before_in_pdf": 58, - "same_interval": true - }, - { - "k": 7, - "xml_offset": 157396, - "physical_line": 6561, - "page_number": 126, - "n_anchors_before_in_xml": 81, - "n_anchors_before_in_pdf": 81, - "same_interval": true - } - ] - }, - { - "text": "(RESCISSIONS OF FUNDS)", - "xml_count": 6, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 71240, - "physical_line": 2966, - "page_number": 57, - "n_anchors_before_in_xml": 28, - "n_anchors_before_in_pdf": 28, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 111450, - "physical_line": 4706, - "page_number": 90, - "n_anchors_before_in_xml": 55, - "n_anchors_before_in_pdf": 55, - "same_interval": true - }, - { - "k": 2, - "xml_offset": 156522, - "physical_line": 6521, - "page_number": 126, - "n_anchors_before_in_xml": 78, - "n_anchors_before_in_pdf": 78, - "same_interval": true - }, - { - "k": 3, - "xml_offset": 156868, - "physical_line": 6537, - "page_number": 126, - "n_anchors_before_in_xml": 80, - "n_anchors_before_in_pdf": 80, - "same_interval": true - }, - { - "k": 4, - "xml_offset": 157548, - "physical_line": 6571, - "page_number": 127, - "n_anchors_before_in_xml": 82, - "n_anchors_before_in_pdf": 82, - "same_interval": true - }, - { - "k": 5, - "xml_offset": 157844, - "physical_line": 6585, - "page_number": 127, - "n_anchors_before_in_xml": 83, - "n_anchors_before_in_pdf": 83, - "same_interval": true - } - ] - }, - { - "text": "(TRANSFER OF FUNDS)", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 142490, - "physical_line": 5944, - "page_number": 115, - "n_anchors_before_in_xml": 76, - "n_anchors_before_in_pdf": 76, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 151951, - "physical_line": 6327, - "page_number": 122, - "n_anchors_before_in_xml": 78, - "n_anchors_before_in_pdf": 78, - "same_interval": true - } - ] - }, - { - "text": "BOARD OF VETERANS APPEALS", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 36739, - "physical_line": 1554, - "page_number": 30, - "n_anchors_before_in_xml": 13, - "n_anchors_before_in_pdf": 13, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 124001, - "physical_line": 5206, - "page_number": 100, - "n_anchors_before_in_xml": 63, - "n_anchors_before_in_pdf": 63, - "same_interval": true - } - ] - }, - { - "text": "COMPENSATION AND PENSIONS", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 26781, - "physical_line": 1117, - "page_number": 23, - "n_anchors_before_in_xml": 9, - "n_anchors_before_in_pdf": 9, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 113387, - "physical_line": 4790, - "page_number": 92, - "n_anchors_before_in_xml": 59, - "n_anchors_before_in_pdf": 59, - "same_interval": true - } - ] - }, - { - "text": "CONSTRUCTION, MAJOR PROJECTS", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 43138, - "physical_line": 1826, - "page_number": 35, - "n_anchors_before_in_xml": 20, - "n_anchors_before_in_pdf": 20, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 130178, - "physical_line": 5444, - "page_number": 105, - "n_anchors_before_in_xml": 70, - "n_anchors_before_in_pdf": 70, - "same_interval": true - } - ] - }, - { - "text": "CONSTRUCTION, MINOR PROJECTS", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 45624, - "physical_line": 1920, - "page_number": 37, - "n_anchors_before_in_xml": 23, - "n_anchors_before_in_pdf": 23, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 134070, - "physical_line": 5610, - "page_number": 108, - "n_anchors_before_in_xml": 74, - "n_anchors_before_in_pdf": 74, - "same_interval": true - } - ] - }, - { - "text": "FOREIGN CURRENCY FLUCTUATIONS ACCOUNT", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 73283, - "physical_line": 3092, - "page_number": 60, - "n_anchors_before_in_xml": 39, - "n_anchors_before_in_pdf": 39, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 162999, - "physical_line": 6835, - "page_number": 132, - "n_anchors_before_in_xml": 84, - "n_anchors_before_in_pdf": 84, - "same_interval": true - } - ] - }, - { - "text": "GENERAL ADMINISTRATION", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 35824, - "physical_line": 1508, - "page_number": 30, - "n_anchors_before_in_xml": 11, - "n_anchors_before_in_pdf": 11, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 123274, - "physical_line": 5178, - "page_number": 100, - "n_anchors_before_in_xml": 62, - "n_anchors_before_in_pdf": 62, - "same_interval": true - } - ] - }, - { - "text": "GRANTS FOR CONSTRUCTION OF VETERANS CEMETERIES", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 47529, - "physical_line": 1996, - "page_number": 39, - "n_anchors_before_in_xml": 25, - "n_anchors_before_in_pdf": 25, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 135975, - "physical_line": 5684, - "page_number": 109, - "n_anchors_before_in_xml": 75, - "n_anchors_before_in_pdf": 75, - "same_interval": true - } - ] - }, - { - "text": "INFORMATION TECHNOLOGY SYSTEMS", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 38037, - "physical_line": 1626, - "page_number": 32, - "n_anchors_before_in_xml": 14, - "n_anchors_before_in_pdf": 14, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 125174, - "physical_line": 5256, - "page_number": 101, - "n_anchors_before_in_xml": 64, - "n_anchors_before_in_pdf": 64, - "same_interval": true - } - ] - }, - { - "text": "MEDICAL AND PROSTHETIC RESEARCH", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 34959, - "physical_line": 1470, - "page_number": 29, - "n_anchors_before_in_xml": 11, - "n_anchors_before_in_pdf": 11, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 122409, - "physical_line": 5140, - "page_number": 99, - "n_anchors_before_in_xml": 62, - "n_anchors_before_in_pdf": 62, - "same_interval": true - } - ] - }, - { - "text": "MEDICAL FACILITIES", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 34015, - "physical_line": 1432, - "page_number": 28, - "n_anchors_before_in_xml": 11, - "n_anchors_before_in_pdf": 11, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 121318, - "physical_line": 5098, - "page_number": 98, - "n_anchors_before_in_xml": 62, - "n_anchors_before_in_pdf": 62, - "same_interval": true - } - ] - }, - { - "text": "MEDICAL SERVICES", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 31161, - "physical_line": 1318, - "page_number": 26, - "n_anchors_before_in_xml": 10, - "n_anchors_before_in_pdf": 10, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 117766, - "physical_line": 4964, - "page_number": 95, - "n_anchors_before_in_xml": 60, - "n_anchors_before_in_pdf": 60, - "same_interval": true - } - ] - }, - { - "text": "MEDICAL SUPPORT AND COMPLIANCE", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 33374, - "physical_line": 1406, - "page_number": 28, - "n_anchors_before_in_xml": 11, - "n_anchors_before_in_pdf": 11, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 120530, - "physical_line": 5066, - "page_number": 97, - "n_anchors_before_in_xml": 62, - "n_anchors_before_in_pdf": 62, - "same_interval": true - } - ] - }, - { - "text": "OFFICE OF INSPECTOR GENERAL", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 42834, - "physical_line": 1812, - "page_number": 35, - "n_anchors_before_in_xml": 18, - "n_anchors_before_in_pdf": 18, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 129888, - "physical_line": 5432, - "page_number": 105, - "n_anchors_before_in_xml": 68, - "n_anchors_before_in_pdf": 68, - "same_interval": true - } - ] - }, - { - "text": "READJUSTMENT BENEFITS", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 28557, - "physical_line": 1203, - "page_number": 24, - "n_anchors_before_in_xml": 9, - "n_anchors_before_in_pdf": 9, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 115161, - "physical_line": 4856, - "page_number": 93, - "n_anchors_before_in_xml": 59, - "n_anchors_before_in_pdf": 59, - "same_interval": true - } - ] - }, - { - "text": "SALARIES AND EXPENSES", - "xml_count": 6, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": false, - "occurrences": [ - { - "k": 0, - "xml_offset": 72575, - "physical_line": 3062, - "page_number": 59, - "n_anchors_before_in_xml": 39, - "n_anchors_before_in_pdf": 39, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 73608, - "physical_line": 3108, - "page_number": 60, - "n_anchors_before_in_xml": 39, - "n_anchors_before_in_pdf": 39, - "same_interval": true - }, - { - "k": 2, - "xml_offset": 74094, - "physical_line": 3135, - "page_number": 60, - "n_anchors_before_in_xml": 39, - "n_anchors_before_in_pdf": 39, - "same_interval": true - }, - { - "k": 3, - "xml_offset": 162291, - "physical_line": 6807, - "page_number": 131, - "n_anchors_before_in_xml": 84, - "n_anchors_before_in_pdf": 84, - "same_interval": true - }, - { - "k": 4, - "xml_offset": 163324, - "physical_line": 6851, - "page_number": 132, - "n_anchors_before_in_xml": 84, - "n_anchors_before_in_pdf": 84, - "same_interval": true - }, - { - "k": 5, - "xml_offset": 163810, - "physical_line": 6873, - "page_number": 132, - "n_anchors_before_in_xml": 84, - "n_anchors_before_in_pdf": 84, - "same_interval": true - } - ] - }, - { - "text": "TRUST FUND", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 74762, - "physical_line": 3165, - "page_number": 61, - "n_anchors_before_in_xml": 39, - "n_anchors_before_in_pdf": 39, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 164479, - "physical_line": 6908, - "page_number": 133, - "n_anchors_before_in_xml": 85, - "n_anchors_before_in_pdf": 85, - "same_interval": true - } - ] - }, - { - "text": "VETERANS HOUSING BENEFIT PROGRAM FUND", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 29579, - "physical_line": 1252, - "page_number": 25, - "n_anchors_before_in_xml": 9, - "n_anchors_before_in_pdf": 9, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 116183, - "physical_line": 4898, - "page_number": 94, - "n_anchors_before_in_xml": 59, - "n_anchors_before_in_pdf": 59, - "same_interval": true - } - ] - }, - { - "text": "VETERANS INSURANCE AND INDEMNITIES", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 29197, - "physical_line": 1236, - "page_number": 25, - "n_anchors_before_in_xml": 9, - "n_anchors_before_in_pdf": 9, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 115801, - "physical_line": 4882, - "page_number": 94, - "n_anchors_before_in_xml": 59, - "n_anchors_before_in_pdf": 59, - "same_interval": true - } - ] - }, - { - "text": "VOCATIONAL REHABILITATION LOANS PROGRAM ACCOUNT", - "xml_count": 2, - "n_usable_anchors": 85, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 30255, - "physical_line": 1278, - "page_number": 25, - "n_anchors_before_in_xml": 9, - "n_anchors_before_in_pdf": 9, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 116860, - "physical_line": 4926, - "page_number": 95, - "n_anchors_before_in_xml": 59, - "n_anchors_before_in_pdf": 59, - "same_interval": true - } - ] - } - ] - }, - "118-hr-8752/1": { - "n_account_records": 41, - "n_groups": 12, - "n_repeated_groups": 4, - "n_paired": 41, - "refusals": [], - "refusal_counts": {}, - "brackets": [ - { - "text": "FEDERAL ASSISTANCE", - "xml_count": 2, - "n_usable_anchors": 111, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 51635, - "physical_line": 2685, - "page_number": 44, - "n_anchors_before_in_xml": 62, - "n_anchors_before_in_pdf": 62, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 68297, - "physical_line": 3601, - "page_number": 59, - "n_anchors_before_in_xml": 103, - "n_anchors_before_in_pdf": 103, - "same_interval": true - } - ] - }, - { - "text": "OPERATIONS AND SUPPORT", - "xml_count": 15, - "n_usable_anchors": 111, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 1069, - "physical_line": 55, - "page_number": 2, - "n_anchors_before_in_xml": 0, - "n_anchors_before_in_pdf": 0, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 1768, - "physical_line": 95, - "page_number": 3, - "n_anchors_before_in_xml": 3, - "n_anchors_before_in_pdf": 3, - "same_interval": true - }, - { - "k": 2, - "xml_offset": 2514, - "physical_line": 131, - "page_number": 3, - "n_anchors_before_in_xml": 6, - "n_anchors_before_in_pdf": 6, - "same_interval": true - }, - { - "k": 3, - "xml_offset": 3061, - "physical_line": 172, - "page_number": 4, - "n_anchors_before_in_xml": 9, - "n_anchors_before_in_pdf": 9, - "same_interval": true - }, - { - "k": 4, - "xml_offset": 10741, - "physical_line": 598, - "page_number": 11, - "n_anchors_before_in_xml": 11, - "n_anchors_before_in_pdf": 11, - "same_interval": true - }, - { - "k": 5, - "xml_offset": 13297, - "physical_line": 720, - "page_number": 13, - "n_anchors_before_in_xml": 20, - "n_anchors_before_in_pdf": 20, - "same_interval": true - }, - { - "k": 6, - "xml_offset": 15847, - "physical_line": 845, - "page_number": 15, - "n_anchors_before_in_xml": 29, - "n_anchors_before_in_pdf": 29, - "same_interval": true - }, - { - "k": 7, - "xml_offset": 17105, - "physical_line": 916, - "page_number": 16, - "n_anchors_before_in_xml": 34, - "n_anchors_before_in_pdf": 34, - "same_interval": true - }, - { - "k": 8, - "xml_offset": 20292, - "physical_line": 1064, - "page_number": 18, - "n_anchors_before_in_xml": 44, - "n_anchors_before_in_pdf": 44, - "same_interval": true - }, - { - "k": 9, - "xml_offset": 50514, - "physical_line": 2618, - "page_number": 43, - "n_anchors_before_in_xml": 54, - "n_anchors_before_in_pdf": 54, - "same_interval": true - }, - { - "k": 10, - "xml_offset": 51091, - "physical_line": 2656, - "page_number": 44, - "n_anchors_before_in_xml": 58, - "n_anchors_before_in_pdf": 58, - "same_interval": true - }, - { - "k": 11, - "xml_offset": 65388, - "physical_line": 3435, - "page_number": 56, - "n_anchors_before_in_xml": 89, - "n_anchors_before_in_pdf": 89, - "same_interval": true - }, - { - "k": 12, - "xml_offset": 65838, - "physical_line": 3467, - "page_number": 57, - "n_anchors_before_in_xml": 91, - "n_anchors_before_in_pdf": 91, - "same_interval": true - }, - { - "k": 13, - "xml_offset": 66780, - "physical_line": 3505, - "page_number": 57, - "n_anchors_before_in_xml": 94, - "n_anchors_before_in_pdf": 94, - "same_interval": true - }, - { - "k": 14, - "xml_offset": 67576, - "physical_line": 3555, - "page_number": 58, - "n_anchors_before_in_xml": 99, - "n_anchors_before_in_pdf": 99, - "same_interval": true - } - ] - }, - { - "text": "PROCUREMENT, CONSTRUCTION, AND IMPROVEMENTS", - "xml_count": 11, - "n_usable_anchors": 111, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 1984, - "physical_line": 105, - "page_number": 3, - "n_anchors_before_in_xml": 4, - "n_anchors_before_in_pdf": 4, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 12862, - "physical_line": 690, - "page_number": 12, - "n_anchors_before_in_xml": 17, - "n_anchors_before_in_pdf": 17, - "same_interval": true - }, - { - "k": 2, - "xml_offset": 15495, - "physical_line": 817, - "page_number": 14, - "n_anchors_before_in_xml": 25, - "n_anchors_before_in_pdf": 25, - "same_interval": true - }, - { - "k": 3, - "xml_offset": 16693, - "physical_line": 882, - "page_number": 15, - "n_anchors_before_in_xml": 31, - "n_anchors_before_in_pdf": 31, - "same_interval": true - }, - { - "k": 4, - "xml_offset": 18430, - "physical_line": 978, - "page_number": 17, - "n_anchors_before_in_xml": 39, - "n_anchors_before_in_pdf": 39, - "same_interval": true - }, - { - "k": 5, - "xml_offset": 22128, - "physical_line": 1154, - "page_number": 20, - "n_anchors_before_in_xml": 47, - "n_anchors_before_in_pdf": 47, - "same_interval": true - }, - { - "k": 6, - "xml_offset": 50826, - "physical_line": 2644, - "page_number": 44, - "n_anchors_before_in_xml": 56, - "n_anchors_before_in_pdf": 56, - "same_interval": true - }, - { - "k": 7, - "xml_offset": 51320, - "physical_line": 2671, - "page_number": 44, - "n_anchors_before_in_xml": 59, - "n_anchors_before_in_pdf": 59, - "same_interval": true - }, - { - "k": 8, - "xml_offset": 66325, - "physical_line": 3487, - "page_number": 57, - "n_anchors_before_in_xml": 93, - "n_anchors_before_in_pdf": 93, - "same_interval": true - }, - { - "k": 9, - "xml_offset": 67138, - "physical_line": 3533, - "page_number": 58, - "n_anchors_before_in_xml": 96, - "n_anchors_before_in_pdf": 96, - "same_interval": true - }, - { - "k": 10, - "xml_offset": 67882, - "physical_line": 3569, - "page_number": 58, - "n_anchors_before_in_xml": 101, - "n_anchors_before_in_pdf": 101, - "same_interval": true - } - ] - }, - { - "text": "RESEARCH AND DEVELOPMENT", - "xml_count": 5, - "n_usable_anchors": 111, - "agrees": true, - "discriminating": true, - "occurrences": [ - { - "k": 0, - "xml_offset": 16913, - "physical_line": 904, - "page_number": 16, - "n_anchors_before_in_xml": 32, - "n_anchors_before_in_pdf": 32, - "same_interval": true - }, - { - "k": 1, - "xml_offset": 19006, - "physical_line": 1000, - "page_number": 17, - "n_anchors_before_in_xml": 40, - "n_anchors_before_in_pdf": 40, - "same_interval": true - }, - { - "k": 2, - "xml_offset": 22437, - "physical_line": 1166, - "page_number": 20, - "n_anchors_before_in_xml": 50, - "n_anchors_before_in_pdf": 50, - "same_interval": true - }, - { - "k": 3, - "xml_offset": 67353, - "physical_line": 3543, - "page_number": 58, - "n_anchors_before_in_xml": 97, - "n_anchors_before_in_pdf": 97, - "same_interval": true - }, - { - "k": 4, - "xml_offset": 68108, - "physical_line": 3591, - "page_number": 59, - "n_anchors_before_in_xml": 102, - "n_anchors_before_in_pdf": 102, - "same_interval": true - } - ] - } - ] - } - }, - "negatives": { - "group": "OPERATIONS AND SUPPORT", - "xml_count": 15, - "pdf_count": 15, - "moved_across": "$281,358,000", - "refusals_under_swapped_anchors": { - "CROSSES_INDEPENDENT_ANCHOR": 4 - } - }, - "anchor_uniqueness_negatives": { - "cases": { - "duplicate class-C money literal elsewhere in the PDF": true, - "two class-C occurrences on ONE printed line": true, - "duplicate class-A printed header": true - }, - "base_anchor_count": 111 - }, - "provenance": { - "provenance": { - "text_content": "SOURCE-DETERMINED", - "punctuation": "SOURCE-DETERMINED", - "whitespace": "SOURCE-DETERMINED", - "case": "PHYSICALLY-OBSERVED", - "margin_number": "PHYSICALLY-OBSERVED" - }, - "n_cross_checked": 96, - "helper_cases": { - "identical_but_for_case": true, - "punctuation_differs": false, - "content_differs": false, - "whitespace_differs": false - }, - "authority": { - "content_case_rules": "billres-details.xsl convertToNeededCase (8279); bills.css approp header classes", - "recorded_in": "docs/gpo-render-conventions.md casing table (#53)", - "why_case_is_not_source_determined": "those rules govern GPO's HTML renderer. docs/gpo-render-conventions.md (#89) records the measurement that the published PDF is typeset by GPO's separate photocomposition system, whose sizes contradict the CSS em values -- so the PDF's realised case is observed, not asserted" - } - }, - "findings_for_ruling": { - "parenthetical_qualifier_sources": { - "by_document": { - "114-hr-2029/4": { - "paired": 55, - "parenthetical": 18 - }, - "118-hr-8752/1": { - "paired": 41, - "parenthetical": 4 - } - }, - "n_parenthetical": 22, - "n_non_parenthetical": 74, - "distinct_texts": [ - "(INCLUDING RESCISSIONS OF FUNDS)", - "(INCLUDING TRANSFER OF FUNDS)", - "(INCLUDING TRANSFERS AND RESCISSIONS OF FUNDS)", - "(INCLUDING TRANSFERS OF FUNDS)", - "(RESCISSION OF FUNDS)", - "(RESCISSIONS OF FUNDS)", - "(TRANSFER OF FUNDS)" - ], - "why_it_matters": "A40.3 requires a candidate to be 'a known real account heading'. These records are GPO transfer/rescission qualifiers printed as their own heading line; in the MilCon division the account NAME sits on the preceding appropriations-intermediate. (RESCISSIONS OF FUNDS) is in the CURRENT selected 8 N-B, so this is result-bearing." - }, - "case_sensitive_grouping_defect": { - "by_document": { - "114-hr-2029/4": { - "case_sensitive_groups": 33, - "case_insensitive_groups": 28 - }, - "118-hr-8752/1": { - "case_sensitive_groups": 12, - "case_insensitive_groups": 12 - } - }, - "why_it_matters": "the previous bridge grouped on the case-SENSITIVE XML string while the comparator is case-insensitive, splitting real groups into phantom halves that then refused on count mismatch. It fails closed, so nothing wrong was paired -- but it shrank 114-hr-2029/4 from 55 paired to 45 and mis-labelled the loss." - } - }, - "stop_conditions": [], - "tests": [ - { - "test": "the independent anchor grid is dense enough for bracketing to mean anything", - "expected": { - "114-hr-2029/4": true, - "118-hr-8752/1": true - }, - "observed": { - "114-hr-2029/4": true, - "118-hr-8752/1": true - }, - "pass": true, - "fails_when": "an anchor grid this sparse makes every occurrence trivially 'inside the interval', so a green bracketing result would carry no information" - }, - { - "test": "XML reading order agrees with physical print order, on ANCHORS ONLY -- zero inversions", - "expected": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "observed": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "pass": true, - "fails_when": "independent anchors invert, so XML order does NOT correspond to physical print order on real DEVELOPMENT material -- a STOP, and the ordinal pairing rule would be unlicensed" - }, - { - "test": "the independent evidence is far larger than x24's unique-occurrence subset (n=1, n=8)", - "expected": { - "114-hr-2029/4": true, - "118-hr-8752/1": true - }, - "observed": { - "114-hr-2029/4": true, - "118-hr-8752/1": true - }, - "pass": true, - "fails_when": "the anchor population is no better than the subset it replaces" - }, - { - "test": "no repeated group that survives is bracket-DISAGREEING (an occurrence outside its interval)", - "expected": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "observed": { - "114-hr-2029/4": 0, - "118-hr-8752/1": 0 - }, - "pass": true, - "fails_when": "a paired occurrence sits between a different set of independent anchors than its XML counterpart, i.e. the k-th XML and k-th physical occurrence are NOT the same heading" - }, - { - "test": "every group that PAIRS is fully discriminated -- consecutive occurrences separated by an anchor", - "expected": { - "114-hr-2029/4": true, - "118-hr-8752/1": true - }, - "observed": { - "114-hr-2029/4": true, - "118-hr-8752/1": true - }, - "pass": true, - "fails_when": "a group pairs while two of its occurrences share an anchor prefix, so their index-to-index association still rests on the ordering assumption under test rather than on evidence" - }, - { - "test": "an undiscriminated group is REFUSED rather than paired", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no group was ever refused for weak bracketing, so the discrimination rule has never once fired and cannot be distinguished from a rule that is not implemented" - }, - { - "test": "NEGATIVE -- swapping two repeated physical occurrences BREAKS the bracketing", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "two occurrences of the same heading can be exchanged without the independent anchors noticing, so the bracketing does not actually constrain which physical line pairs with which XML occurrence" - }, - { - "test": "NEGATIVE -- moving ONE occurrence across an independent neighbour anchor BREAKS bracketing", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "an occurrence can cross an independently identified anchor without detection, which is exactly the mis-association the interval rule exists to prevent" - }, - { - "test": "NEGATIVE -- an ADDED physical occurrence refuses the whole group on GROUP_COUNT_MISMATCH", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "a surplus physical occurrence is paired anyway, so k-to-k pairing would silently associate a heading with the wrong printed line" - }, - { - "test": "...and the whole group is removed, not just the surplus occurrence", - "expected": 0, - "observed": 0, - "pass": true, - "fails_when": "only the extra occurrence is dropped, leaving the rest of an unverifiable group paired" - }, - { - "test": "NEGATIVE -- a REMOVED physical occurrence refuses the whole group on GROUP_COUNT_MISMATCH", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "a missing physical occurrence still pairs, so k XML occurrences would map onto k-1 lines" - }, - { - "test": "NEGATIVE -- swapping two independent anchors is detected as an inversion", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the monotonicity check cannot go red, so its zero-inversion result on the real material proves nothing at all" - }, - { - "test": "...and it also makes real groups REFUSE, so the corruption reaches the pairing decision", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "corrupted anchors change only a summary field and never refuse a group, so the bracketing rule is not actually consuming the anchors it claims to" - }, - { - "test": "NEGATIVES -- each injected duplication REMOVES the anchor it makes ambiguous", - "expected": { - "duplicate class-C money literal elsewhere in the PDF": true, - "two class-C occurrences on ONE printed line": true, - "duplicate class-A printed header": true - }, - "observed": { - "duplicate class-C money literal elsewhere in the PDF": true, - "two class-C occurrences on ONE printed line": true, - "duplicate class-A printed header": true - }, - "pass": true, - "fails_when": "an anchor survives a duplication that destroys its unique identity, so the zero duplicate counts on the real corpus prove nothing about the filter" - }, - { - "test": "the per-dimension provenance rule is committed, not implicit", - "expected": { - "text_content": "SOURCE-DETERMINED", - "punctuation": "SOURCE-DETERMINED", - "whitespace": "SOURCE-DETERMINED", - "case": "PHYSICALLY-OBSERVED", - "margin_number": "PHYSICALLY-OBSERVED" - }, - "observed": { - "text_content": "SOURCE-DETERMINED", - "punctuation": "SOURCE-DETERMINED", - "whitespace": "SOURCE-DETERMINED", - "case": "PHYSICALLY-OBSERVED", - "margin_number": "PHYSICALLY-OBSERVED" - }, - "pass": true, - "fails_when": "the three objects are collapsed, so a PDF reading could silently become the source of account semantics (or a stylesheet rule could be invented for the PDF that governs only HTML)" - }, - { - "test": "every paired record's observed line agrees with source truth under CASE FOLDING ALONE", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "no record was cross-checked, so the source-vs-physical agreement claim is vacuous" - }, - { - "test": "NEGATIVE -- case alone agrees; punctuation, content and whitespace differences REFUSE", - "expected": { - "identical_but_for_case": true, - "punctuation_differs": false, - "content_differs": false, - "whitespace_differs": false - }, - "observed": { - "identical_but_for_case": true, - "punctuation_differs": false, - "content_differs": false, - "whitespace_differs": false - }, - "pass": true, - "fails_when": "the cross-check folds away a dimension the source actually determines, so a genuinely different printed heading would be accepted as the expected one" - }, - { - "test": "a CASE-only difference in the XML still pairs (case is physically observed)", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "case is being treated as source-determined, which contradicts the measured GPO pipeline and would refuse every real heading" - }, - { - "test": "a ONE-LETTER content difference refuses that source instead of pairing it", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "content is being folded away like case, so the printed heading is no longer constrained by the XML at all" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x27_score_metrics.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x27_score_metrics.json deleted file mode 100644 index 6f8c61cb..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x27_score_metrics.json +++ /dev/null @@ -1,3640 +0,0 @@ -{ - "population": "SYNTHETIC + DEVELOPMENT -- no holdout opened, nothing adjudicated by a human or an AI", - "contract": "HARNESS-PLAN section 5 (the scorer) over PRE-REGISTRATION section 6 and section 8", - "question": "does `score_metrics` compute the frozen quantities from committed artifacts, and can each of them be made to go RED?", - "artifacts_created": "NONE of frames.json, oracle_key.json, oracle_blind.json, oracle_adjudicated.json, s1_control.json, cross_engine_control.json, metrics.json, scores.json, EXECUTION-START.json", - "synthetic_adjudication_caveat": "the fixtures' oracle text comes from an arm's own emitted output, so every agreement figure here is evidence about the JOIN and never about accuracy", - "section5_rows": { - "S1 liveness": [ - "S1 FIRING leaves M0 reportable", - "S1 NOT firing makes every M0 rate NOT REPORTABLE", - "...and the RAW counts survive the suppression, so evidence is not destroyed", - "NEGATIVE -- an S1 artifact with no verdict at all REFUSES" - ], - "M3 weld/space fixture": [ - "the weld/space fixture reaches M3 as X_CORRECTS", - "the reported outcome vocabulary IS `m3_boundaries.HeadingOutcome`, complete", - "...and the weld is charged to H at the BOUNDARY level, not as a character error" - ], - "M3 insulation": [ - "the insulation fixture really IS segmentation-discordant, so the control is not vacuous", - "M3 fabricates NO weld or split from a segmentation-only difference", - "NEGATIVE -- M3 is INVARIANT to the segmentation labels, text held identical", - "...and the fixtures really do differ in segmentation, so the invariance is not vacuous" - ], - "negative: delete agency anchors": [ - "deleting the agency anchors makes M4 fall FURTHER than M1" - ], - "negative: shift heading baselines one line-height": [ - "a one-line parent shift makes M4 fall while M1 holds still" - ], - "negative: inject an R E P O R T page": [ - "the injected letter-spaced page is DETECTED for the arm that split it", - "...and the matching text is PRINTED, not merely counted", - "NEGATIVE -- 2 single-character tokens do NOT match and 3 DO: the threshold is live", - "...and an ordinary heading matches nothing, so M7 is not simply always true" - ], - "M0 denominator": [ - "3 BOTH_ABSENT lines leave the risk set at 5 and are reported as a raw count", - "every M0 line-rate denominator IS the risk set, never the line count", - "NEGATIVE -- the all-lines denominator gives a DIFFERENT rate, so the fixture is decisive", - "...and the REPORTED headline is the risk-set rate, not the superseded one" - ], - "vacuity": [ - "a page neither arm emitted reports VACUOUS, not 0.0 and not agreement", - "an arm that emitted nothing reports VACUOUS, not a zero rate" - ], - "section 8 independence": [ - "section 8.1's own fixture reproduces: 14 documents 0.1926 vs 600 headings 0.00498", - "N is the number of DOCUMENTS scored, and the bound is the document-unit one", - "NEGATIVE -- a heading-level table (one document, many rows) REFUSES" - ], - "section 8 zero-event": [ - "at zero events NO bootstrap is reported, and the reason is explicit", - "...and the zero-event bound IS the frozen closed form, to the last bit", - "...and the general bisection AGREES with the closed form at k = 0, so it is not a special case", - "...and the reported wording is an OBSERVATION plus a bound, with an explicit disclaimer" - ], - "section 8 pairing": [ - "R9 -- exactly the two non-constant numeric M9 basis quantities are paired", - "the per-document differences and the UNWEIGHTED mean are exact, over documents", - "...and the heading-count-WEIGHTED mean differs, so the fixture can tell them apart", - "P-head and P-robust are reported SEPARATELY, with no combined mean anywhere", - "...and the POOLED mean is a different number that appears NOWHERE in the payload", - "per-document detail is MANDATORY: every document in each population has a row", - "coverage pairs too, and its P-head mean is the unweighted 0.0 (+0.05 and -0.05)", - "no M1-M5 or M7 quantity is present in the paired block" - ] - }, - "evidence": { - "m0": { - "concordant": { - "risk_set": 8, - "M0a_text": 0, - "M0b_segmentation": 0 - }, - "with_chrome": { - "risk_set": 5, - "both_absent": 3, - "neutral_lines_in_scope": 8 - }, - "vacuous": { - "risk_set": 0, - "status": "VACUOUS" - }, - "segmentation_only": { - "M0a_text": 0, - "M0b_segmentation": 2 - }, - "m0c": { - "regions": 1, - "rate": 1.0 - } - }, - "i9": { - "d_reasons": [ - "TEXT_DISCORDANCE" - ] - }, - "s1": { - "live": { - "fires": true, - "advance_scale": 1.25, - "m0_reportable": true, - "m0_status": "REPORTED", - "gate_owner": "Rule 3 gate vector (A27.6)" - }, - "dead": { - "fires": false, - "advance_scale": 1.25, - "m0_reportable": false, - "m0_status": "NOT_REPORTABLE_S1_DEAD", - "gate_owner": "Rule 3 gate vector (A27.6)" - } - }, - "join": { - "pooled_c_counts": { - "n_adjudicated": 2, - "n_adjudicated_unresolvable": 0, - "n_emitted": { - "H": 2, - "X": 2 - }, - "n_matched": { - "H": 2, - "X": 2 - }, - "m2_exact": { - "H": 2, - "X": 2 - }, - "m2_scored": { - "H": 2, - "X": 2 - }, - "m2_excluded_unreadable": 0, - "m3_clean": { - "H": 2, - "X": 2 - }, - "m3_scored": { - "H": 2, - "X": 2 - }, - "m3_boundary": { - "H": { - "WELD": 0, - "SPLIT": 0, - "OK": 36, - "TEXT_ERROR": 0 - }, - "X": { - "WELD": 0, - "SPLIT": 0, - "OK": 36, - "TEXT_ERROR": 0 - } - }, - "m3_outcomes": { - "X_CORRECTS": 0, - "X_REGRESSES": 0, - "BOTH_CLEAN": 2, - "BOTH_DIRTY": 0, - "UNSCORABLE": 0 - }, - "m3_excluded_no_reference": 0, - "m4_correct": { - "H": 2, - "X": 2 - }, - "m4_scored": { - "H": 2, - "X": 2 - }, - "m4_excluded_unreadable": 0, - "m4_excluded_off_region": 0, - "m4_h_correct_x_wrong": 0, - "m4_x_correct_h_wrong": 0, - "m4_h_correct_x_wrong_keys": [], - "m5_agree": { - "H": 2, - "X": 2 - }, - "m5_scored": { - "H": 2, - "X": 2 - }, - "m5_excluded_unscorable": { - "H": 0, - "X": 0 - }, - "m5_excluded_unreadable": 0, - "n_stimuli": 2 - }, - "i10": { - "recall": { - "numerator": 1, - "denominator": 2, - "metric": "M1 recall", - "value": 0.5, - "status": "REPORTED" - }, - "precision": { - "numerator": 1, - "denominator": 1, - "metric": "M1 precision", - "value": 1.0, - "status": "REPORTED" - } - }, - "n_stimuli": 2 - }, - "m3": { - "weld": { - "X_CORRECTS": 1, - "X_REGRESSES": 0, - "BOTH_CLEAN": 0, - "BOTH_DIRTY": 0, - "UNSCORABLE": 0 - }, - "insulated": { - "H": { - "WELD": 0, - "SPLIT": 0, - "OK": 18, - "TEXT_ERROR": 0 - }, - "X": { - "WELD": 0, - "SPLIT": 0, - "OK": 18, - "TEXT_ERROR": 0 - } - }, - "garbage": { - "X_CORRECTS": 0, - "X_REGRESSES": 1, - "BOTH_CLEAN": 0, - "BOTH_DIRTY": 0, - "UNSCORABLE": 0 - } - }, - "m4": { - "pairing": { - "opposite_pairing": { - "m4_correct": { - "H": 3, - "X": 3 - }, - "m4_scored": { - "H": 4, - "X": 4 - }, - "m4_h_correct_x_wrong": 1 - }, - "near_miss": 0 - }, - "baseline_m4": { - "numerator": 4, - "denominator": 4, - "metric": "M4", - "value": 1.0, - "status": "REPORTED" - }, - "after_delete": { - "numerator": 0, - "denominator": 3, - "metric": "M4", - "value": 0.0, - "status": "REPORTED" - }, - "m1_drop": 0.25, - "m4_drop": 1.0, - "shifted_m4": { - "numerator": 1, - "denominator": 4, - "metric": "M4", - "value": 0.25, - "status": "REPORTED" - }, - "ancestry_m4": { - "numerator": 1, - "denominator": 4, - "metric": "M4", - "value": 0.25, - "status": "REPORTED" - } - }, - "m5": { - "m5": { - "agreement": { - "numerator": 1, - "denominator": 1, - "metric": "M5", - "value": 1.0, - "status": "REPORTED" - }, - "excluded_unscorable": 1 - }, - "promoted_denominator": 2 - }, - "r1": { - "n_pairs": 2, - "identical": { - "text": "PASS", - "role": "PASS" - }, - "disagreeing": { - "text": "FAIL", - "role": "FAIL" - }, - "one_sided_pair": { - "route": "ai", - "primary_blind_id": "1e62e3e14a5992d4", - "repeat_blind_id": "7d194fd0fd347425", - "n_primary_headings": 3, - "n_repeat_headings": 2, - "n_primary_unresolved": 0, - "n_repeat_unresolved": 0, - "n_primary_non_unique": 0, - "n_repeat_non_unique": 0, - "n_matched_pairs": 2, - "n_text_agree": 2, - "n_role_agree": 2, - "denominator": 3 - }, - "unresolved_pair": { - "route": "ai", - "primary_blind_id": "1e62e3e14a5992d4", - "repeat_blind_id": "7d194fd0fd347425", - "n_primary_headings": 3, - "n_repeat_headings": 3, - "n_primary_unresolved": 0, - "n_repeat_unresolved": 1, - "n_primary_non_unique": 0, - "n_repeat_non_unique": 0, - "n_matched_pairs": 2, - "n_text_agree": 2, - "n_role_agree": 2, - "denominator": 4 - }, - "duplicate_pair": { - "route": "ai", - "primary_blind_id": "1e62e3e14a5992d4", - "repeat_blind_id": "7d194fd0fd347425", - "n_primary_headings": 3, - "n_repeat_headings": 3, - "n_primary_unresolved": 0, - "n_repeat_unresolved": 0, - "n_primary_non_unique": 0, - "n_repeat_non_unique": 2, - "n_matched_pairs": 1, - "n_text_agree": 1, - "n_role_agree": 1, - "denominator": 5 - }, - "whitespace": { - "status": "FAIL", - "ratio": 0.0, - "numerator": 0, - "denominator": 3, - "threshold": 0.9 - }, - "unreadable_role": { - "status": "FAIL", - "ratio": 0.0, - "numerator": 0, - "denominator": 3, - "threshold": 0.8 - }, - "ruled_by": "A41.2 R6" - }, - "r8_controls": { - "by_kind": { - "N-A": { - "status": "PASS", - "n_passed": 16, - "n_total": 16 - }, - "N-B": { - "status": "PASS", - "n_passed": 16, - "n_total": 16 - }, - "N-C": { - "status": "PASS", - "n_passed": 8, - "n_total": 8 - } - }, - "n_controls": 20, - "whitespace_reason": "EXPECTED_TARGET_ABSENT", - "route_asymmetry": { - "ai": { - "n_passed": 8, - "n_total": 8 - }, - "human": { - "n_passed": 7, - "n_total": 8 - } - } - }, - "m6_and_i13": { - "m6_surfaces": [], - "n_labelled_surfaces": 22, - "unlabelled": [], - "headline_qualifications": { - "RQ1": "PDFIUM-CONDITIONED FRAME", - "RQ2": "PDFIUM-CONDITIONED FRAME" - } - }, - "m7": { - "injected": { - "n_emitted": 1, - "n_display_split": 1, - "rate": { - "numerator": 1, - "denominator": 1, - "metric": "M7", - "value": 1.0, - "status": "REPORTED" - }, - "instances": [ - { - "page_number": 1, - "line_number": 2, - "text": "R E P O R T", - "n_single_char_tokens": 6, - "longest_single_char_run": 6 - } - ] - }, - "boundary": { - "n_display_split": 1 - } - }, - "rule0": { - "margin": { - "loser": "H", - "fires": true, - "h": 197, - "x": 198, - "deficit": 1 - }, - "band": { - "loser": "H", - "fires": true, - "h": false, - "x": true - }, - "coverage": { - "loser": "H", - "fires": true, - "h": false, - "x": true - } - }, - "section8": { - "zero_event": { - "n": 14, - "bound": 0.19263617565013536 - }, - "nonzero": { - "events": 1, - "bound": 0.29673423894887463 - }, - "bootstrap_interval": [ - 0.0, - 0.21428571428571427 - ], - "excluded_p_robust": [ - "ROBUST/1" - ], - "structured_population": [ - "DOC/0", - "DOC/1", - "DOC/10", - "DOC/11", - "DOC/12", - "DOC/13", - "DOC/2", - "DOC/3", - "DOC/4", - "DOC/5", - "DOC/6", - "DOC/7", - "DOC/8", - "DOC/9" - ] - }, - "pairing": { - "quantities": [ - "n_margin_numbered_lines", - "coverage" - ], - "p_head": { - "n_margin_numbered_lines": 15.0, - "coverage": -5.551115123125783e-17 - }, - "p_robust": { - "n_margin_numbered_lines": 100.0, - "coverage": 0.0 - }, - "weighted_would_be": 17.5, - "pooled_would_be": 43.333333333333336 - }, - "adequacy": { - "head_only": { - "kinds_counted": [ - "account", - "agency", - "grouping" - ], - "population_counted": "P-head", - "n_occurrences": 3, - "strata_filled": 1, - "strata": [ - 1 - ], - "verdict": "INADEQUATE", - "owner": "decide_architecture consumes this (Rule 3); the verdict itself is a population fact" - }, - "with_robust": 3, - "generous": "GENERALISABLE" - }, - "qualification": { - "one": { - "per_document": { - "DOC/0": "PDFIUM-CONDITIONED FRAME", - "DOC/1": null, - "DOC/2": null, - "DOC/3": null, - "DOC/4": null, - "DOC/5": null - }, - "n_documents": 6, - "n_failed": 1, - "failed_documents": [ - "DOC/0" - ], - "both_headlines_qualified": false, - "headline_qualifications": { - "RQ1": null, - "RQ2": null - }, - "decision_blocking": false, - "threshold_owner": "x09_skeleton_cross_engine.gate (document 0.95 / page 0.75)" - }, - "two_of_six": false, - "three_of_six": true - }, - "refusals": { - "refusals": { - "MISSING_REQUIRED_FIELD": "MISSING_REQUIRED_FIELD", - "DUPLICATE_DOCUMENT_IDENTITY": "DUPLICATE_DOCUMENT_IDENTITY", - "UNKNOWN_POPULATION": "UNKNOWN_POPULATION", - "OCCURRENCE_RECORD_INCOMPLETE": "OCCURRENCE_RECORD_INCOMPLETE", - "DUPLICATE_OCCURRENCE_KEY": "DUPLICATE_OCCURRENCE_KEY", - "FRAMES_NOT_A_SEQUENCE": "FRAMES_NOT_A_SEQUENCE", - "STIMULUS_DOCUMENT_NOT_IN_FRAMES": "STIMULUS_DOCUMENT_NOT_IN_FRAMES", - "ADJUDICATION_ROUTE_MISSING": "ADJUDICATION_ROUTE_MISSING", - "CONTROL_STIMULUS_IN_ESTIMAND": "CONTROL_STIMULUS_IN_ESTIMAND" - }, - "duplicate_document_layer": [ - "ScoreInputError", - "DUPLICATE_DOCUMENT_IDENTITY" - ] - }, - "attacks": { - "reidentified_matched": 0 - }, - "controls_and_repeats": { - "n_repeats": 1, - "n_controls": 20, - "c_regions": 8, - "d_regions": 12 - }, - "reproducibility": { - "payload_sha256": "69106c9d8f6a4b48043a79754a9b2d269f6e96127f22ef1bba99e094d6448f73" - }, - "development": { - "document": "118-hr-8752/1", - "page_limit": 8, - "note": "DEV_PAGE_LIMIT is a machinery demonstration window, not a census", - "m0": { - "risk_set": 188, - "both_absent": 22, - "M0a_text": 42, - "M0b_segmentation": 0, - "M0_any": 42 - }, - "m0_rates": { - "M0a_text_rate": 0.22340425531914893, - "M0b_segmentation_rate": 0.0, - "M0_any_rate": 0.22340425531914893 - }, - "n_stimuli": 26, - "keyed_occurrences_in_stimuli": { - "C": 9, - "D": 16 - }, - "matched_occurrences": { - "C": 9, - "D": 16 - }, - "m7": { - "H": 0, - "X": 0 - }, - "rule0": { - "loser": "H", - "fires": true, - "h": 134, - "x": 171, - "deficit": 37 - }, - "section8": { - "events": 1, - "bound": 1.0 - }, - "adequacy": 11 - }, - "boundary": { - "scorer_imports": [ - "__future__", - "build_frames", - "build_oracle", - "collections", - "dataclasses", - "json", - "m3_boundaries", - "math", - "methodology_contracts", - "neutral_identity", - "pathlib", - "sys", - "xml_sources" - ], - "forbidden_calls": { - "run_hybrid.run": false, - "run_extended.run": false, - "x01_contamination": false, - "an engine": false, - "extract_anchors": false, - "filesystem_discovery": false - }, - "renderer_dependency": "NONE at import: `build_oracle` (which imports pymupdf at module scope) is imported lazily by `score_metrics._bo`, so the scorer imports and scores JSON-only quantities with the renderer unimportable. The frozen A38.7/A36.4 helpers are still called, so a derivation that genuinely needs them still requires the renderer", - "renderer_free_child": { - "blocker_live": true, - "scorer_imports": true, - "section8_bound": 0.1926, - "bo_owned_still_delegates": true - }, - "canonical_artifacts_present": { - "frames.json": false, - "oracle_key.json": false, - "oracle_blind.json": false, - "oracle_adjudicated.json": false, - "s1_control.json": false, - "cross_engine_control.json": false, - "metrics.json": false, - "scores.json": false, - "EXECUTION-START.json": false - } - } - }, - "tests": [ - { - "test": "a fully concordant page yields a full risk set and zero discordance", - "expected": [ - 8, - 0, - 0, - 0 - ], - "observed": [ - 8, - 0, - 0, - 0 - ], - "pass": true, - "fails_when": "the fixture is not concordant, so every comparison below starts from a moving baseline", - "row": "" - }, - { - "test": "3 BOTH_ABSENT lines leave the risk set at 5 and are reported as a raw count", - "expected": [ - 5, - 3, - 8 - ], - "observed": [ - 5, - 3, - 8 - ], - "pass": true, - "fails_when": "a jointly-dropped line entered the risk set, which would make the rate a function of how much page furniture GPO set rather than of the seam", - "row": "M0 denominator" - }, - { - "test": "every M0 line-rate denominator IS the risk set, never the line count", - "expected": { - "M0a_text_rate": 5, - "M0b_segmentation_rate": 5, - "M0_any_rate": 5 - }, - "observed": { - "M0a_text_rate": 5, - "M0b_segmentation_rate": 5, - "M0_any_rate": 5 - }, - "pass": true, - "fails_when": "a BOTH_ABSENT line appears in an M0 denominator, i.e. a shared drop is scored as agreement", - "row": "M0 denominator" - }, - { - "test": "...and M0b's DEFINED denominator is reported separately (A23's reporting rule)", - "expected": 5, - "observed": 5, - "pass": true, - "fails_when": "the defined-population rate shares the risk-set denominator, so a zero M0b could be read as 'the arms grouped identically' when it means 'there was nothing to compare'", - "row": "" - }, - { - "test": "NEGATIVE -- the all-lines denominator gives a DIFFERENT rate, so the fixture is decisive", - "expected": [ - 0.2, - 0.125 - ], - "observed": [ - 0.2, - 0.125 - ], - "pass": true, - "fails_when": "both denominators agree on this fixture, so 'BOTH_ABSENT is excluded' was never tested", - "row": "M0 denominator" - }, - { - "test": "...and the REPORTED headline is the risk-set rate, not the superseded one", - "expected": 0.2, - "observed": 0.2, - "pass": true, - "fails_when": "the scorer reports the all-lines rate, which counts a shared drop as agreement", - "row": "M0 denominator" - }, - { - "test": "a page neither arm emitted reports VACUOUS, not 0.0 and not agreement", - "expected": [ - 0, - null, - "VACUOUS", - null, - "VACUOUS" - ], - "observed": [ - 0, - null, - "VACUOUS", - null, - "VACUOUS" - ], - "pass": true, - "fails_when": "a zero-denominator metric is printed as a rate -- 0.0 reads as perfect agreement and 1.0 as total failure, and neither was measured", - "row": "vacuity" - }, - { - "test": "a pure segmentation difference moves M0b and leaves M0a at zero", - "expected": [ - 0, - 2, - 2, - 0 - ], - "observed": [ - 0, - 2, - 2, - 0 - ], - "pass": true, - "fails_when": "grouping and characters are conflated, which is the exact defect A23 corrected", - "row": "" - }, - { - "test": "an anchor-set difference is counted as M0c over REGIONS, with the line rates untouched", - "expected": [ - 1, - 1, - 1.0, - 0, - 0 - ], - "observed": [ - 1, - 1, - 1.0, - 0, - 0 - ], - "pass": true, - "fails_when": "M0c is pooled into or averaged with the per-line rates, which section 5 forbids", - "row": "" - }, - { - "test": "a text-discordant line puts its region in the D-frame (same predicate, one set)", - "expected": [ - true, - [ - "TEXT_DISCORDANCE" - ] - ], - "observed": [ - true, - [ - "TEXT_DISCORDANCE" - ] - ], - "pass": true, - "fails_when": "M0 and the D-frame would be reading different eligibility sets, so Rule 1's census and M0 would be measuring different things", - "row": "" - }, - { - "test": "the clean input SCORES, so the refusals below are not refusing everything", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "the scorer refuses valid input, making every negative control meaningless", - "row": "" - }, - { - "test": "NEGATIVE -- a discordant line in a NON-D region REFUSES (I9)", - "expected": "D_FRAME_ELIGIBILITY_DRIFT", - "observed": "D_FRAME_ELIGIBILITY_DRIFT", - "pass": true, - "fails_when": "two eligibility sets are accepted silently, and the D-frame census that Rule 1 reads would exclude regions M0 counted as discordant", - "row": "" - }, - { - "test": "NEGATIVE -- a D-frame region with NO qualifying predicate REFUSES", - "expected": "D_FRAME_ELIGIBILITY_DRIFT", - "observed": "D_FRAME_ELIGIBILITY_DRIFT", - "pass": true, - "fails_when": "a region can be in the adjudication census for no recorded reason, which would spend human adjudication budget on regions nothing selected", - "row": "" - }, - { - "test": "NEGATIVE -- a committed predicate that disagrees with the frozen rule REFUSES", - "expected": "LINE_STATE_PREDICATE_DRIFT", - "observed": "LINE_STATE_PREDICATE_DRIFT", - "pass": true, - "fails_when": "the scorer trusts a committed boolean over `neutral_identity`'s own predicate, so a frame could report agreement its own texts contradict", - "row": "" - }, - { - "test": "NEGATIVE -- an unknown line state REFUSES rather than defaulting into the risk set", - "expected": "UNKNOWN_LINE_STATE", - "observed": "UNKNOWN_LINE_STATE", - "pass": true, - "fails_when": "an unrecognised state falls through to 'not BOTH_ABSENT' and silently enlarges the M0 denominator", - "row": "" - }, - { - "test": "NEGATIVE -- a region whose d_frame contradicts its own d_reasons REFUSES", - "expected": "D_FRAME_FLAG_DRIFT", - "observed": "D_FRAME_FLAG_DRIFT", - "pass": true, - "fails_when": "`d_frame` and `d_reasons` can disagree, so a region carrying a recorded reason can sit outside the census that reason exists to put it in -- and every per-predicate check above passes on this fixture, which is why the exact invariant is needed", - "row": "" - }, - { - "test": "...and the fixture really is invisible to the other checks, so this is not a duplicate", - "expected": [ - false, - false, - false - ], - "observed": [ - false, - false, - false - ], - "pass": true, - "fails_when": "the fixture also carries a line-level or anchor discordance, so an existing check would have caught it and the new invariant is unproven", - "row": "" - }, - { - "test": "the producer really does emit d_frame == bool(d_reasons), so the invariant is its own", - "expected": [ - [ - true, - true - ], - [ - false, - false - ] - ], - "observed": [ - [ - true, - true - ], - [ - false, - false - ] - ], - "pass": true, - "fails_when": "`build_frames` does not maintain the relationship this check enforces, so the check would be asserting something the producer never promised", - "row": "" - }, - { - "test": "S1 FIRING leaves M0 reportable", - "expected": [ - true, - "REPORTED", - 0.125 - ], - "observed": [ - true, - "REPORTED", - 0.125 - ], - "pass": true, - "fails_when": "a live comparator still suppresses M0, so the control cannot distinguish live from dead", - "row": "S1 liveness" - }, - { - "test": "S1 NOT firing makes every M0 rate NOT REPORTABLE", - "expected": [ - false, - "NOT_REPORTABLE_S1_DEAD", - null - ], - "observed": [ - false, - "NOT_REPORTABLE_S1_DEAD", - null - ], - "pass": true, - "fails_when": "M0 is reported from a comparator that cannot be shown to move -- exactly the phase-1 defect S1 exists to catch", - "row": "S1 liveness" - }, - { - "test": "...and the RAW counts survive the suppression, so evidence is not destroyed", - "expected": [ - 1, - 8 - ], - "observed": [ - 1, - 8 - ], - "pass": true, - "fails_when": "withholding the rate also deleted the counts, so a reader cannot see what was measured", - "row": "S1 liveness" - }, - { - "test": "NEGATIVE -- an S1 artifact with no verdict at all REFUSES", - "expected": "S1_ARTIFACT_MISSING", - "observed": "S1_ARTIFACT_MISSING", - "pass": true, - "fails_when": "a missing liveness verdict is read as firing, so M0 is reported with no live comparator", - "row": "S1 liveness" - }, - { - "test": "the join BINDS: every adjudicated heading matches an emitted occurrence in both arms", - "expected": [ - 2, - 2, - 2, - 0 - ], - "observed": [ - 2, - 2, - 2, - 0 - ], - "pass": true, - "fails_when": "the two sides of the M1-M5 join do not meet, so every matched-heading denominator is zero", - "row": "" - }, - { - "test": "M1 recall and precision are reported with their own denominators", - "expected": [ - 1.0, - 2, - 1.0, - 2 - ], - "observed": [ - 1.0, - 2, - 1.0, - 2 - ], - "pass": true, - "fails_when": "a rate is reported without the denominator it is a fraction of", - "row": "" - }, - { - "test": "only the frames the fixture actually populates are reported", - "expected": [ - "C" - ], - "observed": [ - "C" - ], - "pass": true, - "fails_when": "an unpopulated frame is reported with invented counts, or C and D were pooled", - "row": "" - }, - { - "test": "I10 -- an arm that emitted one of two printed headings scores recall 1/2, not 1/1", - "expected": [ - 1, - 2, - 0.5, - 1, - 1, - 1.0 - ], - "observed": [ - 1, - 2, - 0.5, - 1, - 1, - 1.0 - ], - "pass": true, - "fails_when": "recall's denominator is the EMITTED enumeration, which makes an arm that dropped a heading look perfect -- the denominator shrinks with the failure it should expose", - "row": "" - }, - { - "test": "...and the WRONG denominator would have given a different number, so this is decisive", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the two denominators coincide on this fixture, so I10 was never actually tested", - "row": "" - }, - { - "test": "NEGATIVE -- an unmatchable EMITTED heading enlarges the precision denominator only", - "expected": [ - 2, - 3, - 2, - 2 - ], - "observed": [ - 2, - 3, - 2, - 2 - ], - "pass": true, - "fails_when": "the numerator counts EMITTED rather than MATCHED headings, so emitting more raises the score -- a precision metric that rewards over-emission", - "row": "" - }, - { - "test": "NEGATIVE -- an A30-refused occurrence is KEPT in the precision denominator", - "expected": 3, - "observed": 3, - "pass": true, - "fails_when": "an UNMATCHED occurrence is dropped, shrinking a denominator invisibly -- the one failure mode A38.3 names explicitly", - "row": "" - }, - { - "test": "the weld/space fixture reaches M3 as X_CORRECTS", - "expected": [ - 1, - 0, - 0 - ], - "observed": [ - 1, - 0, - 0 - ], - "pass": true, - "fails_when": "`FAMILYHOUSING` against `FAMILY HOUSING` does not reach M3 as X_CORRECTS -- the primary comparative metric cannot see the failure class the seam choice governs", - "row": "M3 weld/space fixture" - }, - { - "test": "the reported outcome vocabulary IS `m3_boundaries.HeadingOutcome`, complete", - "expected": [ - "BOTH_CLEAN", - "BOTH_DIRTY", - "UNSCORABLE", - "X_CORRECTS", - "X_REGRESSES" - ], - "observed": [ - "BOTH_CLEAN", - "BOTH_DIRTY", - "UNSCORABLE", - "X_CORRECTS", - "X_REGRESSES" - ], - "pass": true, - "fails_when": "an outcome bucket is missing, so headings falling into it vanish from the tally the decision rule counts -- X_REGRESSES above all", - "row": "M3 weld/space fixture" - }, - { - "test": "...and the weld is charged to H at the BOUNDARY level, not as a character error", - "expected": [ - 1, - 0, - 0, - 0 - ], - "observed": [ - 1, - 0, - 0, - 0 - ], - "pass": true, - "fails_when": "a spacing defect is scored as a character defect (or vice versa), which is the conflation section 6.3 exists to prevent", - "row": "M3 weld/space fixture" - }, - { - "test": "...and M2 sees the weld as inexact while X reads exactly", - "expected": [ - 0.0, - 1.0 - ], - "observed": [ - 0.0, - 1.0 - ], - "pass": true, - "fails_when": "M2 cannot separate a welded heading from a correct one under the frozen normalisation", - "row": "" - }, - { - "test": "the insulation fixture really IS segmentation-discordant, so the control is not vacuous", - "expected": [ - 2, - 0, - true - ], - "observed": [ - 2, - 0, - true - ], - "pass": true, - "fails_when": "the fixture carries no segmentation difference, so 'M3 ignored it' proves nothing", - "row": "M3 insulation" - }, - { - "test": "M3 fabricates NO weld or split from a segmentation-only difference", - "expected": [ - 0, - 0, - 0, - 0, - 1 - ], - "observed": [ - 0, - 0, - 0, - 0, - 1 - ], - "pass": true, - "fails_when": "a split-only difference fabricates a weld/split against a clean oracle, so the primary comparative metric would report a boundary defect that was never printed", - "row": "M3 insulation" - }, - { - "test": "NEGATIVE -- M3 is INVARIANT to the segmentation labels, text held identical", - "expected": { - "clean_rate": { - "H": { - "numerator": 1, - "denominator": 1, - "metric": "M3", - "value": 1.0, - "status": "REPORTED" - }, - "X": { - "numerator": 1, - "denominator": 1, - "metric": "M3", - "value": 1.0, - "status": "REPORTED" - } - }, - "boundary_outcomes": { - "H": { - "WELD": 0, - "SPLIT": 0, - "OK": 18, - "TEXT_ERROR": 0 - }, - "X": { - "WELD": 0, - "SPLIT": 0, - "OK": 18, - "TEXT_ERROR": 0 - } - }, - "heading_outcomes": { - "X_CORRECTS": 0, - "X_REGRESSES": 0, - "BOTH_CLEAN": 1, - "BOTH_DIRTY": 0, - "UNSCORABLE": 0 - }, - "decision_unit": "heading occurrence (A3); WELD/SPLIT tallies never substitute for it", - "consumes": "projected text + oracle only -- never a segmentation label (section 5)" - }, - "observed": { - "clean_rate": { - "H": { - "numerator": 1, - "denominator": 1, - "metric": "M3", - "value": 1.0, - "status": "REPORTED" - }, - "X": { - "numerator": 1, - "denominator": 1, - "metric": "M3", - "value": 1.0, - "status": "REPORTED" - } - }, - "boundary_outcomes": { - "H": { - "WELD": 0, - "SPLIT": 0, - "OK": 18, - "TEXT_ERROR": 0 - }, - "X": { - "WELD": 0, - "SPLIT": 0, - "OK": 18, - "TEXT_ERROR": 0 - } - }, - "heading_outcomes": { - "X_CORRECTS": 0, - "X_REGRESSES": 0, - "BOTH_CLEAN": 1, - "BOTH_DIRTY": 0, - "UNSCORABLE": 0 - }, - "decision_unit": "heading occurrence (A3); WELD/SPLIT tallies never substitute for it", - "consumes": "projected text + oracle only -- never a segmentation label (section 5)" - }, - "pass": true, - "fails_when": "M3 reads a segmentation label as truth, which section 5 forbids: it consumes projected text plus the oracle and nothing else", - "row": "M3 insulation" - }, - { - "test": "...and the fixtures really do differ in segmentation, so the invariance is not vacuous", - "expected": [ - 0, - 2 - ], - "observed": [ - 0, - 2 - ], - "pass": true, - "fails_when": "the two fixtures have identical segmentation, so 'M3 did not move' was guaranteed", - "row": "M3 insulation" - }, - { - "test": "severe corruption in one arm is X_REGRESSES, never UNSCORABLE", - "expected": [ - 1, - 0, - 1, - 1 - ], - "observed": [ - 1, - 0, - 1, - 1 - ], - "pass": true, - "fails_when": "an arm's worst failures leave the denominator, which removes exactly the distinguishing cases -- the defect A9 withdrew UNALIGNABLE for", - "row": "" - }, - { - "test": "the M4 baseline is perfect, so any fall below is caused by the injection", - "expected": [ - 1.0, - 1.0, - 4, - 4 - ], - "observed": [ - 1.0, - 1.0, - 4, - 4 - ], - "pass": true, - "fails_when": "the baseline is already imperfect, so a 'fall' cannot be attributed to the injection", - "row": "" - }, - { - "test": "deleting the agency anchors makes M4 fall FURTHER than M1", - "expected": [ - 0.25, - 1.0, - true - ], - "observed": [ - 0.25, - 1.0, - true - ], - "pass": true, - "fails_when": "M4 does not fall further than M1, so the hierarchy metric adds nothing over presence -- it would be reporting the labels again rather than the tree", - "row": "negative: delete agency anchors" - }, - { - "test": "a one-line parent shift makes M4 fall while M1 holds still", - "expected": [ - 0.25, - 1.0, - 4 - ], - "observed": [ - 0.25, - 1.0, - 4 - ], - "pass": true, - "fails_when": "M4 does not fall, so a hierarchy that points at the wrong parent scores as correct", - "row": "negative: shift heading baselines one line-height" - }, - { - "test": "NEGATIVE -- an ancestor that is not the IMMEDIATE parent scores as wrong", - "expected": 0.25, - "observed": 0.25, - "pass": true, - "fails_when": "M4 scored full ancestry, so a heading filed one level off its true parent counts as correct as long as the right agency appears somewhere above it", - "row": "" - }, - { - "test": "...and the adjudicated parent really IS in the emitted breadcrumb, so the trap is live", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the fixture does not contain the ancestry trap, so the control could not have caught it", - "row": "" - }, - { - "test": "literal \"NONE\" against an emitted None scores CORRECT, and the child scores too", - "expected": [ - 2, - 2, - 1.0, - 0, - 0 - ], - "observed": [ - 2, - 2, - 1.0, - 0, - 0 - ], - "pass": true, - "fails_when": "the scorer compares \"NONE\" as ordinary parent TEXT, so a correct root heading scores WRONG -- M4 penalises an architecture for having no parent to name", - "row": "" - }, - { - "test": "NEGATIVE -- \"NONE\" against an emitted parent is WRONG, so the sentinel is not a wildcard", - "expected": [ - 1, - 2, - 0.5 - ], - "observed": [ - 1, - 2, - 0.5 - ], - "pass": true, - "fails_when": "\"NONE\" matches whatever the arm emitted, so a hierarchy claim can never be contradicted", - "row": "" - }, - { - "test": "\"OFF_REGION\" LEAVES M4's population and is counted, never scored as text", - "expected": [ - 1, - 1, - 1.0, - 1 - ], - "observed": [ - 1, - 1, - 1.0, - 1 - ], - "pass": true, - "fails_when": "\"OFF_REGION\" is compared as literal parent text, which can never equal any emitted parent -- so every off-region answer counts against an architecture for a limit of the ORACLE's field of view", - "row": "" - }, - { - "test": "...and the frozen answer vocabulary is stated in the artifact", - "expected": "printed text | NONE | OFF_REGION | UNREADABLE", - "observed": "printed text | NONE | OFF_REGION | UNREADABLE", - "pass": true, - "fails_when": "the artifact does not record which parent answers were recognised, so a reader cannot tell a scored population from an excluded one", - "row": "" - }, - { - "test": "NEGATIVE -- a null parent REFUSES rather than being read as 'no parent'", - "expected": "PARENT_MISSING", - "observed": "PARENT_MISSING", - "pass": true, - "fails_when": "a null is silently read as NONE, which is the representation gap that let the old M4 fixtures pass without ever producing the adjudicator's own encoding", - "row": "" - }, - { - "test": "UNREADABLE stays excluded, as already frozen", - "expected": [ - 1, - 1, - 1 - ], - "observed": [ - 1, - 1, - 1 - ], - "pass": true, - "fails_when": "an UNREADABLE parent entered or left the denominator differently than before", - "row": "" - }, - { - "test": "the per-arm M4 aggregates are EQUAL -- so they cannot express the condition", - "expected": [ - 3, - 3, - 4, - 4, - true - ], - "observed": [ - 3, - 3, - 4, - 4, - true - ], - "pass": true, - "fails_when": "the fixture does not actually produce equal aggregates, so it does not separate the two readings and this whole control proves nothing", - "row": "" - }, - { - "test": "...while the PAIRED fact sees the violation, and names the heading it fired on", - "expected": [ - 1, - 1, - [ - 6 - ] - ], - "observed": [ - 1, - 1, - [ - 6 - ] - ], - "pass": true, - "fails_when": "A5 row 4's per-heading existential was collapsed into a count comparison, which is the reading A42.3 rejected -- an X that breaks one hierarchy H had right would win Rule 1 as long as it repaired a different one", - "row": "" - }, - { - "test": "the paired fact is surfaced on the reported M4 block, where the decider reads it", - "expected": [ - 1, - 1 - ], - "observed": [ - 1, - 1 - ], - "pass": true, - "fails_when": "the quantity exists only in raw counts, so the decider would have to reach past the reported metric to find it", - "row": "" - }, - { - "test": "NEAR-MISS -- both arms wrong on the SAME account: aggregates equal, pairing ZERO", - "expected": [ - 3, - 3, - 0, - 0 - ], - "observed": [ - 3, - 3, - 0, - 0 - ], - "pass": true, - "fails_when": "the paired fact fires whenever an arm is wrong anywhere, so it is a restatement of the aggregates rather than the per-heading comparison Rule 1 needs", - "row": "" - }, - { - "test": "an UNSCORABLE pair leaves the M5 DENOMINATOR and is reported as a raw exclusion", - "expected": [ - 1, - 1, - 1.0, - 1 - ], - "observed": [ - 1, - 1, - 1.0, - 1 - ], - "pass": true, - "fails_when": "an UNSCORABLE pair is counted as a disagreement, penalising an architecture for a role M5 was never licensed to score", - "row": "" - }, - { - "test": "NEGATIVE -- making the UNSCORABLE kind scorable MOVES the denominator", - "expected": [ - 2, - 0 - ], - "observed": [ - 2, - 0 - ], - "pass": true, - "fails_when": "the denominator does not respond to the frozen A36.7 map, so the exclusion is decoration", - "row": "" - }, - { - "test": "NEGATIVE -- an unmapped oracle role REFUSES (A36.7)", - "expected": "UnknownRole", - "observed": "UnknownRole", - "pass": true, - "fails_when": "an unknown role maps to UNSCORABLE and quietly SHRINKS the M5 denominator, which reads as a cleaner result rather than as a defect", - "row": "" - }, - { - "test": "with no R1 pair in the key, the M5 gate is NOT EVALUABLE and is not a pass", - "expected": [ - "NOT_EVALUABLE_NO_R1_PAIRS", - false, - true - ], - "observed": [ - "NOT_EVALUABLE_NO_R1_PAIRS", - false, - true - ], - "pass": true, - "fails_when": "M5 is reported as though its section 6 gate had been checked when no R1 pair exists -- green because nothing measured it", - "row": "" - }, - { - "test": "NO caller channel exists that could hand the gate a verdict", - "expected": [ - false, - false - ], - "observed": [ - false, - false - ], - "pass": true, - "fails_when": "a free scalar can still reach a result-bearing reliability gate, so a PASS can be asserted with no evidence behind it and nothing downstream could tell", - "row": "" - }, - { - "test": "the fixture really carries an R1 pair, so nothing below is vacuous", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "no repeat exists, so every R1 figure below was computed on an empty population", - "row": "" - }, - { - "test": "an identical repeat gives PASS on both dimensions, with all four denominators agreeing", - "expected": [ - "PASS", - "PASS", - 2, - [ - "ai", - "human" - ] - ], - "observed": [ - "PASS", - "PASS", - 2, - [ - "ai", - "human" - ] - ], - "pass": true, - "fails_when": "a perfectly self-consistent adjudicator does not reach PASS, so the reliability gate cannot be satisfied by any real adjudication", - "row": "" - }, - { - "test": "R1 is computed PER ROUTE: a human-only disagreement leaves the AI route untouched", - "expected": [ - "PASS", - "FAIL", - 1.0, - 0.0, - false - ], - "observed": [ - "PASS", - "FAIL", - 1.0, - 0.0, - false - ], - "pass": true, - "fails_when": "AI and human pairs are pooled, so one route's disagreement is averaged into the other -- which measures inter-source disagreement rather than the repeat reliability section 5.6 asks for, and can hide a failing route behind a passing one", - "row": "" - }, - { - "test": "...and the overall gate takes the WORST route, so a passing route cannot mask a failure", - "expected": [ - "FAIL", - true - ], - "observed": [ - "FAIL", - true - ], - "pass": true, - "fails_when": "the overall verdict is an average or the best route, so a route that failed its reliability gate does not reach Rule 3", - "row": "" - }, - { - "test": "NEGATIVE -- a repeat that answers differently drives BOTH gates to FAIL", - "expected": [ - "FAIL", - "FAIL" - ], - "observed": [ - "FAIL", - "FAIL" - ], - "pass": true, - "fails_when": "an adjudicator that contradicts itself on a re-presented stimulus still passes, which is exactly the phase-1 defect R1 exists to catch (six identical stimuli, 3 BOUNDARY / 3 NOT)", - "row": "" - }, - { - "test": "the role comparison uses the FINE section 5.3 role: account vs section is a DISAGREEMENT", - "expected": [ - "FAIL", - "PASS", - "LEAF", - "LEAF" - ], - "observed": [ - "FAIL", - "PASS", - "LEAF", - "LEAF" - ], - "pass": true, - "fails_when": "R1 coarsens the role before comparing, so two DIFFERENT fine roles that collapse to the same M5 class read as agreement -- A36.7 says M5 ALONE coarsens, and an adjudicator flipping account/section would look perfectly reliable", - "row": "" - }, - { - "test": "...and the coarse-blind fixture leaves TEXT agreement untouched, so only the role moved", - "expected": [ - 1.0, - "PASS" - ], - "observed": [ - 1.0, - "PASS" - ], - "pass": true, - "fails_when": "the fixture perturbed the text as well, so a role-only claim cannot be attributed to the role comparison", - "row": "" - }, - { - "test": "R6.1 -- a heading present on ONE side stays in the denominator and earns no numerator", - "expected": [ - 3, - 2, - 2, - 2, - 3, - 0.6666666666666666, - "FAIL" - ], - "observed": [ - 3, - 2, - 2, - 2, - 3, - 0.6666666666666666, - "FAIL" - ], - "pass": true, - "fails_when": "the denominator is the INTERSECTION, so an adjudicator who silently drops a heading scores perfectly reliable -- the enumeration instability section 5.6 exists to detect", - "row": "" - }, - { - "test": "R6.1 NEGATIVE -- one UNRESOLVED repeat heading stays in the denominator, agreement falls", - "expected": [ - 3, - 3, - 1, - 2, - 2, - 4, - 0.5 - ], - "observed": [ - 3, - 3, - 1, - 2, - 2, - 4, - 0.5 - ], - "pass": true, - "fails_when": "an unresolved heading vanishes from the denominator, so a geometric refusal quietly IMPROVES the measured reliability of the adjudicator that produced it", - "row": "" - }, - { - "test": "R6.1 NEGATIVE -- BOTH sides unresolved is denominator-bearing, never vacuous or perfect", - "expected": [ - 1, - 1, - 2, - 2, - 4, - 0.5, - "FAIL" - ], - "observed": [ - 1, - 1, - 2, - 2, - 4, - 0.5, - "FAIL" - ], - "pass": true, - "fails_when": "two unresolved headings cancel into a vacuous or perfect result, so an adjudication the join could not read at all would satisfy the reliability gate", - "row": "" - }, - { - "test": "R6.1 NEGATIVE -- a duplicated occurrence key is NOT pairable and cannot be overwritten", - "expected": [ - 2, - 3, - 3, - 1, - 5, - "FAIL" - ], - "observed": [ - 2, - 3, - 3, - 1, - 5, - "FAIL" - ], - "pass": true, - "fails_when": "the duplicate is collapsed through a dict -- last write wins -- so two answers for one identity become a single perfect match instead of the disagreement evidence they are", - "row": "" - }, - { - "test": "R6.2 -- a whitespace-run difference that M2 would call EQUAL makes R1 text agreement fall", - "expected": [ - "FAIL", - 0.0, - true, - true - ], - "observed": [ - "FAIL", - 0.0, - true, - true - ], - "pass": true, - "fails_when": "R1 normalises the text before comparing, so an adjudicator whose spacing wanders reads as perfectly reliable -- and spacing instability is exactly what the repeat records", - "row": "" - }, - { - "test": "...and the ROLE gate is untouched by the spacing change, so the two dimensions are separate", - "expected": "PASS", - "observed": "PASS", - "pass": true, - "fails_when": "a text-only perturbation moved the role gate, so the two agreements are not independent", - "row": "" - }, - { - "test": "R6.3 NEGATIVE -- UNREADABLE vs UNREADABLE is NOT role agreement", - "expected": [ - "FAIL", - 0.0, - "PASS" - ], - "observed": [ - "FAIL", - 0.0, - "PASS" - ], - "pass": true, - "fails_when": "repeated unreadability manufactures role reliability -- an absence of evidence counted as evidence of consistency, and the one input an adjudicator can always produce", - "row": "" - }, - { - "test": "R6.4 -- the route ratio is a heading-occurrence micro-average of the summed counts", - "expected": [ - true, - 2, - 3 - ], - "observed": [ - true, - 2, - 3 - ], - "pass": true, - "fails_when": "the ratio is not numerator/denominator over occurrences, so a one-heading region and a forty-heading region carry the same weight in the reliability figure", - "row": "" - }, - { - "test": "...and no abstention status survives the ruling", - "expected": [ - false, - false - ], - "observed": [ - false, - false - ], - "pass": true, - "fails_when": "the candidate-denominator machinery or the AMBIGUOUS status is still reachable, so a consumer can read a status the protocol no longer defines", - "row": "" - }, - { - "test": "A36.6 -- a repeat declaring only the PASSING route REFUSES; it cannot delete the failure", - "expected": [ - "ScoreInputError", - "R1_ROUTE_SET_MISMATCH" - ], - "observed": [ - "ScoreInputError", - "R1_ROUTE_SET_MISMATCH" - ], - "pass": true, - "fails_when": "the scorer iterates the routes the REPEAT claims, so a shortened repeat record plus a correspondingly shortened answer set silently drops a FAILING required route and the gate passes on the survivor -- and nothing in the artifact looks wrong", - "row": "" - }, - { - "test": "...and the un-shortened pair still reaches the FAIL, so the refusal is the only difference", - "expected": "FAIL", - "observed": "FAIL", - "pass": true, - "fails_when": "the asymmetric fixture does not actually fail on the human route, so the control above would refuse something that was never going to be a masked failure", - "row": "" - }, - { - "test": "A36.6 -- a repeat whose FRAMES are silently narrowed REFUSES", - "expected": [ - "ScoreInputError", - "R1_FRAME_SET_MISMATCH" - ], - "observed": [ - "ScoreInputError", - "R1_FRAME_SET_MISMATCH" - ], - "pass": true, - "fails_when": "frame membership is taken from the repeat, so narrowing it changes the required route set -- the same masking, one level further back", - "row": "" - }, - { - "test": "...even when frames AND routes are narrowed together, so the record is self-consistent", - "expected": [ - "ScoreInputError", - "R1_FRAME_SET_MISMATCH" - ], - "observed": [ - "ScoreInputError", - "R1_FRAME_SET_MISMATCH" - ], - "pass": true, - "fails_when": "a self-consistent narrowing passes because only frames-vs-routes agreement is checked, never agreement with the PRIMARY -- which is what A36.6 actually freezes", - "row": "" - }, - { - "test": "the frozen frame->route map is derived from build_oracle's own constants", - "expected": [ - [ - "ai" - ], - [ - "human" - ], - [ - "ai", - "human" - ] - ], - "observed": [ - [ - "ai" - ], - [ - "human" - ], - [ - "ai", - "human" - ] - ], - "pass": true, - "fails_when": "the C->AI / D->human / C&D->both mapping is restated here and can drift from A36.4's owner", - "row": "" - }, - { - "test": "NEGATIVE -- a repeat whose primary is absent REFUSES", - "expected": "R1_PRIMARY_MISSING", - "observed": "R1_PRIMARY_MISSING", - "pass": true, - "fails_when": "an orphaned repeat is skipped, shrinking the very population whose size decides whether an unreliable adjudicator can be detected", - "row": "" - }, - { - "test": "the committed 8 / 8 / 4 controls all PASS against their own committed truth, on BOTH routes", - "expected": [ - { - "N-A": "PASS", - "N-B": "PASS", - "N-C": "PASS" - }, - { - "N-A": 16, - "N-B": 16, - "N-C": 8 - }, - 20 - ], - "observed": [ - { - "N-A": "PASS", - "N-B": "PASS", - "N-C": "PASS" - }, - { - "N-A": 16, - "N-B": 16, - "N-C": 8 - }, - 20 - ], - "pass": true, - "fails_when": "the verdicts cannot even reproduce the manifest's own expected answers, so every negative below would be measuring the fixture rather than the rule", - "row": "" - }, - { - "test": "...and every fixture is evaluated on EVERY required route (A36.6), 2 per control", - "expected": { - "N-A": { - "ai": 8, - "human": 8 - }, - "N-B": { - "ai": 8, - "human": 8 - }, - "N-C": { - "ai": 4, - "human": 4 - } - }, - "observed": { - "N-A": { - "ai": 8, - "human": 8 - }, - "N-B": { - "ai": 8, - "human": 8 - }, - "N-C": { - "ai": 4, - "human": 4 - } - }, - "pass": true, - "fails_when": "a control class is missing from an answer route, so it cannot bound that route -- and a route-specific failure would be invisible", - "row": "" - }, - { - "test": "R8 NEGATIVE -- an N-A answer differing only by a whitespace run FAILS as ABSENT", - "expected": [ - "FAIL", - "EXPECTED_TARGET_ABSENT", - true, - true - ], - "observed": [ - "FAIL", - "EXPECTED_TARGET_ABSENT", - true, - true - ], - "pass": true, - "fails_when": "the comparison normalises whitespace, so a WELD or SPLIT control passes whether or not the adjudicator saw the alteration -- the control becomes decoration", - "row": "" - }, - { - "test": "R8 NEGATIVE -- an absent N-A target FAILS", - "expected": [ - "FAIL", - "EXPECTED_TARGET_ABSENT" - ], - "observed": [ - "FAIL", - "EXPECTED_TARGET_ABSENT" - ], - "pass": true, - "fails_when": "a control whose mutated heading was never transcribed still passes, so M2/M3 keep their licence on an oracle that cannot see the failure class", - "row": "" - }, - { - "test": "R8 NEGATIVE -- an absent N-B target FAILS even though a plausible heading is present", - "expected": [ - "FAIL", - "EXPECTED_TARGET_ABSENT", - 1 - ], - "observed": [ - "FAIL", - "EXPECTED_TARGET_ABSENT", - 1 - ], - "pass": true, - "fails_when": "N-B passes on any non-empty answer, so the corroborated heading is not actually being checked -- and N-B is what establishes the adjudicator is reliable at all", - "row": "" - }, - { - "test": "R8 NEGATIVE -- a DUPLICATED N-A target FAILS rather than counting as found", - "expected": [ - "FAIL", - "EXPECTED_TARGET_DUPLICATED" - ], - "observed": [ - "FAIL", - "EXPECTED_TARGET_DUPLICATED" - ], - "pass": true, - "fails_when": "'at least once' is accepted, so an adjudicator reporting the same heading twice satisfies a control whose truth names exactly one occurrence", - "row": "" - }, - { - "test": "R8 NEGATIVE -- ONE fabricated heading FAILS N-C", - "expected": [ - "FAIL", - "HEADING_REPORTED_IN_HEADING_FREE_REGION" - ], - "observed": [ - "FAIL", - "HEADING_REPORTED_IN_HEADING_FREE_REGION" - ], - "pass": true, - "fails_when": "over-triggering passes, so every precision claim keeps its licence while the oracle invents headings -- the exact condition section 5.6 voids precision for", - "row": "" - }, - { - "test": "R8 -- one route cannot mask the other: AI passes, human fails, the KIND fails", - "expected": [ - "FAIL", - 8, - 7, - [ - "human" - ] - ], - "observed": [ - "FAIL", - 8, - 7, - [ - "human" - ] - ], - "pass": true, - "fails_when": "a passing route averages away a failing one, so a control that failed on the route whose labels are actually consumed still reads PASS", - "row": "" - }, - { - "test": "...and a kind PASS requires EVERY fixture on EVERY route -- no tolerance, no percentage", - "expected": [ - true, - "no tolerance" - ], - "observed": [ - true, - "no tolerance" - ], - "pass": true, - "fails_when": "the aggregation admits a threshold or a percentage, which the frozen Rule 3 blockers do not", - "row": "" - }, - { - "test": "R8 NEGATIVE -- swapping two controls' expected truths cannot self-certify", - "expected": [ - "FAIL", - "FAIL" - ], - "observed": [ - "FAIL", - "FAIL" - ], - "pass": true, - "fails_when": "truth attached to the wrong control still passes, which would mean the verdict is not reading the committed truth at all", - "row": "" - }, - { - "test": "...and nothing here is malformed, so the failure is about BINDING rather than shape", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "the swap produced a malformed record, so the control would fail for the wrong reason", - "row": "" - }, - { - "test": "R8 -- deleting ONE N-A control and both its answers REFUSES, it does not certify 7/7", - "expected": [ - "ScoreInputError", - "CONTROL_POPULATION_INCOMPLETE" - ], - "observed": [ - "ScoreInputError", - "CONTROL_POPULATION_INCOMPLETE" - ], - "pass": true, - "fails_when": "a coherent key missing one control reports 7/7 PASS and satisfies a Rule 3 blocker on a SMALLER census than the protocol froze -- self-certification by omission, and the artifact looks complete from the inside", - "row": "" - }, - { - "test": "...and the same key WITH the control present passes, so the refusal is about completeness", - "expected": "PASS", - "observed": "PASS", - "pass": true, - "fails_when": "the complete population is refused too, so the control above proves nothing", - "row": "" - }, - { - "test": "R8 -- a control declaring only ONE route REFUSES; it is not scored on the survivor", - "expected": [ - "ScoreInputError", - "CONTROL_ROUTE_SET_MISMATCH" - ], - "observed": [ - "ScoreInputError", - "CONTROL_ROUTE_SET_MISMATCH" - ], - "pass": true, - "fails_when": "a control is scored on the route it kept, so the route whose labels are actually consumed goes unchecked while the kind still reports PASS (A36.6 gives every control both routes)", - "row": "" - }, - { - "test": "R8 -- a DUPLICATED control identity REFUSES even though the counts still look right", - "expected": [ - "ScoreInputError", - "CONTROL_IDENTITY_DUPLICATED" - ], - "observed": [ - "ScoreInputError", - "CONTROL_IDENTITY_DUPLICATED" - ], - "pass": true, - "fails_when": "identity uniqueness is not checked, so one real fixture goes unexercised while another is scored twice and the 8/8/4 census still reads correct", - "row": "" - }, - { - "test": "the frozen census and required routes are STATED in the artifact, not implicit", - "expected": [ - { - "N-A": 8, - "N-B": 8, - "N-C": 4 - }, - [ - "ai", - "human" - ], - true - ], - "observed": [ - { - "N-A": 8, - "N-B": 8, - "N-C": 4 - }, - [ - "ai", - "human" - ], - true - ], - "pass": true, - "fails_when": "a reader cannot tell which census was enforced, or whether one was present at all", - "row": "" - }, - { - "test": "a key with NO controls reports NOT_EVALUABLE everywhere rather than PASS", - "expected": [ - { - "N-A": "NOT_EVALUABLE", - "N-B": "NOT_EVALUABLE", - "N-C": "NOT_EVALUABLE" - }, - false - ], - "observed": [ - { - "N-A": "NOT_EVALUABLE", - "N-B": "NOT_EVALUABLE", - "N-C": "NOT_EVALUABLE" - }, - false - ], - "pass": true, - "fails_when": "a control-free key reports PASS on a blocker it never evaluated -- and DEVELOPMENT material legitimately carries no controls, so this path must be honest rather than forbidden", - "row": "" - }, - { - "test": "NEGATIVE -- an N-A record with no committed expected heading REFUSES", - "expected": "CONTROL_TRUTH_MALFORMED", - "observed": "CONTROL_TRUTH_MALFORMED", - "pass": true, - "fails_when": "a control with no truth to check against is silently scored, so a manifest defect becomes a PASS", - "row": "" - }, - { - "test": "no architecture decision is taken here -- statuses only", - "expected": [ - true, - true - ], - "observed": [ - true, - true - ], - "pass": true, - "fails_when": "the scorer applies a Rule 3 consequence to a control status, taking a decision section 7 owns", - "row": "" - }, - { - "test": "M6 appears NOWHERE in the finished payload -- no key, no string, at any depth", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a key or value naming M6 is in the result-bearing schema. Section 5 owns 'M0-M9 minus M6', so even a value that says DEFERRED puts a deferred metric in an artifact a consumer reads -- and invites it to be reserved, looked up, or filled in later", - "row": "" - }, - { - "test": "...and the scanner FINDS a planted M6 key, so the emptiness above is not vacuous", - "expected": [ - 2, - [ - "KEY", - "KEY" - ] - ], - "observed": [ - 2, - [ - "KEY", - "KEY" - ] - ], - "pass": true, - "fails_when": "the scanner cannot see an M6 surface even when one is planted in a finished payload, so its clean result proves nothing", - "row": "" - }, - { - "test": "...and it finds one hidden in a nested VALUE too, not only in a key", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "only keys are scanned, so prose reintroducing M6 into the schema would pass", - "row": "" - }, - { - "test": "EVERY per-document result surface carries an explicit I13 qualification field", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a per-document result is reported with no qualification field, so a reader cannot tell whether it was computed on a PDFIUM-conditioned frame -- and I13 requires the label on EVERY RQ1/RQ2 result computed on a failing document, not on a parent block alone", - "row": "" - }, - { - "test": "...and the discovery really found the M0 / M7 / M9 / heading / event / paired surfaces", - "expected": [ - true, - true, - true, - true, - true, - true - ], - "observed": [ - true, - true, - true, - true, - true, - true - ], - "pass": true, - "fails_when": "the content-based discovery missed a surface class, so 'everything is labelled' was asserted over an incomplete set", - "row": "" - }, - { - "test": "a FAILING document is labelled on each surface; a PASSING one carries explicit None", - "expected": [ - "PDFIUM-CONDITIONED FRAME", - "PDFIUM-CONDITIONED FRAME", - "PDFIUM-CONDITIONED FRAME", - null, - null, - null - ], - "observed": [ - "PDFIUM-CONDITIONED FRAME", - "PDFIUM-CONDITIONED FRAME", - "PDFIUM-CONDITIONED FRAME", - null, - null, - null - ], - "pass": true, - "fails_when": "a passing document has an ABSENT field rather than an explicit null, so 'not conditioned' is indistinguishable from 'nobody labelled this'", - "row": "" - }, - { - "test": "the section 8 event vector and EVERY paired-difference detail row carry the label too", - "expected": [ - { - "COND/1": "PDFIUM-CONDITIONED FRAME", - "CLEAN/1": null - }, - true, - 4 - ], - "observed": [ - { - "CLEAN/1": null, - "COND/1": "PDFIUM-CONDITIONED FRAME" - }, - true, - 4 - ], - "pass": true, - "fails_when": "a section 8 row is reported without its qualification, so the statistic's own per-document detail hides which documents were frame-conditioned", - "row": "" - }, - { - "test": "BOTH headline qualifications are emitted from the >1/3 rule, explicitly", - "expected": [ - { - "RQ1": "PDFIUM-CONDITIONED FRAME", - "RQ2": "PDFIUM-CONDITIONED FRAME" - }, - true - ], - "observed": [ - { - "RQ1": "PDFIUM-CONDITIONED FRAME", - "RQ2": "PDFIUM-CONDITIONED FRAME" - }, - true - ], - "pass": true, - "fails_when": "the headline qualifications are left for a reader to derive from a boolean, so a report can omit them without contradicting the artifact", - "row": "" - }, - { - "test": "...and 1 of 2 failing really is MORE than a third, so the fixture exercises the rule", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the fixture does not cross the one-third boundary, so the headline check passed by accident", - "row": "" - }, - { - "test": "cross-engine remains REPORTING-ONLY -- it reaches no decision input", - "expected": [ - false, - false, - false - ], - "observed": [ - false, - false, - false - ], - "pass": true, - "fails_when": "a qualification field reaches a gate or a decision input, which A27.6 forbids: x09 qualifies reporting and blocks nothing", - "row": "" - }, - { - "test": "the injected letter-spaced page is DETECTED for the arm that split it", - "expected": [ - 1, - 1, - 0 - ], - "observed": [ - 1, - 1, - 0 - ], - "pass": true, - "fails_when": "M7 does not detect an injected `R E P O R T`, so phase 2's flip condition cannot be evaluated on fresh data at all", - "row": "negative: inject an R E P O R T page" - }, - { - "test": "...and the matching text is PRINTED, not merely counted", - "expected": "R E P O R T", - "observed": "R E P O R T", - "pass": true, - "fails_when": "only a count is reported, so a reader cannot tell a real signature from a coincidence", - "row": "negative: inject an R E P O R T page" - }, - { - "test": "NEGATIVE -- 2 single-character tokens do NOT match and 3 DO: the threshold is live", - "expected": [ - 1, - [ - false, - true - ], - [ - 2, - 3 - ] - ], - "observed": [ - 1, - [ - false, - true - ], - [ - 2, - 3 - ] - ], - "pass": true, - "fails_when": "the frozen `>= 3 single-character tokens` condition is not what is implemented, so M7 either fires on ordinary prose or cannot fire at all", - "row": "negative: inject an R E P O R T page" - }, - { - "test": "...and an ordinary heading matches nothing, so M7 is not simply always true", - "expected": [ - 0, - 0 - ], - "observed": [ - 0, - 0 - ], - "pass": true, - "fails_when": "M7 fires on an ordinary account heading, so its rate carries no information", - "row": "negative: inject an R E P O R T page" - }, - { - "test": "M7's denominator is the arm's COMPLETE emitted census (100 % of the holdout)", - "expected": [ - 2, - 1 - ], - "observed": [ - 2, - 1 - ], - "pass": true, - "fails_when": "M7 is computed over an adjudicated subset, which section 6 does not license -- it is a self-signature over each architecture's own output", - "row": "" - }, - { - "test": "an arm that emitted nothing reports VACUOUS, not a zero rate", - "expected": [ - "VACUOUS", - null - ], - "observed": [ - "VACUOUS", - null - ], - "pass": true, - "fails_when": "a zero-denominator M7 is printed as 0.0, which reads as 'no display splits observed'", - "row": "vacuity" - }, - { - "test": "a ONE-line margin deficit fires, and names the loser -- no tolerance", - "expected": [ - "H", - true, - 1 - ], - "observed": [ - "H", - true, - 1 - ], - "pass": true, - "fails_when": "a tolerance was introduced, so 'loses margin-numbered lines' became 'loses more than N' -- choosing the sensitivity of a decision rule the frozen text does not parameterise", - "row": "" - }, - { - "test": "...and an EQUAL count does not fire, so the clause is not simply always true", - "expected": [ - null, - false, - 0 - ], - "observed": [ - null, - false, - 0 - ], - "pass": true, - "fails_when": "the margin clause fires on identical counts, which would reject an arm for nothing", - "row": "" - }, - { - "test": "no Rule 0 OUTCOME is emitted here -- only the facts the rule reads", - "expected": [ - null, - true - ], - "observed": [ - null, - true - ], - "pass": true, - "fails_when": "the scorer decided Rule 0, taking a decision section 7 owns and A27.4 gave its own outcome enum", - "row": "" - }, - { - "test": "the band clause fires for exactly ONE losing arm, and NOT when both lose", - "expected": [ - "H", - true, - null, - false - ], - "observed": [ - "H", - true, - null, - false - ], - "pass": true, - "fails_when": "a shared failure is read as an asymmetric loss, which would reject an arm on evidence that distinguishes nothing (section 7.2 rule 0's both-lose branch)", - "row": "" - }, - { - "test": "the coverage clause bites exactly at the frozen 0.85 floor", - "expected": [ - "H", - true, - 0.85 - ], - "observed": [ - "H", - true, - 0.85 - ], - "pass": true, - "fails_when": "the 0.85 floor moved, so an arm keeps or loses a document's whole heading tree by a threshold this study invented", - "row": "" - }, - { - "test": "NEGATIVE -- a committed coverage floor other than production's 0.85 REFUSES", - "expected": "COVERAGE_FLOOR_DRIFT", - "observed": "COVERAGE_FLOOR_DRIFT", - "pass": true, - "fails_when": "a moved threshold is scored silently, which is a DEVIATION presented as a result", - "row": "" - }, - { - "test": "NEGATIVE -- a committed verdict that contradicts its own floor REFUSES", - "expected": "COVERAGE_FLOOR_DRIFT", - "observed": "COVERAGE_FLOOR_DRIFT", - "pass": true, - "fails_when": "the scorer trusts `coverage_meets_floor` over the coverage and the floor beside it", - "row": "" - }, - { - "test": "section 8.1's own fixture reproduces: 14 documents 0.1926 vs 600 headings 0.00498", - "expected": [ - 0.1926, - 0.00498, - 39 - ], - "observed": [ - 0.1926, - 0.00498, - 39 - ], - "pass": true, - "fails_when": "the two bounds agree, so nothing here could detect headings being treated as iid trials -- the 39x overstatement section 8.1 measured", - "row": "section 8 independence" - }, - { - "test": "N is the number of DOCUMENTS scored, and the bound is the document-unit one", - "expected": [ - 14, - 0, - "document", - 0.1926361757 - ], - "observed": [ - 14, - 0, - "document", - 0.1926361757 - ], - "pass": true, - "fails_when": "N counts headings, regions or stimuli, so the bound asserts an independence the design explicitly denies", - "row": "section 8 independence" - }, - { - "test": "NEGATIVE -- a heading-level table (one document, many rows) REFUSES", - "expected": "DUPLICATE_DOCUMENT_IDENTITY", - "observed": "DUPLICATE_DOCUMENT_IDENTITY", - "pass": true, - "fails_when": "a heading-as-rows table is accepted and silently double-weights a document, which is exactly how a per-heading denominator gets in", - "row": "section 8 independence" - }, - { - "test": "at zero events NO bootstrap is reported, and the reason is explicit", - "expected": [ - false, - "ZERO_EVENTS_BOOTSTRAP_REFUSED", - false - ], - "observed": [ - false, - "ZERO_EVENTS_BOOTSTRAP_REFUSED", - false - ], - "pass": true, - "fails_when": "a bootstrap is reported at zero events, where 8.1 measured the percentile interval is [0.0, 0.0] and carries no information about a new document", - "row": "section 8 zero-event" - }, - { - "test": "...and the zero-event bound IS the frozen closed form, to the last bit", - "expected": [ - 0.19263617565013536, - true - ], - "observed": [ - 0.19263617565013536, - true - ], - "pass": true, - "fails_when": "the closed form is not `1 - 0.05 ** (1/N)`, so the one number section 8 froze verbatim was recomputed by something else", - "row": "section 8 zero-event" - }, - { - "test": "...and the general bisection AGREES with the closed form at k = 0, so it is not a special case", - "expected": [ - true, - true, - true - ], - "observed": [ - true, - true, - true - ], - "pass": true, - "fails_when": "the general Clopper-Pearson path disagrees with the frozen closed form, so one of the two is wrong and only the zero-event case would ever reveal it", - "row": "section 8 zero-event" - }, - { - "test": "...and the reported wording is an OBSERVATION plus a bound, with an explicit disclaimer", - "expected": [ - true, - true, - { - "the architectures agree": false, - "are identical": false, - "produced identical": false, - "the rate is zero.": false - } - ], - "observed": [ - true, - true, - { - "the architectures agree": false, - "are identical": false, - "produced identical": false, - "the rate is zero.": false - } - ], - "pass": true, - "fails_when": "the zero-event sentence asserts equivalence, or carries no disclaimer, so a loose bound reads as 'the architectures agree' -- the phrasing section 7.2 forbids", - "row": "section 8 zero-event" - }, - { - "test": "one discordant document gives 1/14, a REPORTED non-gating bootstrap, and a wider bound", - "expected": [ - 1, - 14, - true, - false, - true - ], - "observed": [ - 1, - 14, - true, - false, - true - ], - "pass": true, - "fails_when": "an event does not move the bound or the bootstrap, so section 8 is not reading the per-document evidence at all", - "row": "" - }, - { - "test": "...and the bootstrap interval is deterministic across runs", - "expected": [ - 0.0, - 0.21428571428571427 - ], - "observed": [ - 0.0, - 0.21428571428571427 - ], - "pass": true, - "fails_when": "the interval moves between runs, so no reported figure is reproducible", - "row": "" - }, - { - "test": "a P-robust document is EXCLUDED from the section 8 statistic and named", - "expected": [ - 14, - 0, - [ - "ROBUST/1" - ] - ], - "observed": [ - 14, - 0, - [ - "ROBUST/1" - ] - ], - "pass": true, - "fails_when": "a P-robust document enters a heading-level statistic section 4.4.1 claims nothing about, inflating N and reusing a draw sequence that is not its own", - "row": "" - }, - { - "test": "...and that document really DID carry an event, so the exclusion is not vacuous", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the excluded document had no event anyway, so 'the numbers did not move' proves nothing", - "row": "" - }, - { - "test": "the event is HEADING-level: a body-line text difference is not a section 8 event", - "expected": [ - false, - 0, - 1 - ], - "observed": [ - false, - 0, - 1 - ], - "pass": true, - "fails_when": "a line-level difference on ordinary body text counts as heading-level discordance, so section 8 would answer a different question under its own name", - "row": "" - }, - { - "test": "a P-robust document with a discordance is ABSENT from the P-head per_document vector", - "expected": [ - false, - 14, - 14 - ], - "observed": [ - false, - 14, - 14 - ], - "pass": true, - "fails_when": "an excluded P-robust row sits inside the vector that documents N and the event count, so a reader (or a later consumer) counts it as a member of a statistic it is barred from", - "row": "" - }, - { - "test": "...and it remains explicitly REPORTED, under a name that cannot be read as membership", - "expected": [ - [ - "ROBUST/1" - ], - [ - "ROBUST/1" - ], - true - ], - "observed": [ - [ - "ROBUST/1" - ], - [ - "ROBUST/1" - ], - true - ], - "pass": true, - "fails_when": "the exclusion is silent, so nothing records that a document was held out of a heading statistic at all", - "row": "" - }, - { - "test": "...and adding it changes NO reported figure: N, events, bound and bootstrap all hold", - "expected": [ - 14, - 0, - 0.19263617565013536, - false - ], - "observed": [ - 14, - 0, - 0.19263617565013536, - false - ], - "pass": true, - "fails_when": "a P-robust document moved a reported section 8 figure, so the P-head restriction is decoration rather than a filter", - "row": "" - }, - { - "test": "R9 -- exactly the two non-constant numeric M9 basis quantities are paired", - "expected": [ - "coverage", - "n_margin_numbered_lines" - ], - "observed": [ - "coverage", - "n_margin_numbered_lines" - ], - "pass": true, - "fails_when": "a quantity outside R9's ruling is paired. M1-M5 and M7 can be VACUOUS, so pairing them needs an unruled missingness policy; the booleans are Rule 0's; coverage_floor is a constant; and the glyph-size count is a diagnostic rather than A39.1's quantity", - "row": "section 8 pairing" - }, - { - "test": "the per-document differences and the UNWEIGHTED mean are exact, over documents", - "expected": [ - 10, - 20, - 15.0, - 2 - ], - "observed": [ - 10, - 20, - 15.0, - 2 - ], - "pass": true, - "fails_when": "the mean is not the unweighted mean of the per-document differences", - "row": "section 8 pairing" - }, - { - "test": "...and the heading-count-WEIGHTED mean differs, so the fixture can tell them apart", - "expected": [ - 17.5, - true - ], - "observed": [ - 17.5, - true - ], - "pass": true, - "fails_when": "both means agree on this fixture, so 'unweighted' was never actually tested", - "row": "section 8 pairing" - }, - { - "test": "P-head and P-robust are reported SEPARATELY, with no combined mean anywhere", - "expected": [ - [ - "P-head", - "P-robust" - ], - 15.0, - 100.0, - true - ], - "observed": [ - [ - "P-head", - "P-robust" - ], - 15.0, - 100.0, - true - ], - "pass": true, - "fails_when": "the two populations are pooled into one mean, mixing a heading-bearing population with one the study claims no heading metric on -- and section 4.4.1 never pools them", - "row": "section 8 pairing" - }, - { - "test": "...and the POOLED mean is a different number that appears NOWHERE in the payload", - "expected": [ - 43.333333333333336, - false - ], - "observed": [ - 43.333333333333336, - false - ], - "pass": true, - "fails_when": "the pooled 3-document mean is reported somewhere in the artifact, so a reader can quote a figure the population rule forbids", - "row": "section 8 pairing" - }, - { - "test": "per-document detail is MANDATORY: every document in each population has a row", - "expected": [ - 2, - 1, - 2, - 1 - ], - "observed": [ - 2, - 1, - 2, - 1 - ], - "pass": true, - "fails_when": "only the mean is reported, so a large single-document delta cannot be distinguished from a population-wide shift", - "row": "section 8 pairing" - }, - { - "test": "coverage pairs too, and its P-head mean is the unweighted 0.0 (+0.05 and -0.05)", - "expected": [ - 0.0, - 2 - ], - "observed": [ - -0.0, - 2 - ], - "pass": true, - "fails_when": "the second R9 quantity is not actually paired, or its mean is not unweighted", - "row": "section 8 pairing" - }, - { - "test": "no M1-M5 or M7 quantity is present in the paired block", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a potentially VACUOUS metric is paired, which requires a missingness rule R9 declined to invent -- and the old implementation dropped VACUOUS documents silently", - "row": "section 8 pairing" - }, - { - "test": "only account / agency / grouping occurrences count, and both arms count ONCE", - "expected": [ - 3, - [ - "account", - "agency", - "grouping" - ] - ], - "observed": [ - 3, - [ - "account", - "agency", - "grouping" - ] - ], - "pass": true, - "fails_when": "the adequacy denominator is widened to title/division/section, which would make the holdout look more adequate than the frozen quantity it is compared against", - "row": "" - }, - { - "test": "a P-robust document adds NOTHING to the occurrence count (A28.1 via A30.4)", - "expected": 3, - "observed": 3, - "pass": true, - "fails_when": "a P-robust document inflates the adequacy count, and a larger count reads as MORE adequate -- the caller-obligation hole A30.4 closed", - "row": "" - }, - { - "test": "NEGATIVE -- an EMPTY strata mapping REFUSES instead of reporting a false INADEQUATE", - "expected": "STRATUM_MISSING", - "observed": "STRATUM_MISSING", - "pass": true, - "fails_when": "an empty mapping bypasses the check, so section 4.5 counts zero strata filled and prints INADEQUATE -- turning input the caller never supplied into a finding about the holdout", - "row": "" - }, - { - "test": "NEGATIVE -- ONE missing document in a multi-document set REFUSES", - "expected": "STRATUM_MISSING", - "observed": "STRATUM_MISSING", - "pass": true, - "fails_when": "a partially supplied mapping is accepted, so the strata-filled count silently undercounts and the adequacy verdict is computed on a population that was never fully described", - "row": "" - }, - { - "test": "...and a COMPLETE mapping scores, so the refusals are not refusing everything", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "a complete mapping is refused too, so the two negatives prove nothing about completeness", - "row": "" - }, - { - "test": "the A28.2 state machine's verdict is reported as a fact, with its owner named", - "expected": [ - "INADEQUATE", - true - ], - "observed": [ - "INADEQUATE", - true - ], - "pass": true, - "fails_when": "the scorer applies Rule 3's consequence, taking a decision A27.6 assigns elsewhere", - "row": "" - }, - { - "test": "...and the machine really can return the other states, so INADEQUATE is not hardcoded", - "expected": [ - "GENERALISABLE", - 8 - ], - "observed": [ - "GENERALISABLE", - 8 - ], - "pass": true, - "fails_when": "every population is INADEQUATE, so the verdict carries no information", - "row": "" - }, - { - "test": "a failing document is labelled, and the label never blocks", - "expected": [ - "PDFIUM-CONDITIONED FRAME", - null, - false - ], - "observed": [ - "PDFIUM-CONDITIONED FRAME", - null, - false - ], - "pass": true, - "fails_when": "a cross-engine failure blocks the decision, which A27.6 forbids -- x09 qualifies reporting and nothing else", - "row": "" - }, - { - "test": "EXACTLY one third does not qualify both headlines; MORE than one third does", - "expected": [ - false, - true, - true - ], - "observed": [ - false, - true, - true - ], - "pass": true, - "fails_when": "the one-third rule is implemented as >= rather than >, so a compliant run acquires a qualification it did not earn (or the reverse)", - "row": "" - }, - { - "test": "NEGATIVE -- a scored document missing from the cross-engine artifact REFUSES", - "expected": "CROSS_ENGINE_DOCUMENT_MISSING", - "observed": "CROSS_ENGINE_DOCUMENT_MISSING", - "pass": true, - "fails_when": "a document is scored with no cross-engine verdict, so it could never acquire the qualification the rule attaches to it -- the silent-escape case max(1, ...) exists for", - "row": "" - }, - { - "test": "the exactly-matching population SCORES, so the refusals below are not refusing everything", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "an exact cross-engine population is refused, making every negative below meaningless", - "row": "" - }, - { - "test": "NEGATIVE -- one EXTRA passing row REFUSES rather than diluting the one-third fraction", - "expected": "CROSS_ENGINE_EXTRA_DOCUMENT", - "observed": "CROSS_ENGINE_EXTRA_DOCUMENT", - "pass": true, - "fails_when": "an unscored passing document enlarges the denominator of I13's `> 1/3` rule, so a run that earned the both-headlines qualification can have it diluted away", - "row": "" - }, - { - "test": "NEGATIVE -- one EXTRA failing row REFUSES rather than manufacturing the qualification", - "expected": "CROSS_ENGINE_EXTRA_DOCUMENT", - "observed": "CROSS_ENGINE_EXTRA_DOCUMENT", - "pass": true, - "fails_when": "an unscored FAILING document can push a clean run over `> 1/3` and qualify both headlines on evidence about a document the study did not score", - "row": "" - }, - { - "test": "NEGATIVE -- a DUPLICATED document with conflicting verdicts REFUSES, never order-dependent", - "expected": "CROSS_ENGINE_DUPLICATE_DOCUMENT", - "observed": "CROSS_ENGINE_DUPLICATE_DOCUMENT", - "pass": true, - "fails_when": "two rows for one document are both consumed, so the reported verdict depends on which row is read last -- a silent order dependence in a reported qualification", - "row": "" - }, - { - "test": "...and the reverse row order refuses IDENTICALLY, so the refusal is not itself ordered", - "expected": "CROSS_ENGINE_DUPLICATE_DOCUMENT", - "observed": "CROSS_ENGINE_DUPLICATE_DOCUMENT", - "pass": true, - "fails_when": "the duplicate refusal depends on row order, so one arrangement would still be accepted", - "row": "" - }, - { - "test": "NEGATIVE -- a row missing `passed` REFUSES structurally, before any verdict is read", - "expected": [ - "ScoreInputError", - "CROSS_ENGINE_ROW_MALFORMED" - ], - "observed": [ - "ScoreInputError", - "CROSS_ENGINE_ROW_MALFORMED" - ], - "pass": true, - "fails_when": "a row with no verdict reaches `qualification`, where an absent key either raises far from the cause or -- with a `.get` default -- invents a verdict for a real document", - "row": "" - }, - { - "test": "NEGATIVE -- a row missing `document` REFUSES too, deterministically and not as a KeyError", - "expected": [ - "ScoreInputError", - "CROSS_ENGINE_ROW_MALFORMED" - ], - "observed": [ - "ScoreInputError", - "CROSS_ENGINE_ROW_MALFORMED" - ], - "pass": true, - "fails_when": "an anonymous row is counted into I13's denominator, so a verdict about nothing changes the both-headlines qualification", - "row": "" - }, - { - "test": "the refusal happens at INPUT VALIDATION, before any qualification is computed", - "expected": [ - "CROSS_ENGINE_EXTRA_DOCUMENT", - "validate_inputs" - ], - "observed": [ - "CROSS_ENGINE_EXTRA_DOCUMENT", - "validate_inputs" - ], - "pass": true, - "fails_when": "the population check runs after qualification, so a wrong denominator is computed and only then rejected -- and a partial payload may already have been produced", - "row": "" - }, - { - "test": "every malformed or incomplete input REFUSES with its own reason", - "expected": { - "MISSING_REQUIRED_FIELD": "MISSING_REQUIRED_FIELD", - "DUPLICATE_DOCUMENT_IDENTITY": "DUPLICATE_DOCUMENT_IDENTITY", - "UNKNOWN_POPULATION": "UNKNOWN_POPULATION", - "OCCURRENCE_RECORD_INCOMPLETE": "OCCURRENCE_RECORD_INCOMPLETE", - "DUPLICATE_OCCURRENCE_KEY": "DUPLICATE_OCCURRENCE_KEY", - "FRAMES_NOT_A_SEQUENCE": "FRAMES_NOT_A_SEQUENCE", - "STIMULUS_DOCUMENT_NOT_IN_FRAMES": "STIMULUS_DOCUMENT_NOT_IN_FRAMES", - "ADJUDICATION_ROUTE_MISSING": "ADJUDICATION_ROUTE_MISSING", - "CONTROL_STIMULUS_IN_ESTIMAND": "CONTROL_STIMULUS_IN_ESTIMAND" - }, - "observed": { - "MISSING_REQUIRED_FIELD": "MISSING_REQUIRED_FIELD", - "DUPLICATE_DOCUMENT_IDENTITY": "DUPLICATE_DOCUMENT_IDENTITY", - "UNKNOWN_POPULATION": "UNKNOWN_POPULATION", - "OCCURRENCE_RECORD_INCOMPLETE": "OCCURRENCE_RECORD_INCOMPLETE", - "DUPLICATE_OCCURRENCE_KEY": "DUPLICATE_OCCURRENCE_KEY", - "FRAMES_NOT_A_SEQUENCE": "FRAMES_NOT_A_SEQUENCE", - "STIMULUS_DOCUMENT_NOT_IN_FRAMES": "STIMULUS_DOCUMENT_NOT_IN_FRAMES", - "ADJUDICATION_ROUTE_MISSING": "ADJUDICATION_ROUTE_MISSING", - "CONTROL_STIMULUS_IN_ESTIMAND": "CONTROL_STIMULUS_IN_ESTIMAND" - }, - "pass": true, - "fails_when": "a malformed record is skipped, scored, or refused under the wrong reason -- and a skipped record moves a denominator with nothing to show for it", - "row": "" - }, - { - "test": "a duplicated document is refused by the SCORER, not only by the section 8 helper", - "expected": [ - "ScoreInputError", - "DUPLICATE_DOCUMENT_IDENTITY" - ], - "observed": [ - "ScoreInputError", - "DUPLICATE_DOCUMENT_IDENTITY" - ], - "pass": true, - "fails_when": "the scorer's own duplicate-document guard is gone and the refusal is coming from the frozen section 8 vector validator instead -- which shares the reason STRING, so the check above cannot see the difference and stays green with the guard deleted", - "row": "" - }, - { - "test": "...and the equivalent WELL-FORMED input still scores, so nothing above refuses blindly", - "expected": null, - "observed": null, - "pass": true, - "fails_when": "valid input is refused too, so the refusals prove nothing about malformedness", - "row": "" - }, - { - "test": "ATTACK -- changing an occurrence's identity while keeping the COUNT identical is caught", - "expected": [ - 2, - 0 - ], - "observed": [ - 2, - 0 - ], - "pass": true, - "fails_when": "the join counts records rather than identities, so a heading matched to the wrong occurrence scores as a match", - "row": "" - }, - { - "test": "ATTACK -- swapping two document identities MOVES the per-document evidence", - "expected": [ - { - "A/1": true, - "B/1": false - }, - { - "A/1": false, - "B/1": true - } - ], - "observed": [ - { - "A/1": true, - "B/1": false - }, - { - "A/1": false, - "B/1": true - } - ], - "pass": true, - "fails_when": "the per-document record does not follow the document identity, so a finding could be reported against the wrong bill", - "row": "" - }, - { - "test": "...while the aggregate is unchanged, which is why the per-document detail is mandatory", - "expected": [ - 1, - 2 - ], - "observed": [ - 1, - 2 - ], - "pass": true, - "fails_when": "the aggregate moved too, so this attack could have been caught by a total alone", - "row": "" - }, - { - "test": "ATTACK -- zeroing the frame's own `counts` summary changes NOTHING", - "expected": 16, - "observed": 16, - "pass": true, - "fails_when": "the scorer reads the committed summary instead of the records, so a wrong summary silently becomes the reported result", - "row": "" - }, - { - "test": "ATTACK -- reversing the document order gives a byte-identical payload", - "expected": "{\"adequacy_4_5\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 2, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"control_verdicts\": {\"aggregation\": \"a kind PASSES only if every fixture passes on every required route; no tolerance\", \"by_kind\": {\"N-A\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-B\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-C\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}}, \"comparison\": \"exact raw string equality; NO normalisation (R8)\", \"decision_owner\": \"decide_architecture -- Rule 3's gate vector (A27.6) consumes these three statuses; they are the FACTS, not the decision\", \"frozen_population\": {\"N-A\": 8, \"N-B\": 8, \"N-C\": 4}, \"n_controls\": 0, \"per_control\": [], \"population_present\": false, \"required_routes\": [\"ai\", \"human\"], \"ruled_by\": \"A41.2 R8\"}, \"cross_engine_qualification\": {\"both_headlines_qualified\": false, \"decision_blocking\": false, \"failed_documents\": [], \"headline_qualifications\": {\"RQ1\": null, \"RQ2\": null}, \"n_documents\": 2, \"n_failed\": 0, \"per_document\": {\"SYNTHETIC/1\": null, \"SYNTHETIC/2\": null}, \"threshold_owner\": \"x09_skeleton_cross_engine.gate (document 0.95 / page 0.75)\"}, \"decision_owner\": \"decide_architecture (section 7) -- no Rule 0/1/3 outcome appears here\", \"decision_taken_here\": false, \"documents_scored\": [\"SYNTHETIC/1\", \"SYNTHETIC/2\"], \"heading_population_notes\": {\"estimand_purposes\": [\"c_metrics\", \"d_decision\", \"c_audit\"], \"excluded_stimuli\": {\"control\": 0, \"r1_repeat\": 0}, \"pooling_rule\": \"C and D are separate estimands (A36.3); they are never summed together\", \"routing\": {\"c_metrics\": \"ai\", \"d_decision\": \"human\"}}, \"headings_pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"owns\": \"PRE-REGISTRATION section 6 metrics M0, M1, M2, M3, M4, M5, M7 and M9; section 8; HARNESS-PLAN section 5\", \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 16, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 16, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 16, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 16, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 16, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 2, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 16, \"qualification\": null, \"risk_set\": 16, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}, \"SYNTHETIC/2\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 8, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 8, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 8, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 8, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 8, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 1, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 8, \"qualification\": null, \"risk_set\": 8, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 0, \"rate\": {\"denominator\": 0, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"VACUOUS\", \"value\": null}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 0, \"rate\": {\"denominator\": 0, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"VACUOUS\", \"value\": null}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/2\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"r1_reliability\": {\"aggregation\": \"heading-occurrence micro-average per route; worst required route (R6.4)\", \"decision_owner\": \"Rule 3 gate vector (A27.6); no consequence is applied here\", \"matching\": \"one-to-one on uniquely resolved A30 occurrence keys; symmetric union denominator (R6.1)\", \"n_pairs\": 0, \"pairs\": [], \"per_route\": {}, \"pooled_across_routes\": false, \"role\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}, \"role_comparison\": \"exact fine section 5.3 role; UNREADABLE never agrees (R6.3)\", \"ruled_by\": \"A41.2 R6\", \"text\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9}, \"text_comparison\": \"EXACT equality of the values as returned -- no normalisation (R6.2)\"}, \"s1\": {\"advance_scale\": 1.25, \"fires\": true, \"gate_owner\": \"Rule 3 gate vector (A27.6)\", \"m0_reportable\": true, \"m0_status\": \"REPORTED\"}, \"schema\": \"metrics/1\", \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.7763932022500211, \"events\": 0, \"gating\": false, \"n_documents\": 2, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.7763932022500211, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 2, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 2, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}, {\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/2\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 2, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}, {\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/2\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}, {\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/2\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 2 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.7764. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "observed": "{\"adequacy_4_5\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 2, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"control_verdicts\": {\"aggregation\": \"a kind PASSES only if every fixture passes on every required route; no tolerance\", \"by_kind\": {\"N-A\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-B\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-C\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}}, \"comparison\": \"exact raw string equality; NO normalisation (R8)\", \"decision_owner\": \"decide_architecture -- Rule 3's gate vector (A27.6) consumes these three statuses; they are the FACTS, not the decision\", \"frozen_population\": {\"N-A\": 8, \"N-B\": 8, \"N-C\": 4}, \"n_controls\": 0, \"per_control\": [], \"population_present\": false, \"required_routes\": [\"ai\", \"human\"], \"ruled_by\": \"A41.2 R8\"}, \"cross_engine_qualification\": {\"both_headlines_qualified\": false, \"decision_blocking\": false, \"failed_documents\": [], \"headline_qualifications\": {\"RQ1\": null, \"RQ2\": null}, \"n_documents\": 2, \"n_failed\": 0, \"per_document\": {\"SYNTHETIC/1\": null, \"SYNTHETIC/2\": null}, \"threshold_owner\": \"x09_skeleton_cross_engine.gate (document 0.95 / page 0.75)\"}, \"decision_owner\": \"decide_architecture (section 7) -- no Rule 0/1/3 outcome appears here\", \"decision_taken_here\": false, \"documents_scored\": [\"SYNTHETIC/1\", \"SYNTHETIC/2\"], \"heading_population_notes\": {\"estimand_purposes\": [\"c_metrics\", \"d_decision\", \"c_audit\"], \"excluded_stimuli\": {\"control\": 0, \"r1_repeat\": 0}, \"pooling_rule\": \"C and D are separate estimands (A36.3); they are never summed together\", \"routing\": {\"c_metrics\": \"ai\", \"d_decision\": \"human\"}}, \"headings_pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"owns\": \"PRE-REGISTRATION section 6 metrics M0, M1, M2, M3, M4, M5, M7 and M9; section 8; HARNESS-PLAN section 5\", \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 16, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 16, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 16, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 16, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 16, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 2, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 16, \"qualification\": null, \"risk_set\": 16, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}, \"SYNTHETIC/2\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 8, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 8, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 8, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 8, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 8, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 1, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 8, \"qualification\": null, \"risk_set\": 8, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 0, \"rate\": {\"denominator\": 0, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"VACUOUS\", \"value\": null}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 0, \"rate\": {\"denominator\": 0, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"VACUOUS\", \"value\": null}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/2\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"r1_reliability\": {\"aggregation\": \"heading-occurrence micro-average per route; worst required route (R6.4)\", \"decision_owner\": \"Rule 3 gate vector (A27.6); no consequence is applied here\", \"matching\": \"one-to-one on uniquely resolved A30 occurrence keys; symmetric union denominator (R6.1)\", \"n_pairs\": 0, \"pairs\": [], \"per_route\": {}, \"pooled_across_routes\": false, \"role\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}, \"role_comparison\": \"exact fine section 5.3 role; UNREADABLE never agrees (R6.3)\", \"ruled_by\": \"A41.2 R6\", \"text\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9}, \"text_comparison\": \"EXACT equality of the values as returned -- no normalisation (R6.2)\"}, \"s1\": {\"advance_scale\": 1.25, \"fires\": true, \"gate_owner\": \"Rule 3 gate vector (A27.6)\", \"m0_reportable\": true, \"m0_status\": \"REPORTED\"}, \"schema\": \"metrics/1\", \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.7763932022500211, \"events\": 0, \"gating\": false, \"n_documents\": 2, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.7763932022500211, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 2, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 2, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}, {\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/2\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 2, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}, {\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/2\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}, {\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/2\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 2 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.7764. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "pass": true, - "fails_when": "the caller's listing order changes a reported number, so two runs over the same population print different results", - "row": "" - }, - { - "test": "...and reversing the PAGE order leaves every M0 quantity identical", - "expected": { - "risk_set": 16, - "M0a_text": 0, - "M0b_segmentation": 0, - "both_absent": 0 - }, - "observed": { - "risk_set": 16, - "M0a_text": 0, - "M0b_segmentation": 0, - "both_absent": 0 - }, - "pass": true, - "fails_when": "page order changes M0, so the artifact's page ordering is load-bearing when it must not be", - "row": "" - }, - { - "test": "the unattacked payload is reproducible, so 'it changed' is evidence", - "expected": "{\"adequacy_4_5\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 2, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"control_verdicts\": {\"aggregation\": \"a kind PASSES only if every fixture passes on every required route; no tolerance\", \"by_kind\": {\"N-A\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-B\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-C\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}}, \"comparison\": \"exact raw string equality; NO normalisation (R8)\", \"decision_owner\": \"decide_architecture -- Rule 3's gate vector (A27.6) consumes these three statuses; they are the FACTS, not the decision\", \"frozen_population\": {\"N-A\": 8, \"N-B\": 8, \"N-C\": 4}, \"n_controls\": 0, \"per_control\": [], \"population_present\": false, \"required_routes\": [\"ai\", \"human\"], \"ruled_by\": \"A41.2 R8\"}, \"cross_engine_qualification\": {\"both_headlines_qualified\": false, \"decision_blocking\": false, \"failed_documents\": [], \"headline_qualifications\": {\"RQ1\": null, \"RQ2\": null}, \"n_documents\": 1, \"n_failed\": 0, \"per_document\": {\"SYNTHETIC/1\": null}, \"threshold_owner\": \"x09_skeleton_cross_engine.gate (document 0.95 / page 0.75)\"}, \"decision_owner\": \"decide_architecture (section 7) -- no Rule 0/1/3 outcome appears here\", \"decision_taken_here\": false, \"documents_scored\": [\"SYNTHETIC/1\"], \"heading_population_notes\": {\"estimand_purposes\": [\"c_metrics\", \"d_decision\", \"c_audit\"], \"excluded_stimuli\": {\"control\": 0, \"r1_repeat\": 0}, \"pooling_rule\": \"C and D are separate estimands (A36.3); they are never summed together\", \"routing\": {\"c_metrics\": \"ai\", \"d_decision\": \"human\"}}, \"headings_pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"owns\": \"PRE-REGISTRATION section 6 metrics M0, M1, M2, M3, M4, M5, M7 and M9; section 8; HARNESS-PLAN section 5\", \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 16, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 16, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 16, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 16, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 16, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 2, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 16, \"qualification\": null, \"risk_set\": 16, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"r1_reliability\": {\"aggregation\": \"heading-occurrence micro-average per route; worst required route (R6.4)\", \"decision_owner\": \"Rule 3 gate vector (A27.6); no consequence is applied here\", \"matching\": \"one-to-one on uniquely resolved A30 occurrence keys; symmetric union denominator (R6.1)\", \"n_pairs\": 0, \"pairs\": [], \"per_route\": {}, \"pooled_across_routes\": false, \"role\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}, \"role_comparison\": \"exact fine section 5.3 role; UNREADABLE never agrees (R6.3)\", \"ruled_by\": \"A41.2 R6\", \"text\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9}, \"text_comparison\": \"EXACT equality of the values as returned -- no normalisation (R6.2)\"}, \"s1\": {\"advance_scale\": 1.25, \"fires\": true, \"gate_owner\": \"Rule 3 gate vector (A27.6)\", \"m0_reportable\": true, \"m0_status\": \"REPORTED\"}, \"schema\": \"metrics/1\", \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.95, \"events\": 0, \"gating\": false, \"n_documents\": 1, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.95, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 1, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 1, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 1, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 1 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.9500. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "observed": "{\"adequacy_4_5\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 2, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"control_verdicts\": {\"aggregation\": \"a kind PASSES only if every fixture passes on every required route; no tolerance\", \"by_kind\": {\"N-A\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-B\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-C\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}}, \"comparison\": \"exact raw string equality; NO normalisation (R8)\", \"decision_owner\": \"decide_architecture -- Rule 3's gate vector (A27.6) consumes these three statuses; they are the FACTS, not the decision\", \"frozen_population\": {\"N-A\": 8, \"N-B\": 8, \"N-C\": 4}, \"n_controls\": 0, \"per_control\": [], \"population_present\": false, \"required_routes\": [\"ai\", \"human\"], \"ruled_by\": \"A41.2 R8\"}, \"cross_engine_qualification\": {\"both_headlines_qualified\": false, \"decision_blocking\": false, \"failed_documents\": [], \"headline_qualifications\": {\"RQ1\": null, \"RQ2\": null}, \"n_documents\": 1, \"n_failed\": 0, \"per_document\": {\"SYNTHETIC/1\": null}, \"threshold_owner\": \"x09_skeleton_cross_engine.gate (document 0.95 / page 0.75)\"}, \"decision_owner\": \"decide_architecture (section 7) -- no Rule 0/1/3 outcome appears here\", \"decision_taken_here\": false, \"documents_scored\": [\"SYNTHETIC/1\"], \"heading_population_notes\": {\"estimand_purposes\": [\"c_metrics\", \"d_decision\", \"c_audit\"], \"excluded_stimuli\": {\"control\": 0, \"r1_repeat\": 0}, \"pooling_rule\": \"C and D are separate estimands (A36.3); they are never summed together\", \"routing\": {\"c_metrics\": \"ai\", \"d_decision\": \"human\"}}, \"headings_pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"owns\": \"PRE-REGISTRATION section 6 metrics M0, M1, M2, M3, M4, M5, M7 and M9; section 8; HARNESS-PLAN section 5\", \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 16, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 16, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 16, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 16, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 16, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 2, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 16, \"qualification\": null, \"risk_set\": 16, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"r1_reliability\": {\"aggregation\": \"heading-occurrence micro-average per route; worst required route (R6.4)\", \"decision_owner\": \"Rule 3 gate vector (A27.6); no consequence is applied here\", \"matching\": \"one-to-one on uniquely resolved A30 occurrence keys; symmetric union denominator (R6.1)\", \"n_pairs\": 0, \"pairs\": [], \"per_route\": {}, \"pooled_across_routes\": false, \"role\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}, \"role_comparison\": \"exact fine section 5.3 role; UNREADABLE never agrees (R6.3)\", \"ruled_by\": \"A41.2 R6\", \"text\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9}, \"text_comparison\": \"EXACT equality of the values as returned -- no normalisation (R6.2)\"}, \"s1\": {\"advance_scale\": 1.25, \"fires\": true, \"gate_owner\": \"Rule 3 gate vector (A27.6)\", \"m0_reportable\": true, \"m0_status\": \"REPORTED\"}, \"schema\": \"metrics/1\", \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.95, \"events\": 0, \"gating\": false, \"n_documents\": 1, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.95, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 1, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 1, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 1, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 1 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.9500. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "pass": true, - "fails_when": "the scorer is not deterministic, so no attack above can be distinguished from noise", - "row": "" - }, - { - "test": "the fixture really contains R1 repeats and real controls, so the invariance is not vacuous", - "expected": [ - true, - 20 - ], - "observed": [ - true, - 20 - ], - "pass": true, - "fails_when": "no repeat or control is present, so 'adding them changed nothing' was guaranteed", - "row": "" - }, - { - "test": "adding 20 REAL controls moves no metric, no denominator and no section 8 figure", - "expected": "{\"adequacy\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 12, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 12, \"M0_any_rate\": {\"denominator\": 96, \"metric\": \"M0-any\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 0.125}, \"M0_any_rate_ALL_LINES_superseded\": 0.125, \"M0a_only_text\": 12, \"M0a_text\": 12, \"M0a_text_rate\": {\"denominator\": 96, \"metric\": \"M0a\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 0.125}, \"M0b_defined\": 96, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 96, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 96, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 12, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 96, \"qualification\": null, \"risk_set\": 96, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 12, \"rate\": {\"denominator\": 12, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 12, \"rate\": {\"denominator\": 12, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 8, \"metric\": \"M1 precision\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 8, \"metric\": \"M1 recall\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 8, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 8, \"metric\": \"M1 precision\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 8, \"metric\": \"M1 recall\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 8, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 8, \"metric\": \"M2\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M2\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 8, \"metric\": \"M3\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M3\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 8, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 8, \"metric\": \"M4\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M4\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 8, \"metric\": \"M5\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 8, \"metric\": \"M5\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 8, \"X\": 8}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 8, \"X\": 8}, \"m3_boundary\": {\"H\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 8, \"X\": 8}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 8, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 8, \"X\": 8}, \"m4_correct\": {\"H\": 8, \"X\": 8}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 8, \"X\": 8}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 8, \"X\": 8}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 8, \"X\": 8}, \"n_adjudicated\": 8, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 8, \"X\": 8}, \"n_matched\": {\"H\": 8, \"X\": 8}, \"n_stimuli\": 8}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}, \"D\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 12, \"metric\": \"M1 precision\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 12, \"metric\": \"M1 recall\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 12, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 12, \"metric\": \"M1 precision\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 12, \"metric\": \"M1 recall\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 12, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 12, \"metric\": \"M2\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M2\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 12, \"metric\": \"M3\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M3\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 12, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 12, \"metric\": \"M4\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M4\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 12, \"metric\": \"M5\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 12, \"metric\": \"M5\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 12, \"X\": 12}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 12, \"X\": 12}, \"m3_boundary\": {\"H\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 12, \"X\": 12}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 12, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 12, \"X\": 12}, \"m4_correct\": {\"H\": 12, \"X\": 12}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 12, \"X\": 12}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 12, \"X\": 12}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 12, \"X\": 12}, \"n_adjudicated\": 12, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 12, \"X\": 12}, \"n_matched\": {\"H\": 12, \"X\": 12}, \"n_stimuli\": 12}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 8, \"metric\": \"M1 precision\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 8, \"metric\": \"M1 recall\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 8, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 8, \"metric\": \"M1 precision\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 8, \"metric\": \"M1 recall\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 8, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 8, \"metric\": \"M2\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M2\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 8, \"metric\": \"M3\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M3\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 8, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 8, \"metric\": \"M4\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M4\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 8, \"metric\": \"M5\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 8, \"metric\": \"M5\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 8, \"X\": 8}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 8, \"X\": 8}, \"m3_boundary\": {\"H\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 8, \"X\": 8}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 8, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 8, \"X\": 8}, \"m4_correct\": {\"H\": 8, \"X\": 8}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 8, \"X\": 8}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 8, \"X\": 8}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 8, \"X\": 8}, \"n_adjudicated\": 8, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 8, \"X\": 8}, \"n_matched\": {\"H\": 8, \"X\": 8}, \"n_stimuli\": 8}, \"r1_text_gate\": {\"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}, \"D\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 12, \"metric\": \"M1 precision\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 12, \"metric\": \"M1 recall\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 12, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 12, \"metric\": \"M1 precision\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 12, \"metric\": \"M1 recall\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 12, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 12, \"metric\": \"M2\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M2\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 12, \"metric\": \"M3\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M3\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 12, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 12, \"metric\": \"M4\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M4\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 12, \"metric\": \"M5\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 12, \"metric\": \"M5\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 12, \"X\": 12}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 12, \"X\": 12}, \"m3_boundary\": {\"H\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 12, \"X\": 12}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 12, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 12, \"X\": 12}, \"m4_correct\": {\"H\": 12, \"X\": 12}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 12, \"X\": 12}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 12, \"X\": 12}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 12, \"X\": 12}, \"n_adjudicated\": 12, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 12, \"X\": 12}, \"n_matched\": {\"H\": 12, \"X\": 12}, \"n_stimuli\": 12}, \"r1_text_gate\": {\"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.95, \"events\": 0, \"gating\": false, \"n_documents\": 1, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.95, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 1, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 1, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 1, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 1 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.9500. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "observed": "{\"adequacy\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 12, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 12, \"M0_any_rate\": {\"denominator\": 96, \"metric\": \"M0-any\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 0.125}, \"M0_any_rate_ALL_LINES_superseded\": 0.125, \"M0a_only_text\": 12, \"M0a_text\": 12, \"M0a_text_rate\": {\"denominator\": 96, \"metric\": \"M0a\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 0.125}, \"M0b_defined\": 96, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 96, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 96, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 12, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 96, \"qualification\": null, \"risk_set\": 96, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 12, \"rate\": {\"denominator\": 12, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 12, \"rate\": {\"denominator\": 12, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 8, \"metric\": \"M1 precision\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 8, \"metric\": \"M1 recall\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 8, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 8, \"metric\": \"M1 precision\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 8, \"metric\": \"M1 recall\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 8, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 8, \"metric\": \"M2\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M2\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 8, \"metric\": \"M3\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M3\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 8, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 8, \"metric\": \"M4\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M4\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 8, \"metric\": \"M5\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 8, \"metric\": \"M5\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 8, \"X\": 8}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 8, \"X\": 8}, \"m3_boundary\": {\"H\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 8, \"X\": 8}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 8, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 8, \"X\": 8}, \"m4_correct\": {\"H\": 8, \"X\": 8}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 8, \"X\": 8}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 8, \"X\": 8}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 8, \"X\": 8}, \"n_adjudicated\": 8, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 8, \"X\": 8}, \"n_matched\": {\"H\": 8, \"X\": 8}, \"n_stimuli\": 8}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}, \"D\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 12, \"metric\": \"M1 precision\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 12, \"metric\": \"M1 recall\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 12, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 12, \"metric\": \"M1 precision\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 12, \"metric\": \"M1 recall\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 12, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 12, \"metric\": \"M2\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M2\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 12, \"metric\": \"M3\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M3\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 12, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 12, \"metric\": \"M4\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M4\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 12, \"metric\": \"M5\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 12, \"metric\": \"M5\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 12, \"X\": 12}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 12, \"X\": 12}, \"m3_boundary\": {\"H\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 12, \"X\": 12}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 12, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 12, \"X\": 12}, \"m4_correct\": {\"H\": 12, \"X\": 12}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 12, \"X\": 12}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 12, \"X\": 12}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 12, \"X\": 12}, \"n_adjudicated\": 12, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 12, \"X\": 12}, \"n_matched\": {\"H\": 12, \"X\": 12}, \"n_stimuli\": 12}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 8, \"metric\": \"M1 precision\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 8, \"metric\": \"M1 recall\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 8, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 8, \"metric\": \"M1 precision\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 8, \"metric\": \"M1 recall\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 8, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 8, \"metric\": \"M2\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M2\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 8, \"metric\": \"M3\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M3\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 8, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 8, \"metric\": \"M4\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 8, \"metric\": \"M4\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 8, \"metric\": \"M5\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 8, \"metric\": \"M5\", \"numerator\": 8, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 8, \"X\": 8}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 8, \"X\": 8}, \"m3_boundary\": {\"H\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 144, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 8, \"X\": 8}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 8, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 8, \"X\": 8}, \"m4_correct\": {\"H\": 8, \"X\": 8}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 8, \"X\": 8}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 8, \"X\": 8}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 8, \"X\": 8}, \"n_adjudicated\": 8, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 8, \"X\": 8}, \"n_matched\": {\"H\": 8, \"X\": 8}, \"n_stimuli\": 8}, \"r1_text_gate\": {\"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}, \"D\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 12, \"metric\": \"M1 precision\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 12, \"metric\": \"M1 recall\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 12, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 12, \"metric\": \"M1 precision\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 12, \"metric\": \"M1 recall\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 12, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 12, \"metric\": \"M2\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M2\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 12, \"metric\": \"M3\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M3\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 12, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 12, \"metric\": \"M4\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 12, \"metric\": \"M4\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 12, \"metric\": \"M5\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 12, \"metric\": \"M5\", \"numerator\": 12, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.8}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 12, \"X\": 12}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 12, \"X\": 12}, \"m3_boundary\": {\"H\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 216, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 12, \"X\": 12}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 12, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 12, \"X\": 12}, \"m4_correct\": {\"H\": 12, \"X\": 12}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 12, \"X\": 12}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 12, \"X\": 12}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 12, \"X\": 12}, \"n_adjudicated\": 12, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 12, \"X\": 12}, \"n_matched\": {\"H\": 12, \"X\": 12}, \"n_stimuli\": 12}, \"r1_text_gate\": {\"observed_by_route\": {\"ai\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}, \"human\": {\"denominator\": 1, \"numerator\": 1, \"ratio\": 1.0, \"status\": \"PASS\", \"threshold\": 0.9}}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"PASS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.95, \"events\": 0, \"gating\": false, \"n_documents\": 1, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.95, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 1, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 1, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 1, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 1 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.9500. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "pass": true, - "fails_when": "a control contributed to an estimand it exists only to bound, which would put deliberately altered text into a correctness denominator", - "row": "" - }, - { - "test": "...and the exclusions are COUNTED rather than silent", - "expected": [ - 20, - 1 - ], - "observed": [ - 20, - 1 - ], - "pass": true, - "fails_when": "the excluded stimuli are invisible in the artifact, so a reader cannot see what was left out of the denominators", - "row": "" - }, - { - "test": "an R1 repeat does not double-count its primary's region, in EITHER frame", - "expected": { - "C": [ - 8, - 8 - ], - "D": [ - 12, - 12 - ] - }, - "observed": { - "C": [ - 8, - 8 - ], - "D": [ - 12, - 12 - ] - }, - "pass": true, - "fails_when": "a repeat is scored as an independent stimulus, so one physical occurrence enters a denominator twice -- and the repeat exists to measure the ADJUDICATOR, not the arms", - "row": "" - }, - { - "test": "...and the two frames really are different sizes here, so C and D are not interchangeable", - "expected": [ - 8, - 12, - true - ], - "observed": [ - 8, - 12, - true - ], - "pass": true, - "fails_when": "C and D have the same population in this fixture, so a scorer that pooled them or read the wrong one would look correct", - "row": "" - }, - { - "test": "two runs over the same inputs produce byte-identical output", - "expected": "{\"adequacy_4_5\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 2, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"control_verdicts\": {\"aggregation\": \"a kind PASSES only if every fixture passes on every required route; no tolerance\", \"by_kind\": {\"N-A\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-B\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-C\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}}, \"comparison\": \"exact raw string equality; NO normalisation (R8)\", \"decision_owner\": \"decide_architecture -- Rule 3's gate vector (A27.6) consumes these three statuses; they are the FACTS, not the decision\", \"frozen_population\": {\"N-A\": 8, \"N-B\": 8, \"N-C\": 4}, \"n_controls\": 0, \"per_control\": [], \"population_present\": false, \"required_routes\": [\"ai\", \"human\"], \"ruled_by\": \"A41.2 R8\"}, \"cross_engine_qualification\": {\"both_headlines_qualified\": false, \"decision_blocking\": false, \"failed_documents\": [], \"headline_qualifications\": {\"RQ1\": null, \"RQ2\": null}, \"n_documents\": 1, \"n_failed\": 0, \"per_document\": {\"SYNTHETIC/1\": null}, \"threshold_owner\": \"x09_skeleton_cross_engine.gate (document 0.95 / page 0.75)\"}, \"decision_owner\": \"decide_architecture (section 7) -- no Rule 0/1/3 outcome appears here\", \"decision_taken_here\": false, \"documents_scored\": [\"SYNTHETIC/1\"], \"heading_population_notes\": {\"estimand_purposes\": [\"c_metrics\", \"d_decision\", \"c_audit\"], \"excluded_stimuli\": {\"control\": 0, \"r1_repeat\": 0}, \"pooling_rule\": \"C and D are separate estimands (A36.3); they are never summed together\", \"routing\": {\"c_metrics\": \"ai\", \"d_decision\": \"human\"}}, \"headings_pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"owns\": \"PRE-REGISTRATION section 6 metrics M0, M1, M2, M3, M4, M5, M7 and M9; section 8; HARNESS-PLAN section 5\", \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 16, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 16, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 16, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 16, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 16, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 2, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 16, \"qualification\": null, \"risk_set\": 16, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"r1_reliability\": {\"aggregation\": \"heading-occurrence micro-average per route; worst required route (R6.4)\", \"decision_owner\": \"Rule 3 gate vector (A27.6); no consequence is applied here\", \"matching\": \"one-to-one on uniquely resolved A30 occurrence keys; symmetric union denominator (R6.1)\", \"n_pairs\": 0, \"pairs\": [], \"per_route\": {}, \"pooled_across_routes\": false, \"role\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}, \"role_comparison\": \"exact fine section 5.3 role; UNREADABLE never agrees (R6.3)\", \"ruled_by\": \"A41.2 R6\", \"text\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9}, \"text_comparison\": \"EXACT equality of the values as returned -- no normalisation (R6.2)\"}, \"s1\": {\"advance_scale\": 1.25, \"fires\": true, \"gate_owner\": \"Rule 3 gate vector (A27.6)\", \"m0_reportable\": true, \"m0_status\": \"REPORTED\"}, \"schema\": \"metrics/1\", \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.95, \"events\": 0, \"gating\": false, \"n_documents\": 1, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.95, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 1, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 1, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 1, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 1 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.9500. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "observed": "{\"adequacy_4_5\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 2, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"control_verdicts\": {\"aggregation\": \"a kind PASSES only if every fixture passes on every required route; no tolerance\", \"by_kind\": {\"N-A\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-B\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-C\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}}, \"comparison\": \"exact raw string equality; NO normalisation (R8)\", \"decision_owner\": \"decide_architecture -- Rule 3's gate vector (A27.6) consumes these three statuses; they are the FACTS, not the decision\", \"frozen_population\": {\"N-A\": 8, \"N-B\": 8, \"N-C\": 4}, \"n_controls\": 0, \"per_control\": [], \"population_present\": false, \"required_routes\": [\"ai\", \"human\"], \"ruled_by\": \"A41.2 R8\"}, \"cross_engine_qualification\": {\"both_headlines_qualified\": false, \"decision_blocking\": false, \"failed_documents\": [], \"headline_qualifications\": {\"RQ1\": null, \"RQ2\": null}, \"n_documents\": 1, \"n_failed\": 0, \"per_document\": {\"SYNTHETIC/1\": null}, \"threshold_owner\": \"x09_skeleton_cross_engine.gate (document 0.95 / page 0.75)\"}, \"decision_owner\": \"decide_architecture (section 7) -- no Rule 0/1/3 outcome appears here\", \"decision_taken_here\": false, \"documents_scored\": [\"SYNTHETIC/1\"], \"heading_population_notes\": {\"estimand_purposes\": [\"c_metrics\", \"d_decision\", \"c_audit\"], \"excluded_stimuli\": {\"control\": 0, \"r1_repeat\": 0}, \"pooling_rule\": \"C and D are separate estimands (A36.3); they are never summed together\", \"routing\": {\"c_metrics\": \"ai\", \"d_decision\": \"human\"}}, \"headings_pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"owns\": \"PRE-REGISTRATION section 6 metrics M0, M1, M2, M3, M4, M5, M7 and M9; section 8; HARNESS-PLAN section 5\", \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 16, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 16, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 16, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 16, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 16, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 2, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 16, \"qualification\": null, \"risk_set\": 16, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"r1_reliability\": {\"aggregation\": \"heading-occurrence micro-average per route; worst required route (R6.4)\", \"decision_owner\": \"Rule 3 gate vector (A27.6); no consequence is applied here\", \"matching\": \"one-to-one on uniquely resolved A30 occurrence keys; symmetric union denominator (R6.1)\", \"n_pairs\": 0, \"pairs\": [], \"per_route\": {}, \"pooled_across_routes\": false, \"role\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}, \"role_comparison\": \"exact fine section 5.3 role; UNREADABLE never agrees (R6.3)\", \"ruled_by\": \"A41.2 R6\", \"text\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9}, \"text_comparison\": \"EXACT equality of the values as returned -- no normalisation (R6.2)\"}, \"s1\": {\"advance_scale\": 1.25, \"fires\": true, \"gate_owner\": \"Rule 3 gate vector (A27.6)\", \"m0_reportable\": true, \"m0_status\": \"REPORTED\"}, \"schema\": \"metrics/1\", \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.95, \"events\": 0, \"gating\": false, \"n_documents\": 1, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.95, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 1, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 1, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 1, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 1 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.9500. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "pass": true, - "fails_when": "the scorer is not deterministic, so no reported number is reproducible", - "row": "" - }, - { - "test": "scoring COMMITTED JSON alone reproduces the same payload -- no PDF, no runner", - "expected": "{\"adequacy_4_5\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 2, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"control_verdicts\": {\"aggregation\": \"a kind PASSES only if every fixture passes on every required route; no tolerance\", \"by_kind\": {\"N-A\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-B\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-C\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}}, \"comparison\": \"exact raw string equality; NO normalisation (R8)\", \"decision_owner\": \"decide_architecture -- Rule 3's gate vector (A27.6) consumes these three statuses; they are the FACTS, not the decision\", \"frozen_population\": {\"N-A\": 8, \"N-B\": 8, \"N-C\": 4}, \"n_controls\": 0, \"per_control\": [], \"population_present\": false, \"required_routes\": [\"ai\", \"human\"], \"ruled_by\": \"A41.2 R8\"}, \"cross_engine_qualification\": {\"both_headlines_qualified\": false, \"decision_blocking\": false, \"failed_documents\": [], \"headline_qualifications\": {\"RQ1\": null, \"RQ2\": null}, \"n_documents\": 1, \"n_failed\": 0, \"per_document\": {\"SYNTHETIC/1\": null}, \"threshold_owner\": \"x09_skeleton_cross_engine.gate (document 0.95 / page 0.75)\"}, \"decision_owner\": \"decide_architecture (section 7) -- no Rule 0/1/3 outcome appears here\", \"decision_taken_here\": false, \"documents_scored\": [\"SYNTHETIC/1\"], \"heading_population_notes\": {\"estimand_purposes\": [\"c_metrics\", \"d_decision\", \"c_audit\"], \"excluded_stimuli\": {\"control\": 0, \"r1_repeat\": 0}, \"pooling_rule\": \"C and D are separate estimands (A36.3); they are never summed together\", \"routing\": {\"c_metrics\": \"ai\", \"d_decision\": \"human\"}}, \"headings_pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"owns\": \"PRE-REGISTRATION section 6 metrics M0, M1, M2, M3, M4, M5, M7 and M9; section 8; HARNESS-PLAN section 5\", \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 16, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 16, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 16, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 16, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 16, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 2, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 16, \"qualification\": null, \"risk_set\": 16, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"r1_reliability\": {\"aggregation\": \"heading-occurrence micro-average per route; worst required route (R6.4)\", \"decision_owner\": \"Rule 3 gate vector (A27.6); no consequence is applied here\", \"matching\": \"one-to-one on uniquely resolved A30 occurrence keys; symmetric union denominator (R6.1)\", \"n_pairs\": 0, \"pairs\": [], \"per_route\": {}, \"pooled_across_routes\": false, \"role\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}, \"role_comparison\": \"exact fine section 5.3 role; UNREADABLE never agrees (R6.3)\", \"ruled_by\": \"A41.2 R6\", \"text\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9}, \"text_comparison\": \"EXACT equality of the values as returned -- no normalisation (R6.2)\"}, \"s1\": {\"advance_scale\": 1.25, \"fires\": true, \"gate_owner\": \"Rule 3 gate vector (A27.6)\", \"m0_reportable\": true, \"m0_status\": \"REPORTED\"}, \"schema\": \"metrics/1\", \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.95, \"events\": 0, \"gating\": false, \"n_documents\": 1, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.95, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 1, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 1, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 1, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 1 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.9500. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "observed": "{\"adequacy_4_5\": {\"kinds_counted\": [\"account\", \"agency\", \"grouping\"], \"n_occurrences\": 2, \"owner\": \"decide_architecture consumes this (Rule 3); the verdict itself is a population fact\", \"population_counted\": \"P-head\", \"strata\": [1], \"strata_filled\": 1, \"verdict\": \"INADEQUATE\"}, \"control_verdicts\": {\"aggregation\": \"a kind PASSES only if every fixture passes on every required route; no tolerance\", \"by_kind\": {\"N-A\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-B\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}, \"N-C\": {\"by_route\": {}, \"failures\": [], \"n_passed\": 0, \"n_total\": 0, \"status\": \"NOT_EVALUABLE\"}}, \"comparison\": \"exact raw string equality; NO normalisation (R8)\", \"decision_owner\": \"decide_architecture -- Rule 3's gate vector (A27.6) consumes these three statuses; they are the FACTS, not the decision\", \"frozen_population\": {\"N-A\": 8, \"N-B\": 8, \"N-C\": 4}, \"n_controls\": 0, \"per_control\": [], \"population_present\": false, \"required_routes\": [\"ai\", \"human\"], \"ruled_by\": \"A41.2 R8\"}, \"cross_engine_qualification\": {\"both_headlines_qualified\": false, \"decision_blocking\": false, \"failed_documents\": [], \"headline_qualifications\": {\"RQ1\": null, \"RQ2\": null}, \"n_documents\": 1, \"n_failed\": 0, \"per_document\": {\"SYNTHETIC/1\": null}, \"threshold_owner\": \"x09_skeleton_cross_engine.gate (document 0.95 / page 0.75)\"}, \"decision_owner\": \"decide_architecture (section 7) -- no Rule 0/1/3 outcome appears here\", \"decision_taken_here\": false, \"documents_scored\": [\"SYNTHETIC/1\"], \"heading_population_notes\": {\"estimand_purposes\": [\"c_metrics\", \"d_decision\", \"c_audit\"], \"excluded_stimuli\": {\"control\": 0, \"r1_repeat\": 0}, \"pooling_rule\": \"C and D are separate estimands (A36.3); they are never summed together\", \"routing\": {\"c_metrics\": \"ai\", \"d_decision\": \"human\"}}, \"headings_pooled\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"owns\": \"PRE-REGISTRATION section 6 metrics M0, M1, M2, M3, M4, M5, M7 and M9; section 8; HARNESS-PLAN section 5\", \"per_document\": {\"SYNTHETIC/1\": {\"M0\": {\"M0_any\": 0, \"M0_any_rate\": {\"denominator\": 16, \"metric\": \"M0-any\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0_any_rate_ALL_LINES_superseded\": 0.0, \"M0a_only_text\": 0, \"M0a_text\": 0, \"M0a_text_rate\": {\"denominator\": 16, \"metric\": \"M0a\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_defined\": 16, \"M0b_only_segmentation\": 0, \"M0b_rate_on_defined\": {\"denominator\": 16, \"metric\": \"M0b_on_defined\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0b_segmentation\": 0, \"M0b_segmentation_rate\": {\"denominator\": 16, \"metric\": \"M0b_on_risk_set\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"M0c_anchor_regions\": 0, \"M0c_pooling\": \"REGION-level; never pooled with the line rates (section 5)\", \"M0c_rate\": {\"denominator\": 2, \"metric\": \"M0c\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}, \"both_absent\": 0, \"both_absent_in_any_denominator\": false, \"neutral_lines_in_scope\": 16, \"qualification\": null, \"risk_set\": 16, \"risk_set_definition\": \"neutral lines with state != BOTH_ABSENT (A23, I3)\"}, \"M7\": {\"H\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"X\": {\"instances\": [], \"n_display_split\": 0, \"n_emitted\": 2, \"rate\": {\"denominator\": 2, \"metric\": \"M7\", \"numerator\": 0, \"status\": \"REPORTED\", \"value\": 0.0}}, \"qualification\": null, \"threshold_single_char_tokens\": 3}, \"M9\": {\"H\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"X\": {\"coverage\": 1.0, \"coverage_floor\": 0.85, \"coverage_meets_floor\": true, \"derive_size_bands_returns_a_band\": true, \"n_lines_total\": 140, \"n_margin_numbered_lines\": 120, \"n_margin_numbered_with_glyph_size\": 120, \"rule0_comparison\": \"RAW FACTS ONLY\"}, \"band_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"coverage_floor\": 0.85, \"coverage_loss\": {\"fires\": false, \"h\": true, \"loser\": null, \"x\": true}, \"document\": \"SYNTHETIC/1\", \"margin_line_loss\": {\"deficit\": 0, \"fires\": false, \"h\": 120, \"loser\": null, \"x\": 120}, \"margin_line_quantity\": \"count of Page.lines where line_number is not None (A39.1); no tolerance\", \"population\": \"P-head\", \"qualification\": null, \"rule0_outcome\": null, \"rule0_owner\": \"decide_architecture -- A27.4's outcome enum is deliberately absent here\"}, \"headings_by_frame\": {\"C\": {\"M1\": {\"H\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"X\": {\"precision\": {\"denominator\": 2, \"metric\": \"M1 precision\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall\": {\"denominator\": 2, \"metric\": \"M1 recall\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"recall_sensitivity_excluding_unresolvable\": {\"denominator\": 2, \"metric\": \"M1 recall (SECONDARY)\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}}, \"M2\": {\"H\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M2\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"M3\": {\"boundary_outcomes\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"clean_rate\": {\"H\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M3\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}}, \"consumes\": \"projected text + oracle only -- never a segmentation label (section 5)\", \"decision_unit\": \"heading occurrence (A3); WELD/SPLIT tallies never substitute for it\", \"heading_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}}, \"M4\": {\"H\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"X\": {\"denominator\": 2, \"metric\": \"M4\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_off_region\": 0, \"excluded_unreadable\": 0, \"h_correct_x_wrong\": 0, \"h_correct_x_wrong_occurrences\": [], \"paired_population\": \"headings scored under M4 for BOTH arms; an unmatched heading is M1 recall's miss and M3's TEXT_ERROR, never charged twice here\", \"parent_answers\": \"printed text | NONE | OFF_REGION | UNREADABLE\", \"scope\": \"immediate parent only (A38.4 penultimate breadcrumb element)\", \"veto_owner\": \"decide_architecture -- Rule 1 condition 4; no consequence is applied here\", \"x_correct_h_wrong\": 0}, \"M5\": {\"H\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"X\": {\"agreement\": {\"denominator\": 2, \"metric\": \"M5\", \"numerator\": 2, \"status\": \"REPORTED\", \"value\": 1.0}, \"excluded_unscorable\": 0}, \"licenses\": \"corroboration only -- section 6 forbids M5 deciding anything\", \"r1_role_gate\": {\"evidence\": \"computed from the committed oracle key + adjudications, never supplied\", \"m5_void\": false, \"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}}, \"counts\": {\"m2_exact\": {\"H\": 2, \"X\": 2}, \"m2_excluded_unreadable\": 0, \"m2_scored\": {\"H\": 2, \"X\": 2}, \"m3_boundary\": {\"H\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}, \"X\": {\"OK\": 36, \"SPLIT\": 0, \"TEXT_ERROR\": 0, \"WELD\": 0}}, \"m3_clean\": {\"H\": 2, \"X\": 2}, \"m3_excluded_no_reference\": 0, \"m3_outcomes\": {\"BOTH_CLEAN\": 2, \"BOTH_DIRTY\": 0, \"UNSCORABLE\": 0, \"X_CORRECTS\": 0, \"X_REGRESSES\": 0}, \"m3_scored\": {\"H\": 2, \"X\": 2}, \"m4_correct\": {\"H\": 2, \"X\": 2}, \"m4_excluded_off_region\": 0, \"m4_excluded_unreadable\": 0, \"m4_h_correct_x_wrong\": 0, \"m4_h_correct_x_wrong_keys\": [], \"m4_scored\": {\"H\": 2, \"X\": 2}, \"m4_x_correct_h_wrong\": 0, \"m5_agree\": {\"H\": 2, \"X\": 2}, \"m5_excluded_unreadable\": 0, \"m5_excluded_unscorable\": {\"H\": 0, \"X\": 0}, \"m5_scored\": {\"H\": 2, \"X\": 2}, \"n_adjudicated\": 2, \"n_adjudicated_unresolvable\": 0, \"n_emitted\": {\"H\": 2, \"X\": 2}, \"n_matched\": {\"H\": 2, \"X\": 2}, \"n_stimuli\": 2}, \"qualification\": null, \"r1_text_gate\": {\"observed_by_route\": {}, \"owner\": \"Rule 3 gate vector (A27.6); this module reports the FACTS only\", \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9, \"voids_text_metrics_if_FAIL\": [\"M2\", \"M3\"]}}}, \"population\": \"P-head\", \"qualification\": null, \"stratum\": 1}}, \"r1_reliability\": {\"aggregation\": \"heading-occurrence micro-average per route; worst required route (R6.4)\", \"decision_owner\": \"Rule 3 gate vector (A27.6); no consequence is applied here\", \"matching\": \"one-to-one on uniquely resolved A30 occurrence keys; symmetric union denominator (R6.1)\", \"n_pairs\": 0, \"pairs\": [], \"per_route\": {}, \"pooled_across_routes\": false, \"role\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.8}, \"role_comparison\": \"exact fine section 5.3 role; UNREADABLE never agrees (R6.3)\", \"ruled_by\": \"A41.2 R6\", \"text\": {\"by_route\": {}, \"status\": \"NOT_EVALUABLE_NO_R1_PAIRS\", \"threshold\": 0.9}, \"text_comparison\": \"EXACT equality of the values as returned -- no normalisation (R6.2)\"}, \"s1\": {\"advance_scale\": 1.25, \"fires\": true, \"gate_owner\": \"Rule 3 gate vector (A27.6)\", \"m0_reportable\": true, \"m0_status\": \"REPORTED\"}, \"schema\": \"metrics/1\", \"section8\": {\"bootstrap\": {\"clopper_pearson_upper_bound\": 0.95, \"events\": 0, \"gating\": false, \"n_documents\": 1, \"reason\": \"ZERO_EVENTS_BOOTSTRAP_REFUSED\", \"reported\": false}, \"clopper_pearson_alpha\": 0.05, \"clopper_pearson_upper_bound\": 0.95, \"estimand\": \"pi = P(a document from the target population shows >= 1 heading-level H/X discordance)\", \"events\": 0, \"excluded_diagnostics_not_in_statistic\": [], \"excluded_documents_not_p_head\": [], \"forbidden\": \"no per-heading probability; no heading-as-iid-trial denominator (section 8.3)\", \"independent_unit\": \"document\", \"n_documents\": 1, \"paired\": {\"by_population\": {\"P-head\": {\"coverage\": {\"n_documents\": 1, \"per_document\": [{\"H\": 1.0, \"X\": 1.0, \"difference\": 0.0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}, \"n_margin_numbered_lines\": {\"n_documents\": 1, \"per_document\": [{\"H\": 120, \"X\": 120, \"difference\": 0, \"document\": \"SYNTHETIC/1\", \"population\": \"P-head\", \"qualification\": null}], \"status\": \"REPORTED\", \"unweighted_mean_difference\": 0.0}}}, \"difference\": \"X - H\", \"excluded_quantities\": {\"M1-M5, M7\": \"can be VACUOUS -- pairing them would require an unruled missingness policy\", \"coverage_floor\": \"a frozen constant, so its difference is always zero\", \"derive_size_bands_returns_a_band, coverage_meets_floor\": \"booleans consumed by Rule 0, not numeric\", \"n_margin_numbered_with_glyph_size\": \"a diagnostic; A39.1's quantity is the margin-line count\"}, \"no_combined_mean\": \"P-head and P-robust are reported separately; no 17-document mean exists\", \"populations_never_pooled\": true, \"quantities\": [\"n_margin_numbered_lines\", \"coverage\"], \"quantity_source\": \"frames.m9.{H,X} -- the numeric M9 bases (A38.8/A39.1)\", \"ruled_by\": \"A41.2 R9\", \"weighting\": \"UNWEIGHTED mean over documents (\\u00a78.3); never weighted by heading count\"}, \"per_document\": [{\"definition\": \"any region whose emitted heading (Anchor) sets differ between H and X\", \"document\": \"SYNTHETIC/1\", \"event\": false, \"evidence\": [], \"n_anchor_discordant_regions\": 0, \"population\": \"P-head\", \"qualification\": null}], \"population\": \"P-head\", \"statement\": \"no heading-level discordance observed on 1 P-head document(s); exact one-sided 95 % upper bound on the per-document rate: 0.9500. This is an OBSERVATION, not a claim that the true rate is zero.\", \"status\": \"REPORTED\", \"zero_event_closed_form_used\": true}}", - "pass": true, - "fails_when": "a fact the scorer needs survives only in a live Python object, so it would have to reopen a PDF or rerun an architecture to score", - "row": "" - }, - { - "test": "the REAL producer's frame passes every input validation unchanged", - "expected": [ - "118-hr-8752/1", - "P-head" - ], - "observed": [ - "118-hr-8752/1", - "P-head" - ], - "pass": true, - "fails_when": "the scorer's contract disagrees with what `build_frames` actually emits, so every synthetic fixture above encodes a belief rather than the producer's shape", - "row": "" - }, - { - "test": "M0 is computed over a real, non-empty risk set with BOTH_ABSENT held out", - "expected": [ - true, - true, - true - ], - "observed": [ - true, - true, - true - ], - "pass": true, - "fails_when": "the real document yields no risk set or no chrome, so the M0 controls above ran on material unlike anything the study will score", - "row": "" - }, - { - "test": "the A38.7 join binds real occurrences: every keyed occurrence in a stimulus matches", - "expected": [ - true, - { - "C": 9, - "D": 16 - } - ], - "observed": [ - true, - { - "C": 9, - "D": 16 - } - ], - "pass": true, - "fails_when": "the geometric round trip through `pdf_x_to_pixel` / `pixel_to_pdf_x` does not resolve back to the same glyph on real material, so the join is unproven where it matters", - "row": "" - }, - { - "test": "M9's real facts carry production's own coverage floor", - "expected": [ - 0.85, - true - ], - "observed": [ - 0.85, - true - ], - "pass": true, - "fails_when": "the real M9 basis does not carry the frozen floor, so Rule 0's coverage clause could not be evaluated from committed facts", - "row": "" - }, - { - "test": "writing the CANONICAL metrics.json REFUSES before a valid execution boundary", - "expected": "CONFIRMATORY_WRITE_BEFORE_EXECUTION", - "observed": "CONFIRMATORY_WRITE_BEFORE_EXECUTION", - "pass": true, - "fails_when": "a confirmatory scoring artifact can be created before execution is authorised, which is the one-way boundary A11 exists to enforce", - "row": "" - }, - { - "test": "...and an ordinary path still writes, so the guard is not refusing everything", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the writer refuses every path, so the control above proves nothing about the boundary", - "row": "" - }, - { - "test": "the scorer imports NOTHING outside the frozen consumer allowlist", - "expected": "set()", - "observed": "set()", - "pass": true, - "fails_when": "the scorer imports an architecture, an engine, a runner or the contamination probe, which would make it a producer rather than a consumer of committed facts", - "row": "" - }, - { - "test": "...and it neither runs an arm, re-extracts anchors, nor enumerates the repository", - "expected": { - "run_hybrid.run": false, - "run_extended.run": false, - "x01_contamination": false, - "an engine": false, - "extract_anchors": false, - "filesystem_discovery": false - }, - "observed": { - "run_hybrid.run": false, - "run_extended.run": false, - "x01_contamination": false, - "an engine": false, - "extract_anchors": false, - "filesystem_discovery": false - }, - "pass": true, - "fails_when": "the scorer can run an architecture, re-run anchor recognition, or discover files on disk -- and the contamination evidence is exactly what repo discovery would rewrite", - "row": "" - }, - { - "test": "the scorer IMPORTS with the PDF renderer made unimportable -- and the blocker is live", - "expected": { - "blocker_live": true, - "scorer_imports": true - }, - "observed": { - "blocker_live": true, - "scorer_imports": true - }, - "pass": true, - "fails_when": "importing `score_metrics` re-acquires a PDF renderer, so A38's pure-consumer property holds for the data path but not for the import graph -- or `blocker_live` is False, in which case the renderer was importable after all and this control proves nothing", - "row": "" - }, - { - "test": "...and JSON-only scoring still COMPUTES there: A27.5's own bound, section 8.1's 0.1926", - "expected": 0.1926, - "observed": 0.1926, - "pass": true, - "fails_when": "the scorer imports without a renderer but cannot actually score without one, so the property is cosmetic -- an import that reaches no working function is not a consumer", - "row": "" - }, - { - "test": "...and the frozen `build_oracle` owners are still DELEGATED to, never locally restated", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "a `build_oracle`-owned derivation now answers from a local copy: the dependency was removed by restating a frozen rule, and two copies of a rule are two rules", - "row": "" - }, - { - "test": "the control is DECISIVE: `build_oracle` still imports the renderer at module scope", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "`build_oracle` no longer imports a renderer eagerly, so the renderer-free checks above would pass however `score_metrics` imported it -- they must be re-pointed, not trusted", - "row": "" - }, - { - "test": "NO canonical confirmatory artifact exists after this probe", - "expected": { - "frames.json": false, - "oracle_key.json": false, - "oracle_blind.json": false, - "oracle_adjudicated.json": false, - "s1_control.json": false, - "cross_engine_control.json": false, - "metrics.json": false, - "scores.json": false, - "EXECUTION-START.json": false - }, - "observed": { - "frames.json": false, - "oracle_key.json": false, - "oracle_blind.json": false, - "oracle_adjudicated.json": false, - "s1_control.json": false, - "cross_engine_control.json": false, - "metrics.json": false, - "scores.json": false, - "EXECUTION-START.json": false - }, - "pass": true, - "fails_when": "a canonical artifact was created, which would cross the execution boundary this session may not cross", - "row": "" - }, - { - "test": "every section 5 control row has at least one executable check", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a frozen control row is unimplemented, so the scorer carries a quantity nothing can falsify", - "row": "" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x28_decide_architecture.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x28_decide_architecture.json deleted file mode 100644 index e704c820..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x28_decide_architecture.json +++ /dev/null @@ -1,1390 +0,0 @@ -{ - "population": "SYNTHETIC only -- no holdout opened, nothing adjudicated, no real decision taken", - "contract": "HARNESS-PLAN section 6 (the decider) over PRE-REGISTRATION section 7.2, A5, A10, A20, A27.3, A27.4, A27.6, A28.2, A39.1 and A41", - "question": "does `decide_architecture` implement the frozen Rule 0 / Rule 1 / Rule 3 machinery, and can each result-bearing branch be made to go RED?", - "artifacts_created": "NONE of frames.json, oracle_key.json, oracle_blind.json, oracle_adjudicated.json, s1_control.json, cross_engine_control.json, metrics.json, scores.json, EXECUTION-START.json", - "fixture_caveat": "every payload is real `score_metrics.score(...)` output; Rule 3 gate STATUSES are overwritten to reach later rules and are fixtures, never evidence", - "open_ruling": "Rule 1's M4 condition (A5 row 4 / A20) has NO producer -- the scorer emits per-arm M4 counts only. The verdict is SUPPLIED and its absence REFUSES; see A42", - "section6_rows": { - "synthetic X win (6 corrects, 0 regressions, no M4 regression)": [ - "6 corrections, 0 regressions, no M4 regression -> EXTENDED_BY_RULE_1" - ], - "synthetic 5 corrects and 1 regression": [ - "5 corrections AND 1 regression -> INSUFFICIENT, never EXTENDED_BY_RULE_1, never HYBRID" - ], - "synthetic 5 corrects and 1 M4 regression": [ - "5 corrections, 0 regressions, ONE M4 parent regression -> INSUFFICIENT (A5 row 4 / A20)", - "...and it fires while the per-arm M4 AGGREGATES are equal -- the A42.3 payload" - ], - "synthetic census of 61 regions": [ - "D = 61 regions -> INSUFFICIENT, and Rule 1 is NOT reached" - ], - "synthetic asymmetric M9 loss for H only": [ - "BAND -- H loses the band, X keeps it -> EXTENDED_BY_RULE_0_M9" - ], - "synthetic asymmetric M9 loss on BOTH arms, different documents": [ - "A27.4 -- an asymmetric loss on EACH arm, different documents -> INSUFFICIENT, no ranking" - ], - "any Rule 3 gate set to FAIL": [ - "a FAILING N-A control -> INSUFFICIENT_COMPARATIVE_EVIDENCE", - "a FAILING N-B control -> INSUFFICIENT_COMPARATIVE_EVIDENCE", - "a FAILING N-C control -> INSUFFICIENT_COMPARATIVE_EVIDENCE", - "a FAILING R1 text reliability gate -> INSUFFICIENT", - "a FAILING R1 role reliability gate -> INSUFFICIENT", - "S1 not firing -> INSUFFICIENT (the comparator is not live)", - "a FAILING confirmatory X2A -> INSUFFICIENT", - "a FAILING confirmatory X2B -> INSUFFICIENT" - ], - "x09 set to FAIL": [ - "x09 FAILING on every sampled document changes NO outcome -- it only adds a label" - ], - "synthetic empty census": [ - "an EMPTY census -> HYBRID_BY_PRIOR, and the text makes no empirical claim for hybrid" - ], - "wording gate": [ - "the prior conclusion carries NO comparative-accuracy claim for hybrid", - "the insufficient by veto conclusion carries NO comparative-accuracy claim for hybrid", - "the insufficient by budget conclusion carries NO comparative-accuracy claim for hybrid", - "the insufficient by gate conclusion carries NO comparative-accuracy claim for hybrid", - "the gate FIRES on a planted superiority claim (non-vacuity)" - ], - "M9 one-arm loss": [ - "PRECEDENCE -- Rule 0 fires while Rule 1 would have chosen X: Rule 0 decides" - ] - }, - "evidence": { - "contract": { - "n_metrics_paths": 14, - "n_document_paths": 8 - }, - "rule0": { - "two_sided": { - "H_asymmetric_loss_documents": [ - "SYNTHETIC/1" - ], - "X_asymmetric_loss_documents": [ - "SYNTHETIC/2" - ] - } - }, - "rule3": { - "gate_names": [ - "R1", - "N-A", - "N-B", - "N-C", - "S1", - "X2-a", - "X2-b", - "M9_evaluability", - "adequacy_4_5" - ] - }, - "qualification": { - "n_failed": 1 - }, - "budget": { - "census_60": { - "d_frame_census": 60, - "per_document": { - "SYNTHETIC/1": 60 - }, - "budget": 60, - "within_budget": true, - "n_adjudicated_d_regions": 60, - "census_fully_adjudicated": true, - "unit": "REGION (A27.3); the census is enumerated complete BEFORE any sampling", - "rule": "<= 60 -> Rule 1 may be evaluated; > 60 -> Rule 1 cannot choose X (A10)" - }, - "census_61": { - "d_frame_census": 61, - "per_document": { - "SYNTHETIC/1": 61 - }, - "budget": 60, - "within_budget": false, - "n_adjudicated_d_regions": 61, - "census_fully_adjudicated": true, - "unit": "REGION (A27.3); the census is enumerated complete BEFORE any sampling", - "rule": "<= 60 -> Rule 1 may be evaluated; > 60 -> Rule 1 cannot choose X (A10)" - } - }, - "rule1": { - "conditions": { - "x_corrects_at_least_5": true, - "x_regresses_exactly_0": true, - "no_m4_parent_regression": true - }, - "five_and_one": { - "evaluable": true, - "x_corrects": 5, - "x_regresses": 1, - "m4_h_correct_x_wrong": 0, - "m4_vetoing_occurrences": [], - "thresholds": { - "x_corrects_min": 5, - "x_regresses_max": 0 - }, - "conditions": { - "x_corrects_at_least_5": true, - "x_regresses_exactly_0": false, - "no_m4_parent_regression": true - }, - "all_conditions_hold": false, - "condition_1_holds": true, - "vetoes_failing": [ - "x_regresses_exactly_0" - ], - "decision_unit": "heading occurrence (A3); m3_outcomes, never WELD/SPLIT boundary tallies", - "m6_veto": "STRUCK by A20 -- M6 is deferred and may not veto", - "m4_condition_source": "metrics.headings_pooled.D.M4.h_correct_x_wrong -- the PAIRED per-heading existential (A42.3); the per-arm m4_correct rates are never substituted" - } - }, - "enums": { - "reached": { - "HYBRID_BY_PRIOR": 2, - "EXTENDED_BY_RULE_1": 4, - "INSUFFICIENT_COMPARATIVE_EVIDENCE": 54, - "EXTENDED_BY_RULE_0_M9": 30, - "HYBRID_BY_RULE_0_M9": 30 - } - }, - "wording": { - "prior_conclusion": "Corrected extended glyph did not meet Rule 1's win conditions on the D-frame census. The pre-stated architectural prior therefore stands and hybrid is retained BY PRIOR. No comparative-accuracy finding about hybrid is made or implied." - }, - "refusals": { - "n_refusal_classes": 11 - }, - "injection": { - "faults": [ - { - "fault": "rule1_threshold_lowered", - "anchor": "X_CORRECTS_MIN = 5", - "replacement": "X_CORRECTS_MIN = 4", - "detected": true, - "crashed": false - }, - { - "fault": "regression_tolerance_restored", - "anchor": "\"x_regresses_exactly_0\": regresses == X_REGRESSES_MAX,", - "replacement": "\"x_regresses_exactly_0\": regresses <= 1,", - "detected": true, - "crashed": false - }, - { - "fault": "budget_relaxed_to_120", - "anchor": "D_FRAME_REGION_BUDGET = 60", - "replacement": "D_FRAME_REGION_BUDGET = 120", - "detected": true, - "crashed": false - }, - { - "fault": "m4_veto_disabled", - "anchor": "\"no_m4_parent_regression\": m4_regressions == 0,", - "replacement": "\"no_m4_parent_regression\": True,", - "detected": true, - "crashed": false - }, - { - "fault": "m4_veto_inferred_from_aggregates_the_A42_3_rejected_reading", - "anchor": " m4_regressions = pooled_d[\"M4\"][\"h_correct_x_wrong\"]", - "replacement": " _c = pooled_d[\"counts\"][\"m4_correct\"]\n m4_regressions = max(0, _c[\"H\"] - _c[\"X\"])", - "detected": true, - "crashed": false - }, - { - "fault": "rule0_two_sided_ranking_invented", - "anchor": "if h_documents and x_documents:", - "replacement": "if False:", - "detected": true, - "crashed": false - }, - { - "fault": "not_evaluable_accepted_as_pass", - "anchor": "failing = [name for name, block in ordered.items() if block[\"status\"] != GATE_PASS]", - "replacement": "failing = [name for name, block in ordered.items() if block[\"status\"] == GATE_FAIL]", - "detected": true, - "crashed": false - }, - { - "fault": "five_and_one_collapsed_into_the_prior", - "anchor": "elif rule1_block[\"condition_1_holds\"]:", - "replacement": "elif False:", - "detected": true, - "crashed": false - }, - { - "fault": "rule0_no_longer_supersedes", - "anchor": "elif rule0_block[\"fires\"]:", - "replacement": "elif False:", - "detected": true, - "crashed": false - }, - { - "fault": "wording_gate_disabled", - "anchor": "hits = sorted({p for p in SUPERIORITY_PATTERNS if re.search(p, text, re.IGNORECASE)})", - "replacement": "hits = []", - "detected": true, - "crashed": false - }, - { - "fault": "closed_enum_guard_removed", - "anchor": "if outcome not in ARCHITECTURE_OUTCOMES:", - "replacement": "if False:", - "detected": true, - "crashed": false - } - ], - "n_faults": 11 - }, - "boundary": { - "checked": [ - "results/frames.json", - "results/oracle_key.json", - "results/oracle_blind.json", - "results/oracle_adjudicated.json", - "results/s1_control.json", - "results/cross_engine_control.json", - "results/metrics.json", - "results/scores.json", - "EXECUTION-START.json", - "results/_x28_faulted.py" - ] - } - }, - "tests": [ - { - "test": "every metrics path the decider reads is present in a REAL score_metrics payload", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the decider reads a field name the scorer does not emit -- the mismatch a hand-written fixture cannot see, because it encodes the decider's belief about the producer", - "row": "" - }, - { - "test": "every per-document M9 path the decider reads is present in real scorer output", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "Rule 0 reads an M9 clause field the scorer does not emit", - "row": "" - }, - { - "test": "the PAIRED M4 fact the decider reads is emitted by the real scorer (A42.3)", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the scorer stopped emitting A5 row 4's paired quantity, so Rule 1's fourth condition has no producer again and the decider can only refuse", - "row": "" - }, - { - "test": "every frame path the budget reads is present in real build_frames output", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the A27.3 census is read from a key `build_frames` does not commit", - "row": "" - }, - { - "test": "the contract walker DETECTS an absent path (non-vacuity)", - "expected": [ - null, - false - ], - "observed": [ - null, - false - ], - "pass": true, - "fails_when": "the walker returns 'present' for everything, so the three contract checks are vacuous", - "row": "" - }, - { - "test": "the scorer still declares it took no decision", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "the scorer began deciding Rule 0, so two components own the architecture outcome", - "row": "" - }, - { - "test": "BAND -- H loses the band, X keeps it -> EXTENDED_BY_RULE_0_M9", - "expected": "EXTENDED_BY_RULE_0_M9", - "observed": "EXTENDED_BY_RULE_0_M9", - "pass": true, - "fails_when": "an arm that lost the whole heading tree is not rejected outright, so section 7.2 rule 0's largest available heading failure decides nothing", - "row": "synthetic asymmetric M9 loss for H only" - }, - { - "test": "BAND near-miss -- BOTH arms lose the band -> Rule 0 does NOT fire", - "expected": [ - false, - "HYBRID_BY_PRIOR" - ], - "observed": [ - false, - "HYBRID_BY_PRIOR" - ], - "pass": true, - "fails_when": "a shared failure is read as an asymmetric loss, rejecting an arm on evidence that distinguishes nothing (section 7.2 rule 0's both-lose branch)", - "row": "" - }, - { - "test": "BAND near-miss -- NEITHER arm loses the band -> Rule 0 does not fire", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "Rule 0 fires on a clean document, rejecting an arm for nothing", - "row": "" - }, - { - "test": "COVERAGE -- H at 0.84, X at 0.85 -> HYBRID rejected, EXTENDED_BY_RULE_0_M9", - "expected": "EXTENDED_BY_RULE_0_M9", - "observed": "EXTENDED_BY_RULE_0_M9", - "pass": true, - "fails_when": "the frozen 0.85 floor stopped biting, so an arm keeps a document's heading tree by a threshold this study invented", - "row": "" - }, - { - "test": "COVERAGE near-miss -- BOTH exactly at 0.85 -> the clause does not fire", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "the floor is read as a strict inequality, so a document meeting it exactly is a loss", - "row": "" - }, - { - "test": "COVERAGE mirror -- X at 0.84, H at 0.85 -> HYBRID_BY_RULE_0_M9", - "expected": "HYBRID_BY_RULE_0_M9", - "observed": "HYBRID_BY_RULE_0_M9", - "pass": true, - "fails_when": "Rule 0 is one-directional, so it can only ever reject hybrid", - "row": "" - }, - { - "test": "MARGIN -- a ONE-line deficit for H fires (A39.1: no tolerance) -> EXTENDED_BY_RULE_0_M9", - "expected": "EXTENDED_BY_RULE_0_M9", - "observed": "EXTENDED_BY_RULE_0_M9", - "pass": true, - "fails_when": "a tolerance was introduced, so 'loses margin-numbered lines' became 'loses more than N' -- choosing the sensitivity of a decision rule the frozen text does not parameterise", - "row": "" - }, - { - "test": "MARGIN near-miss -- EQUAL counts do not fire", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "the margin clause fires on identical counts, rejecting an arm for nothing", - "row": "" - }, - { - "test": "SAME document, DIFFERENT clauses (H band, X margin) -> Rule 0 does NOT fire", - "expected": [ - false, - [], - [], - [ - "SYNTHETIC/1" - ], - "HYBRID_BY_PRIOR" - ], - "observed": [ - false, - [], - [], - [ - "SYNTHETIC/1" - ], - "HYBRID_BY_PRIOR" - ], - "pass": true, - "fails_when": "the clauses were read independently instead of per document, so one document both arms lost was counted as an asymmetric loss for each -- rejecting BOTH architectures on evidence section 7.2 rule 0 calls neutral, and inventing A27.4's two-sided branch on a single document when A27.4 requires different ones", - "row": "" - }, - { - "test": "...and BOTH clauses really did fire on that document, so the trap is live", - "expected": [ - "H", - "X" - ], - "observed": [ - "H", - "X" - ], - "pass": true, - "fails_when": "the fixture does not actually produce two opposite clause losses on one document, so the control could not have caught the per-clause reading", - "row": "" - }, - { - "test": "A27.4 -- an asymmetric loss on EACH arm, different documents -> INSUFFICIENT, no ranking", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - [ - "SYNTHETIC/1" - ], - [ - "SYNTHETIC/2" - ] - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - [ - "SYNTHETIC/1" - ], - [ - "SYNTHETIC/2" - ] - ], - "pass": true, - "fails_when": "a severity or count comparison was invented between two rejected arms, which A27.4 explicitly forbids", - "row": "synthetic asymmetric M9 loss on BOTH arms, different documents" - }, - { - "test": "PRECEDENCE -- Rule 0 fires while Rule 1 would have chosen X: Rule 0 decides", - "expected": [ - "HYBRID_BY_RULE_0_M9", - "RULE_0_M9", - false - ], - "observed": [ - "HYBRID_BY_RULE_0_M9", - "RULE_0_M9", - false - ], - "pass": true, - "fails_when": "Rule 1 was consulted after an arm had already been rejected outright -- section 7.2 rule 0 rejects 'regardless of every other metric'", - "row": "M9 one-arm loss" - }, - { - "test": "PRECEDENCE -- Rule 0 fires while Rule 3 gates fail: Rule 0 decides, gates still reported", - "expected": [ - "EXTENDED_BY_RULE_0_M9", - "RULE_0_M9", - false - ], - "observed": [ - "EXTENDED_BY_RULE_0_M9", - "RULE_0_M9", - false - ], - "pass": true, - "fails_when": "either M9 stopped superseding 'everything below' (section 7.2 rule 0), or the failing gates vanished from the artifact when another rule decided", - "row": "" - }, - { - "test": "PRECEDENCE -- the FULL gate vector is emitted whatever decided", - "expected": [ - "R1", - "N-A", - "N-B", - "N-C", - "S1", - "X2-a", - "X2-b", - "M9_evaluability", - "adequacy_4_5" - ], - "observed": [ - "R1", - "N-A", - "N-B", - "N-C", - "S1", - "X2-a", - "X2-b", - "M9_evaluability", - "adequacy_4_5" - ], - "pass": true, - "fails_when": "a gate that failed is invisible in the artifact because Rule 0 took the outcome first", - "row": "" - }, - { - "test": "a FAILING N-A control -> INSUFFICIENT_COMPARATIVE_EVIDENCE", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_3_GATE", - [ - "N-A" - ] - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_3_GATE", - [ - "N-A" - ] - ], - "pass": true, - "fails_when": "a failed N-A Rule 3 blocker does not block, so the study decides on evidence its own controls say is unreliable", - "row": "any Rule 3 gate set to FAIL" - }, - { - "test": "an UNEVALUABLE N-A control is not a pass either (A41.2.1)", - "expected": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "observed": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "pass": true, - "fails_when": "a blocker that was never measured certifies itself, which is exactly the self-consistent-but-incomplete artifact A41.2.1 closed", - "row": "" - }, - { - "test": "a FAILING N-B control -> INSUFFICIENT_COMPARATIVE_EVIDENCE", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_3_GATE", - [ - "N-B" - ] - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_3_GATE", - [ - "N-B" - ] - ], - "pass": true, - "fails_when": "a failed N-B Rule 3 blocker does not block, so the study decides on evidence its own controls say is unreliable", - "row": "any Rule 3 gate set to FAIL" - }, - { - "test": "an UNEVALUABLE N-B control is not a pass either (A41.2.1)", - "expected": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "observed": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "pass": true, - "fails_when": "a blocker that was never measured certifies itself, which is exactly the self-consistent-but-incomplete artifact A41.2.1 closed", - "row": "" - }, - { - "test": "a FAILING N-C control -> INSUFFICIENT_COMPARATIVE_EVIDENCE", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_3_GATE", - [ - "N-C" - ] - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_3_GATE", - [ - "N-C" - ] - ], - "pass": true, - "fails_when": "a failed N-C Rule 3 blocker does not block, so the study decides on evidence its own controls say is unreliable", - "row": "any Rule 3 gate set to FAIL" - }, - { - "test": "an UNEVALUABLE N-C control is not a pass either (A41.2.1)", - "expected": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "observed": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "pass": true, - "fails_when": "a blocker that was never measured certifies itself, which is exactly the self-consistent-but-incomplete artifact A41.2.1 closed", - "row": "" - }, - { - "test": "a FAILING R1 text reliability gate -> INSUFFICIENT", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - [ - "R1" - ] - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - [ - "R1" - ] - ], - "pass": true, - "fails_when": "section 5.6's text threshold stopped blocking Rule 3", - "row": "any Rule 3 gate set to FAIL" - }, - { - "test": "a FAILING R1 role reliability gate -> INSUFFICIENT", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - [ - "R1" - ] - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - [ - "R1" - ] - ], - "pass": true, - "fails_when": "section 5.6's role threshold stopped blocking Rule 3", - "row": "any Rule 3 gate set to FAIL" - }, - { - "test": "S1 not firing -> INSUFFICIENT (the comparator is not live)", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - [ - "S1" - ] - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - [ - "S1" - ] - ], - "pass": true, - "fails_when": "a dead comparator does not block the decision, so an architecture is chosen on a comparison that was never shown to move", - "row": "any Rule 3 gate set to FAIL" - }, - { - "test": "section 4.5 INADEQUATE blocks; LIMITED does NOT (A28.2)", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "HYBRID_BY_PRIOR" - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "HYBRID_BY_PRIOR" - ], - "pass": true, - "fails_when": "either INADEQUATE stopped failing Rule 3, or LIMITED started to -- A28.2 says LIMITED does not fail Rule 3, and treating it as a blocker would void a valid study", - "row": "" - }, - { - "test": "a FAILING confirmatory X2A -> INSUFFICIENT", - "expected": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "observed": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "pass": true, - "fails_when": "X's own contract assertions failed and the study decided anyway", - "row": "any Rule 3 gate set to FAIL" - }, - { - "test": "a FAILING confirmatory X2B -> INSUFFICIENT", - "expected": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "observed": "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "pass": true, - "fails_when": "X's own contract assertions failed and the study decided anyway", - "row": "any Rule 3 gate set to FAIL" - }, - { - "test": "an M9 clause missing from a document -> M9 NOT EVALUABLE -> INSUFFICIENT", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "NOT_EVALUABLE" - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "NOT_EVALUABLE" - ], - "pass": true, - "fails_when": "an unevaluable M9 gate lets Rule 0 report 'nothing fired', which is indistinguishable from 'both arms were structurally viable everywhere'", - "row": "" - }, - { - "test": "the gate vector is exactly A27.6's nine, in its order", - "expected": 9, - "observed": 9, - "pass": true, - "fails_when": "a decision-blocking condition was added to or dropped from A27.6's vector", - "row": "" - }, - { - "test": "x09 FAILING on every sampled document changes NO outcome -- it only adds a label", - "expected": [ - "HYBRID_BY_PRIOR", - "RULE_2_PRIOR", - true - ], - "observed": [ - "HYBRID_BY_PRIOR", - "RULE_2_PRIOR", - true - ], - "pass": true, - "fails_when": "a cross-engine failure moved the architecture decision, making a REPORTING qualification decision-blocking against A27.6", - "row": "x09 set to FAIL" - }, - { - "test": "x09 is absent from the Rule 3 gate vector", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "cross-engine became a Rule 3 gate", - "row": "" - }, - { - "test": "the decision records the qualification as non-blocking", - "expected": false, - "observed": false, - "pass": true, - "fails_when": "the artifact claims the qualification blocks", - "row": "" - }, - { - "test": "D = 60 regions -> Rule 1 IS evaluated (and X's win stands)", - "expected": [ - "EXTENDED_BY_RULE_1", - 60, - true - ], - "observed": [ - "EXTENDED_BY_RULE_1", - 60, - true - ], - "pass": true, - "fails_when": "the frozen 60-region budget moved, so a raw count is applied to a census it was never valid on -- or Rule 1 stopped running on a census it may run on", - "row": "" - }, - { - "test": "D = 61 regions -> INSUFFICIENT, and Rule 1 is NOT reached", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "BUDGET_A10_A27_3", - false, - 61 - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "BUDGET_A10_A27_3", - false, - 61 - ], - "pass": true, - "fails_when": "a census over the human budget still chose X on raw counts -- A10's whole subject", - "row": "synthetic census of 61 regions" - }, - { - "test": "the 61-region census came from build_frames, not from the fixture", - "expected": 61, - "observed": 61, - "pass": true, - "fails_when": "the census is a number the probe wrote down, so the budget check is tested against the fixture's belief rather than the producer's count", - "row": "" - }, - { - "test": "a census of 40 with only 39 adjudicated -> INSUFFICIENT (A27.3: never a sample)", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "BUDGET_A10_A27_3", - false - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "BUDGET_A10_A27_3", - false - ], - "pass": true, - "fails_when": "Rule 1 ran on a SAMPLE of the census, which A27.3 forbids in the same words as the 60/120 clause it supersedes", - "row": "" - }, - { - "test": "an EMPTY census -> HYBRID_BY_PRIOR, and the text makes no empirical claim for hybrid", - "expected": [ - "HYBRID_BY_PRIOR", - "RULE_2_PRIOR", - true - ], - "observed": [ - "HYBRID_BY_PRIOR", - "RULE_2_PRIOR", - true - ], - "pass": true, - "fails_when": "an empty census produced something other than the prior, or the conclusion claimed hybrid won empirically on no evidence at all", - "row": "synthetic empty census" - }, - { - "test": "X_CORRECTS = 4 -> HYBRID_BY_PRIOR (condition 1 fails, so Rule 2's prior stands)", - "expected": [ - "HYBRID_BY_PRIOR", - "RULE_2_PRIOR", - false - ], - "observed": [ - "HYBRID_BY_PRIOR", - "RULE_2_PRIOR", - false - ], - "pass": true, - "fails_when": "the >= 5 threshold moved down, so four corrections overturn the architectural prior", - "row": "" - }, - { - "test": "X_CORRECTS = 5 -> EXTENDED_BY_RULE_1 (the threshold is >= 5, not > 5)", - "expected": [ - "EXTENDED_BY_RULE_1", - "RULE_1" - ], - "observed": [ - "EXTENDED_BY_RULE_1", - "RULE_1" - ], - "pass": true, - "fails_when": "the threshold moved up, so exactly five corrections -- the number section 7.3 names as falsifying the hybrid hypothesis -- no longer does", - "row": "" - }, - { - "test": "6 corrections, 0 regressions, no M4 regression -> EXTENDED_BY_RULE_1", - "expected": "EXTENDED_BY_RULE_1", - "observed": "EXTENDED_BY_RULE_1", - "pass": true, - "fails_when": "a clean synthetic X win does not produce an X win, so no evidence could ever flip the ADR", - "row": "synthetic X win (6 corrects, 0 regressions, no M4 regression)" - }, - { - "test": "X_REGRESSES = 0 wins; X_REGRESSES = 1 does NOT (A5 tightened '<= 1' to zero)", - "expected": [ - "EXTENDED_BY_RULE_1", - "INSUFFICIENT_COMPARATIVE_EVIDENCE" - ], - "observed": [ - "EXTENDED_BY_RULE_1", - "INSUFFICIENT_COMPARATIVE_EVIDENCE" - ], - "pass": true, - "fails_when": "the regression tolerance came back, and at an expected denominator near zero '<= 1' is not a tolerance -- it is 20 % of the win threshold", - "row": "" - }, - { - "test": "5 corrections AND 1 regression -> INSUFFICIENT, never EXTENDED_BY_RULE_1, never HYBRID", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_1", - [ - "x_regresses_exactly_0" - ] - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_1", - [ - "x_regresses_exactly_0" - ] - ], - "pass": true, - "fails_when": "A5's asymmetry was lost: condition 1 holding with a veto failing is 'insufficient evidence / review, NEVER an X win' -- and equally never a hybrid victory", - "row": "synthetic 5 corrects and 1 regression" - }, - { - "test": "5 corrections, 0 regressions, ONE M4 parent regression -> INSUFFICIENT (A5 row 4 / A20)", - "expected": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_1", - [ - "no_m4_parent_regression" - ], - 1 - ], - "observed": [ - "INSUFFICIENT_COMPARATIVE_EVIDENCE", - "RULE_1", - [ - "no_m4_parent_regression" - ], - 1 - ], - "pass": true, - "fails_when": "the M4 parent veto stopped blocking, so X wins while breaking a hierarchy H had right", - "row": "synthetic 5 corrects and 1 M4 regression" - }, - { - "test": "...and it fires while the per-arm M4 AGGREGATES are equal -- the A42.3 payload", - "expected": [ - 3, - 3, - true, - true - ], - "observed": [ - 3, - 3, - true, - true - ], - "pass": true, - "fails_when": "the fixture's aggregates are NOT equal, so it does not separate the two readings of A5 row 4 and the veto could have fired for the wrong reason", - "row": "synthetic 5 corrects and 1 M4 regression" - }, - { - "test": "the M4 veto only bites when condition 1 holds -- 4 corrections still yields the PRIOR", - "expected": "HYBRID_BY_PRIOR", - "observed": "HYBRID_BY_PRIOR", - "pass": true, - "fails_when": "a failing veto beneath the win threshold was reported as insufficient evidence, when the census simply yielded no X win and Rule 2's prior applies", - "row": "" - }, - { - "test": "condition 4 is READ from the scorer's paired fact, not supplied and not inferred", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the supplied-fact channel came back, so a caller can assert Rule 1's fourth condition with nothing behind it (R5's closed channel, reopened)", - "row": "" - }, - { - "test": "M6 is STRUCK by A20 and is not a Rule 1 condition", - "expected": [ - "no_m4_parent_regression", - "x_corrects_at_least_5", - "x_regresses_exactly_0" - ], - "observed": [ - "no_m4_parent_regression", - "x_corrects_at_least_5", - "x_regresses_exactly_0" - ], - "pass": true, - "fails_when": "A5's M6 amount-attribution veto came back, reviving a metric A20 deferred from the study", - "row": "" - }, - { - "test": "the frozen enum is exactly the five outcomes A10 and A27.4 name", - "expected": [ - "EXTENDED_BY_RULE_0_M9", - "EXTENDED_BY_RULE_1", - "HYBRID_BY_PRIOR", - "HYBRID_BY_RULE_0_M9", - "INSUFFICIENT_COMPARATIVE_EVIDENCE" - ], - "observed": [ - "EXTENDED_BY_RULE_0_M9", - "EXTENDED_BY_RULE_1", - "HYBRID_BY_PRIOR", - "HYBRID_BY_RULE_0_M9", - "INSUFFICIENT_COMPARATIVE_EVIDENCE" - ], - "pass": true, - "fails_when": "an outcome was added, removed or renamed, so a consumer reads a decision the protocol does not define", - "row": "" - }, - { - "test": "no SIXTH outcome-shaped literal exists anywhere in the module (AST, not grep)", - "expected": "set()", - "observed": "set()", - "pass": true, - "fails_when": "a branch invents an outcome alias, which would leave the module as a string no gate, report or reader knows how to treat", - "row": "" - }, - { - "test": "a 240-payload sweep emits ONLY the five frozen outcomes", - "expected": "set()", - "observed": "set()", - "pass": true, - "fails_when": "some payload reaches an outcome outside the closed enum", - "row": "" - }, - { - "test": "and ALL FIVE are reachable by an intentional fixture", - "expected": [ - "EXTENDED_BY_RULE_0_M9", - "EXTENDED_BY_RULE_1", - "HYBRID_BY_PRIOR", - "HYBRID_BY_RULE_0_M9", - "INSUFFICIENT_COMPARATIVE_EVIDENCE" - ], - "observed": [ - "EXTENDED_BY_RULE_0_M9", - "EXTENDED_BY_RULE_1", - "HYBRID_BY_PRIOR", - "HYBRID_BY_RULE_0_M9", - "INSUFFICIENT_COMPARATIVE_EVIDENCE" - ], - "pass": true, - "fails_when": "an outcome the protocol defines can never be produced, so a real result could have no expressible answer", - "row": "" - }, - { - "test": "the closed-enum assertion REFUSES a fabricated outcome (non-vacuity)", - "expected": "OUTCOME_NOT_IN_FROZEN_ENUM", - "observed": "OUTCOME_NOT_IN_FROZEN_ENUM", - "pass": true, - "fails_when": "the guard cannot fire, so 'no sixth outcome can be emitted' is certified by a check that was never able to fail", - "row": "" - }, - { - "test": "a pre-committed sentence exists for EXACTLY the five outcomes -- the second layer's basis", - "expected": [ - "EXTENDED_BY_RULE_0_M9", - "EXTENDED_BY_RULE_1", - "HYBRID_BY_PRIOR", - "HYBRID_BY_RULE_0_M9", - "INSUFFICIENT_COMPARATIVE_EVIDENCE" - ], - "observed": [ - "EXTENDED_BY_RULE_0_M9", - "EXTENDED_BY_RULE_1", - "HYBRID_BY_PRIOR", - "HYBRID_BY_RULE_0_M9", - "INSUFFICIENT_COMPARATIVE_EVIDENCE" - ], - "pass": true, - "fails_when": "an outcome has no pre-committed sentence (so the wording would be chosen once the result is known), or a sentence exists for an outcome the enum does not define", - "row": "" - }, - { - "test": "the prior conclusion carries NO comparative-accuracy claim for hybrid", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "'X failed to prove a win' was written as 'H empirically beat X', which A10 forbids in those words", - "row": "wording gate" - }, - { - "test": "the insufficient by veto conclusion carries NO comparative-accuracy claim for hybrid", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "'X failed to prove a win' was written as 'H empirically beat X', which A10 forbids in those words", - "row": "wording gate" - }, - { - "test": "the insufficient by budget conclusion carries NO comparative-accuracy claim for hybrid", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "'X failed to prove a win' was written as 'H empirically beat X', which A10 forbids in those words", - "row": "wording gate" - }, - { - "test": "the insufficient by gate conclusion carries NO comparative-accuracy claim for hybrid", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "'X failed to prove a win' was written as 'H empirically beat X', which A10 forbids in those words", - "row": "wording gate" - }, - { - "test": "every prior/insufficient conclusion says hybrid stands BY PRIOR", - "expected": [ - true, - true, - true, - true - ], - "observed": [ - true, - true, - true, - true - ], - "pass": true, - "fails_when": "a non-win is reported without saying hybrid survives by prior rather than by victory", - "row": "" - }, - { - "test": "the gate FIRES on a planted superiority claim (non-vacuity)", - "expected": "CONCLUSION_CLAIMS_SUPERIORITY", - "observed": "CONCLUSION_CLAIMS_SUPERIORITY", - "pass": true, - "fails_when": "the wording gate cannot fire, so every conclusion above passed a check that was never able to fail -- the vacuous-pass shape this study keeps finding", - "row": "wording gate" - }, - { - "test": "and on each forbidden pattern individually, not just the one example", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a forbidden pattern is in the list but matches nothing, so it is decoration", - "row": "" - }, - { - "test": "a MISSING X2-a status refuses", - "expected": "GATE_STATUS_MISSING", - "observed": "GATE_STATUS_MISSING", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "a MISSING X2-b status refuses", - "expected": "GATE_STATUS_MISSING", - "observed": "GATE_STATUS_MISSING", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "an UNKNOWN gate status refuses", - "expected": "GATE_STATUS_UNKNOWN", - "observed": "GATE_STATUS_UNKNOWN", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "a D block carrying NO paired M4 fact refuses -- the aggregates are not a fallback", - "expected": "M4_VETO_FACT_MISSING", - "observed": "M4_VETO_FACT_MISSING", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "an unknown metrics SCHEMA refuses", - "expected": "METRICS_SCHEMA_UNKNOWN", - "observed": "METRICS_SCHEMA_UNKNOWN", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "a metrics payload MISSING a required block refuses", - "expected": "MISSING_REQUIRED_FACT", - "observed": "MISSING_REQUIRED_FACT", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "a scorer claiming it took the decision refuses", - "expected": "SCORER_TOOK_A_DECISION", - "observed": "SCORER_TOOK_A_DECISION", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "frames that do not match the scored documents refuse", - "expected": "FRAME_DOCUMENT_SET_MISMATCH", - "observed": "FRAME_DOCUMENT_SET_MISMATCH", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "a frame with no committed D census refuses", - "expected": "D_CENSUS_MISSING", - "observed": "D_CENSUS_MISSING", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "a census COUNT that disagrees with its own census LIST refuses", - "expected": "D_CENSUS_DRIFT", - "observed": "D_CENSUS_DRIFT", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "a TRUNCATED census refuses -- A27.3 requires the complete enumeration", - "expected": "D_CENSUS_TRUNCATED", - "observed": "D_CENSUS_TRUNCATED", - "pass": true, - "fails_when": "missing or malformed evidence was silently treated as zero or as a pass, which at the architecture decision is the one substitution nothing downstream can undo", - "row": "" - }, - { - "test": "M4's refusal NAMES the rejected fallback rather than silently taking it", - "expected": true, - "observed": true, - "pass": true, - "fails_when": "the refusal is anonymous, so a reader cannot tell the decider declined to substitute the aggregates rather than simply failing to find a key", - "row": "" - }, - { - "test": "an ABSENT D block over a ZERO census is not a refusal -- the prior stands", - "expected": [ - "HYBRID_BY_PRIOR", - 0, - true - ], - "observed": [ - "HYBRID_BY_PRIOR", - 0, - true - ], - "pass": true, - "fails_when": "an empty census was turned into a refusal, so the most likely real outcome of the whole study could not be expressed at all", - "row": "" - }, - { - "test": "FAULT rule1_threshold_lowered changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT regression_tolerance_restored changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT budget_relaxed_to_120 changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT m4_veto_disabled changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT m4_veto_inferred_from_aggregates_the_A42_3_rejected_reading changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT rule0_two_sided_ranking_invented changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT not_evaluable_accepted_as_pass changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT five_and_one_collapsed_into_the_prior changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT rule0_no_longer_supersedes changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT wording_gate_disabled changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "FAULT closed_enum_guard_removed changes a result-bearing answer (and is caught by a NAMED check)", - "expected": [ - true, - false - ], - "observed": [ - true, - false - ], - "pass": true, - "fails_when": "the fault is invisible, so the control asserting the correct behaviour is not discriminating -- or it was detected only by a crash, which cannot distinguish 'the rule broke' from 'the probe has a bug'", - "row": "" - }, - { - "test": "NO confirmatory or execution artifact exists after this probe", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a probe wrote a confirmatory artifact, or left its own faulted module on disk inside the protected tree", - "row": "" - }, - { - "test": "the decider takes no path that could WRITE anything", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "the decider acquired a writer, so an architecture decision could be persisted before the frozen start procedure has been performed", - "row": "" - }, - { - "test": "every HARNESS-PLAN section 6 control row has at least one executable check", - "expected": [], - "observed": [], - "pass": true, - "fails_when": "a frozen control row is unimplemented, so the decider carries a rule nothing can falsify", - "row": "" - } - ], - "failures": [] -} \ No newline at end of file diff --git a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x29_execute_study.json b/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x29_execute_study.json deleted file mode 100644 index ec2fe29a..00000000 --- a/docs/research/pdf-backend-bakeoff/validation/external-validity/results/x29_execute_study.json +++ /dev/null @@ -1,676 +0,0 @@ -{ - "population": "SYNTHETIC + DEVELOPMENT -- no holdout opened by any extractor, nothing scored", - "amendment": "A43 + A45", - "canonical_frames_json_created": false, - "holdout_extraction_performed": false, - "development_documents": [ - "113-hr-3547", - "118-hr-2882" - ], - "canonical_entrypoint": "execute_study.canonical_population() -> write_frames() -> load_frames()", - "population_authority": "results/holdout_membership.json", - "page_limit": null, - "refusal_classes": [ - "CANONICAL_AUTHORITY_NOT_FROZEN", - "CANONICAL_AUTHORITY_UNCOMMITTED", - "DECLARED_COUNT_MISMATCH", - "DESCRIPTOR_METADATA_MISMATCH", - "DUPLICATE_FRAME", - "DUPLICATE_MEMBER_ID", - "EXTRACTION_SOURCE_MISMATCH", - "FRAMES_ARTIFACT_MALFORMED", - "FRAMES_ARTIFACT_MISSING", - "FRAMES_ARTIFACT_UNCOMMITTED", - "FRAME_METADATA_MISMATCH", - "FRAME_POPULATION_MISMATCH", - "FRAME_SOURCE_MISMATCH", - "INVALID_STRATUM", - "MEMBERSHIP_MALFORMED", - "MEMBERSHIP_MISSING", - "MEMBER_MALFORMED", - "NON_CANONICAL_AUTHORITY", - "POPULATION_DUPLICATED", - "POPULATION_HAS_EXTRA", - "POPULATION_INCOMPLETE", - "POPULATION_SUBSTITUTED", - "SOURCE_FILE_MISSING", - "SOURCE_SHA_MISMATCH", - "UNKNOWN_POPULATION" - ], - "identity": { - "n_members": 2, - "mutations": [ - "pdf_path", - "population", - "stratum", - "canonical_authority", - "readback" - ] - }, - "handoff": { - "canonical_handoff_executes": true, - "record_fields": [ - "document", - "document_sha256", - "pdf_path" - ], - "n_rows": 2, - "artifact_population": "NON-HOLDOUT (development / synthetic)", - "consumers_exercised": [ - "cross_engine_control.write_cross_engine_control", - "s1_control.write_s1_control" - ], - "authority_gate": "cross_engine_control.assert_records_from_authority via execute_study.authority_index", - "canonical_records_accepted": 17 - }, - "authority": { - "canonical_authority_verified_at": [ - "canonical_population", - "build_document_frame_for", - "write_frames", - "load_frames" - ] - }, - "guard": { - "n_guarded": 17, - "equals_membership": true - }, - "boundary": { - "boundary_state": "ABSENT" - }, - "g5": { - "g5_green": true, - "surface_size": 15 - }, - "population_evidence": { - "membership": "/membership.json", - "docs_root": "/holdout", - "n_members": 2 - }, - "frames_evidence": { - "frames_written": "/frames.json", - "n_frames": 2, - "c_frame_selected_p_head": 9, - "c_frame_selected_p_robust": 0 - }, - "checks": [ - { - "check": "a well-formed membership loads every member with its metadata", - "ok": true, - "fails_when": "the loader drops, reorders or invents a member", - "observed": "2 descriptors" - }, - { - "check": "population and stratum are READ from the authority, never inferred", - "ok": true, - "fails_when": "a descriptor carries a population or stratum the membership does not state", - "observed": "" - }, - { - "check": "MUTATION omit one frozen member -> POPULATION_INCOMPLETE", - "ok": true, - "fails_when": "a 1-of-2 population is accepted", - "observed": "POPULATION_INCOMPLETE" - }, - { - "check": "MUTATION append a non-member -> POPULATION_HAS_EXTRA", - "ok": true, - "fails_when": "an extra document is scored", - "observed": "POPULATION_HAS_EXTRA" - }, - { - "check": "MUTATION duplicate a member -> POPULATION_DUPLICATED", - "ok": true, - "fails_when": "one document is counted twice", - "observed": "POPULATION_DUPLICATED" - }, - { - "check": "MUTATION duplicate id IN the membership -> DUPLICATE_MEMBER_ID", - "ok": true, - "fails_when": "the loader accepts it", - "observed": "DUPLICATE_MEMBER_ID" - }, - { - "check": "MUTATION substitute a member -> POPULATION_SUBSTITUTED, not merely INCOMPLETE", - "ok": true, - "fails_when": "a swap is reported as the same failure as an omission, hiding which happened", - "observed": "POPULATION_SUBSTITUTED" - }, - { - "check": "MUTATION recorded sha256 != source bytes -> SOURCE_SHA_MISMATCH", - "ok": true, - "fails_when": "a document swapped after the freeze check is extracted anyway", - "observed": "SOURCE_SHA_MISMATCH" - }, - { - "check": "MUTATION source file absent -> SOURCE_FILE_MISSING", - "ok": true, - "fails_when": "a missing document silently shrinks the study", - "observed": "SOURCE_FILE_MISSING" - }, - { - "check": "MUTATION population -> 'P-heads' -> UNKNOWN_POPULATION", - "ok": true, - "fails_when": "an unknown population yields 0 C-regions", - "observed": "UNKNOWN_POPULATION" - }, - { - "check": "MUTATION stratum -> 99 -> INVALID_STRATUM", - "ok": true, - "fails_when": "section 4.5's strata-filled count reads a phantom stratum", - "observed": "INVALID_STRATUM" - }, - { - "check": "MUTATION n_members -> 99 -> DECLARED_COUNT_MISMATCH", - "ok": true, - "fails_when": "a corrupted authority passes as a smaller one", - "observed": "DECLARED_COUNT_MISMATCH" - }, - { - "check": "MUTATION delete a member's stratum -> MEMBER_MALFORMED", - "ok": true, - "fails_when": "a member without metadata is defaulted", - "observed": "MEMBER_MALFORMED" - }, - { - "check": "a stratum mutation ALTERS the downstream input (document_strata), so the guard is not vacuous", - "ok": true, - "fails_when": "document_strata returns the same map whatever the membership says", - "observed": "{'113-hr-3547': 1, '118-hr-2882': 2} -> {'113-hr-3547': 3, '118-hr-2882': 2}" - }, - { - "check": "write_frames emits exactly one frame per member, in the membership's shape", - "ok": true, - "fails_when": "a member is dropped or duplicated between population and artifact", - "observed": "2 frames" - }, - { - "check": "the artifact records the whole-document scope, not a prefix", - "ok": true, - "fails_when": "a page limit reaches the canonical path and shrinks every denominator", - "observed": "" - }, - { - "check": "load_frames accepts the artifact it just wrote", - "ok": true, - "fails_when": "the round-trip loses a frame", - "observed": "" - }, - { - "check": "MUTATION load frames against a 1-of-2 population -> refuses", - "ok": true, - "fails_when": "a later stage reads a subset", - "observed": "POPULATION_INCOMPLETE" - }, - { - "check": "MUTATION delete a frame FROM the artifact -> FRAME_POPULATION_MISMATCH", - "ok": true, - "fails_when": "a truncated frames.json is consumed as a complete study", - "observed": "FRAME_POPULATION_MISMATCH" - }, - { - "check": "MUTATION consume an UNCOMMITTED frames.json -> FRAMES_ARTIFACT_UNCOMMITTED", - "ok": true, - "fails_when": "the oracle reads frames that exist only in the working tree, which no reviewer can read", - "observed": "FRAMES_ARTIFACT_UNCOMMITTED" - }, - { - "check": "oracle_documents carries the membership's stratum and the real pdf_path", - "ok": true, - "fails_when": "a stratum is transcribed by hand or a path is invented", - "observed": "" - }, - { - "check": "MUTATION oracle_documents on a subset -> refuses at the handover", - "ok": true, - "fails_when": "the oracle is built on 1 of 2", - "observed": "POPULATION_INCOMPLETE" - }, - { - "check": "control_documents gives S1 and cross-engine the same complete population", - "ok": true, - "fails_when": "S1 and cross-engine measure a different set than the frames", - "observed": "" - }, - { - "check": "MUTATION control_documents on a subset -> refuses at the handover", - "ok": true, - "fails_when": "S1 runs on 1 of 2", - "observed": "POPULATION_INCOMPLETE" - }, - { - "check": "a population mutation ALTERS the C-frame draw, so the field is result-bearing", - "ok": true, - "fails_when": "P-head and P-robust produce the same C-frame, i.e. the population field is inert", - "observed": "P-head c_frame_selected=9, P-robust=0" - }, - { - "check": "NON-VACUITY: the unmutated population passes assert_population_complete", - "ok": true, - "fails_when": "the handover refuses everything", - "observed": "accepted" - }, - { - "check": "NON-VACUITY: the unmutated population passes control_documents", - "ok": true, - "fails_when": "the handover refuses everything", - "observed": "accepted" - }, - { - "check": "NON-VACUITY: the unmutated population passes document_strata", - "ok": true, - "fails_when": "the handover refuses everything", - "observed": "accepted" - }, - { - "check": "NON-VACUITY: the unmutated population passes build_document_frame_for", - "ok": true, - "fails_when": "the handover refuses everything", - "observed": "accepted" - }, - { - "check": "MUTATION swap pdf_path to another DEVELOPMENT pdf, keeping id -> DESCRIPTOR_METADATA_MISMATCH", - "ok": true, - "fails_when": "the frame is built from substituted bytes while carrying the frozen document_sha256", - "observed": "DESCRIPTOR_METADATA_MISMATCH" - }, - { - "check": "...and the bytes are RE-HASHED at the point of extraction -> EXTRACTION_SOURCE_MISMATCH", - "ok": true, - "fails_when": "a path substituted after load reaches the runners; hashing at load only proves what was true at load", - "observed": "EXTRACTION_SOURCE_MISMATCH" - }, - { - "check": "MUTATION swap population to the other VALID value, keeping id -> DESCRIPTOR_METADATA_MISMATCH", - "ok": true, - "fails_when": "the C-frame draw silently empties (P-head-only) under an unchanged document id", - "observed": "DESCRIPTOR_METADATA_MISMATCH" - }, - { - "check": "...and frames_document refuses before building anything", - "ok": true, - "fails_when": "the mutated frame is produced", - "observed": "DESCRIPTOR_METADATA_MISMATCH" - }, - { - "check": "MUTATION swap stratum to another VALID value, keeping id -> DESCRIPTOR_METADATA_MISMATCH", - "ok": true, - "fails_when": "section 4.5's strata-filled count reads a stratum the authority never assigned", - "observed": "DESCRIPTOR_METADATA_MISMATCH" - }, - { - "check": "...and document_strata refuses rather than reporting the mutated map", - "ok": true, - "fails_when": "the mutated map is returned", - "observed": "DESCRIPTOR_METADATA_MISMATCH" - }, - { - "check": "MUTATION write the CANONICAL frames.json from a synthetic membership -> NON_CANONICAL_AUTHORITY", - "ok": true, - "fails_when": "a fixture population is written to the canonical path and records itself as the authority", - "observed": "NON_CANONICAL_AUTHORITY" - }, - { - "check": "NON-VACUITY: the same synthetic membership is accepted at a NON-canonical path", - "ok": true, - "fails_when": "", - "observed": "accepted at a non-canonical path" - }, - { - "check": "MUTATION frame document_sha256 disagrees with the authority -> FRAME_SOURCE_MISMATCH", - "ok": true, - "fails_when": "a frame built from substituted bytes is consumed downstream", - "observed": "FRAME_SOURCE_MISMATCH" - }, - { - "check": "MUTATION frame population disagrees with the authority -> FRAME_SOURCE_MISMATCH", - "ok": true, - "fails_when": "a frame built under the wrong population is consumed downstream", - "observed": "FRAME_SOURCE_MISMATCH" - }, - { - "check": "MUTATION drop a frame, every survivor still valid, population=None -> FRAME_POPULATION_MISMATCH", - "ok": true, - "fails_when": "a 1-of-2 artifact is consumed as a complete study whenever the caller passes no population", - "observed": "FRAME_POPULATION_MISMATCH" - }, - { - "check": "MUTATION duplicate a VALID frame, population=None -> DUPLICATE_FRAME", - "ok": true, - "fails_when": "one document is scored twice; the duplicate is individually valid so no per-frame check sees it", - "observed": "DUPLICATE_FRAME" - }, - { - "check": "NON-VACUITY: the INTACT artifact still loads with no population argument", - "ok": true, - "fails_when": "the new bijection refuses a correct artifact, making the two negatives meaningless", - "observed": "2 frames" - }, - { - "check": "THE CANONICAL CALL RUNS: write_cross_engine_control(control_documents(pop)) -> artifact", - "ok": true, - "fails_when": "the canonical handoff raises, so no cross_engine_control.json exists and score_metrics has no cross-engine input -- the Run 1 failure", - "observed": "2 rows" - }, - { - "check": "R2-A3: exactly one artifact row per canonical member, no omission, duplicate or extra", - "ok": true, - "fails_when": "the artifact's document set is not the population's -- score_metrics requires EXACT set equality with the scored frames, so a drift here moves the qualification denominator", - "observed": "['113-hr-3547', '118-hr-2882']" - }, - { - "check": "R2-A2: every row's document_sha256 IS the canonical membership SHA", - "ok": true, - "fails_when": "the SHA reaching A39.2's page-sample identity is not the authority's, so the sampled pages are a different set for the same document -- silently and reproducibly", - "observed": "" - }, - { - "check": "the probe stayed off the holdout: the artifact declares a NON-HOLDOUT population", - "ok": true, - "fails_when": "this control opened a confirmatory document", - "observed": "NON-HOLDOUT (development / synthetic)" - }, - { - "check": "the SAME record also satisfies the OTHER consumer, write_s1_control", - "ok": true, - "fails_when": "control_documents' claim to serve S1 and cross-engine alike is false for one of them", - "observed": "" - }, - { - "check": "NON-VACUITY: G5 is GREEN on the repaired tree before the handoff fault is injected", - "ok": true, - "fails_when": "G5 is already red, so the mutation below cannot be shown to have caused anything", - "observed": "all 15 result-bearing files committed; execution path live" - }, - { - "check": "MUTATION delete document_sha256 from the canonical handoff -> the consumer REFUSES", - "ok": true, - "fails_when": "the producer may drop a result-bearing field its consumer requires and nothing notices until execution", - "observed": "DOCUMENT_RECORD_INCOMPLETE" - }, - { - "check": "...and the refusal happens BEFORE any artifact is written", - "ok": true, - "fails_when": "a partial confirmatory artifact is left on disk by a failed handoff", - "observed": "" - }, - { - "check": "MUTATION delete document_sha256 from the canonical handoff -> G5 RED, naming the handoff", - "ok": true, - "fails_when": "readiness stays GREEN while the canonical handoff cannot be consumed -- exactly the state Run 1 was authorized in; or G5 is red for an unrelated reason and the handoff arm never fired", - "observed": "control_documents' record is REFUSED by its own consumer (cross_engine_control.document_inputs): CrossEngineError: DOCUMENT_RECORD_INCOMPLETE {'missing': ['document_sha256'], 'record_keys': ['document', 'pdf_path']}" - }, - { - "check": "NON-VACUITY: the handoff check and G5 are GREEN again once the producer is restored", - "ok": true, - "fails_when": "the check refuses the repaired handoff, which would make the mutation above meaningless", - "observed": "all 15 result-bearing files committed; execution path live" - }, - { - "check": "MUTATION substitute another member's SHA in the descriptor -> execute_study refuses", - "ok": true, - "fails_when": "a descriptor carrying another document's SHA is handed to the cross-engine producer", - "observed": "DESCRIPTOR_METADATA_MISMATCH" - }, - { - "check": "...and the WRITER refuses it too, at the authority gate, before any measurement", - "ok": true, - "fails_when": "a hand-assembled record carrying another member's SHA reaches the engines", - "observed": "RECORD_AUTHORITY_MISMATCH" - }, - { - "check": "...and INDEPENDENTLY, verified_sha256 still refuses the same swap at cross_engine_result", - "ok": true, - "fails_when": "the byte check was weakened or absorbed when the authority gate was added -- the two prove different facts and A45.6 may not cost us the older one", - "observed": "SOURCE_SHA256_MISMATCH" - }, - { - "check": "MUTATION substitute another member's pdf_path in the descriptor -> execute_study refuses", - "ok": true, - "fails_when": "the cross-engine measurement is taken on the wrong document's bytes", - "observed": "DESCRIPTOR_METADATA_MISMATCH" - }, - { - "check": "...and the WRITER refuses it too, at the authority gate, before any measurement", - "ok": true, - "fails_when": "a hand-assembled record pointing at another member's bytes reaches the engines", - "observed": "RECORD_AUTHORITY_MISMATCH" - }, - { - "check": "...and INDEPENDENTLY, verified_sha256 still refuses the same swap at cross_engine_result", - "ok": true, - "fails_when": "the byte check no longer sees a path pointing at different bytes than the recorded SHA", - "observed": "SOURCE_SHA256_MISMATCH" - }, - { - "check": "MUTATION alter a source byte while keeping the recorded SHA -> the producer refuses", - "ok": true, - "fails_when": "hashing at load only proves what was true at load; a document swapped afterwards is measured anyway and the artifact reports the frozen SHA for mutated bytes", - "observed": "SOURCE_SHA256_MISMATCH" - }, - { - "check": "the mutated source is restored byte-identically", - "ok": true, - "fails_when": "fault injection leaves the fixture modified", - "observed": "" - }, - { - "check": "MUTATION omit one canonical member -> refused before the handoff is built", - "ok": true, - "fails_when": "a 1-of-2 cross-engine artifact is written and score_metrics' set-equality check is the only thing left standing between that and a moved qualification denominator", - "observed": "POPULATION_INCOMPLETE" - }, - { - "check": "MUTATION duplicate one canonical member -> refused before the handoff is built", - "ok": true, - "fails_when": "one document is measured twice; a duplicate row is individually valid so no per-row check sees it", - "observed": "POPULATION_DUPLICATED" - }, - { - "check": "MUTATION A's id + B's VALID path + B's VALID SHA -> RECORD_AUTHORITY_MISMATCH", - "ok": true, - "fails_when": "one member's qualification is measured on another member's pages and reported under the first member's name -- invisible to the byte check and to the scorer's set equality", - "observed": "RECORD_AUTHORITY_MISMATCH" - }, - { - "check": "...and NO canonical artifact was written by the refused call", - "ok": true, - "fails_when": "a substituted row is persisted before the refusal", - "observed": "" - }, - { - "check": "...and the BYTE check accepts that very pair, so only the authority gate can have refused", - "ok": true, - "fails_when": "verified_sha256 also rejects this pair, so the control above does not isolate A45.6", - "observed": "verified_sha256(B.path, B.sha) -> accepted" - }, - { - "check": "...and the gate refuses it directly, named, without going through the writer", - "ok": true, - "fails_when": "the refusal is a side effect of the writer rather than a stateable invariant", - "observed": "RECORD_AUTHORITY_MISMATCH" - }, - { - "check": "MUTATION a non-member id -> DOCUMENT_NOT_A_MEMBER, distinct from a field mismatch", - "ok": true, - "fails_when": "an outsider and a substituted member collapse to one refusal, hiding which happened", - "observed": "DOCUMENT_NOT_A_MEMBER" - }, - { - "check": "MUTATION write the CANONICAL cross_engine_control.json against a synthetic membership -> refused", - "ok": true, - "fails_when": "a fixture population vouches for the canonical confirmatory artifact -- the A43.6 defect, in the writer A43.6 did not cover", - "observed": "NON_CANONICAL_AUTHORITY" - }, - { - "check": "...and the canonical artifact still does not exist", - "ok": true, - "fails_when": "this probe created a canonical confirmatory artifact", - "observed": "" - }, - { - "check": "POSITIVE: control_documents(canonical_population()) is ACCEPTED by the authority gate, for every frozen member", - "ok": true, - "fails_when": "the new gate refuses the real canonical composition, which would block Run 2 outright and make every refusal above meaningless", - "observed": "17 records accepted" - }, - { - "check": "NON-VACUITY: the clean committed authority is ACCEPTED by canonical_population", - "ok": true, - "fails_when": "the new check refuses the frozen authority, which would make the negative meaningless", - "observed": "17 descriptors" - }, - { - "check": "NON-VACUITY: the clean committed authority passes assert_canonical_authority", - "ok": true, - "fails_when": "", - "observed": "accepted" - }, - { - "check": "NON-VACUITY: the clean committed authority passes load_frames (authority arm)", - "ok": true, - "fails_when": "", - "observed": "FRAMES_ARTIFACT_MISSING" - }, - { - "check": "a SYNTHETIC fixture membership is exempt from the committed requirement", - "ok": true, - "fails_when": "the restriction attaches to the concept rather than the canonical path, refusing every control", - "observed": "" - }, - { - "check": "the simulated boundary does not create EXECUTION-START.json", - "ok": true, - "fails_when": "the probe authorized execution", - "observed": "" - }, - { - "check": "MUTATION repoint a member at another member's bytes AFTER the gate -> CANONICAL_AUTHORITY_UNCOMMITTED", - "ok": true, - "fails_when": "the authority is edited between x04's gate and execution, and every descriptor check still agrees because the AUTHORITY ITSELF moved", - "observed": "CANONICAL_AUTHORITY_UNCOMMITTED" - }, - { - "check": "...and the invariant refuses directly, not only through the loader", - "ok": true, - "fails_when": "", - "observed": "CANONICAL_AUTHORITY_UNCOMMITTED" - }, - { - "check": "...and a canonical artifact may not be CONSUMED against the drifted authority", - "ok": true, - "fails_when": "the artifact is compared to a manifest that is no longer the frozen one", - "observed": "CANONICAL_AUTHORITY_UNCOMMITTED" - }, - { - "check": "the canonical authority is restored byte-identically after fault injection", - "ok": true, - "fails_when": "the frozen manifest is left modified, which F10 would then fail on", - "observed": "1a8588c436e1" - }, - { - "check": "the authority invariant is GREEN again after the fault is reverted", - "ok": true, - "fails_when": "", - "observed": "" - }, - { - "check": "build_oracle.HOLDOUT_GUARD equals the committed membership", - "ok": true, - "fails_when": "the guard is a transcription that drifts from the authority", - "observed": "17 ids, symmetric difference 0" - }, - { - "check": "every frozen member is refused pre-boundary by assert_source_permitted", - "ok": true, - "fails_when": "a frozen member is not recognised as holdout, so the source gate never fires on it", - "observed": "" - }, - { - "check": "MUTATION drop a real member and add a phantom -> HOLDOUT_GUARD_DIVERGED", - "ok": true, - "fails_when": "the guard may disagree with the committed population and nothing says so", - "observed": "HOLDOUT_GUARD_DIVERGED" - }, - { - "check": "...and the divergence names BOTH directions", - "ok": true, - "fails_when": "an unguarded member is not detectable through the guard's own API", - "observed": "113-hr-933 unguarded while diverged" - }, - { - "check": "MUTATION guard divergence turns G5 RED", - "ok": true, - "fails_when": "readiness stays green while the source gate is open on real members", - "observed": "1 problem(s)" - }, - { - "check": "the guard is restored after fault injection", - "ok": true, - "fails_when": "debris is left behind", - "observed": "" - }, - { - "check": "MUTATION unreadable membership -> HOLDOUT_POPULATION_UNAVAILABLE, not an empty guard", - "ok": true, - "fails_when": "a missing authority disables the holdout guard entirely and looks like a clean load", - "observed": "HOLDOUT_POPULATION_UNAVAILABLE" - }, - { - "check": "the execution boundary is ABSENT in this session", - "ok": true, - "fails_when": "the probe ran post-authorization", - "observed": "ABSENT" - }, - { - "check": "canonical results/frames.json may not be written pre-boundary", - "ok": true, - "fails_when": "the first confirmatory artifact is the one the write guard cannot see", - "observed": "CONFIRMATORY_WRITE_BEFORE_EXECUTION" - }, - { - "check": "extraction of a frozen member refuses pre-boundary, BEFORE any arm runs", - "ok": true, - "fails_when": "a frozen member is extracted while the boundary is ABSENT", - "observed": "HOLDOUT_BEFORE_EXECUTION_BOUNDARY" - }, - { - "check": "G5 is GREEN on the repaired tree", - "ok": true, - "fails_when": "the repair itself does not satisfy G5", - "observed": "all 15 result-bearing files committed; execution path live" - }, - { - "check": "MUTATION delete execute_study.write_frames -> G5 RED", - "ok": true, - "fails_when": "readiness stays green while the canonical writer is gone", - "observed": "execute_study.write_frames is missing or not callable" - }, - { - "check": "MUTATION set a canonical PAGE_LIMIT -> G5 RED", - "ok": true, - "fails_when": "a prefix scope reaches the canonical path and silently shrinks every denominator", - "observed": "PAGE_LIMIT is 8; the canonical scope is the whole document" - }, - { - "check": "MUTATION remove probes/execute_study.py -> G5 RED", - "ok": true, - "fails_when": "the surface no longer covers the component that decides which documents enter the study", - "observed": "MISSING 1/15: probes/execute_study.py" - }, - { - "check": "the component file is restored byte-identically after fault injection", - "ok": true, - "fails_when": "fault injection leaves the tree modified", - "observed": "" - }, - { - "check": "G5 is GREEN again after every injected fault is reverted", - "ok": true, - "fails_when": "a fault was not fully reverted", - "observed": "" - } - ], - "n_checks": 96, - "n_failed": 0 -} \ No newline at end of file