From 989d1941bcbc0280963bf1e33127f7f425f28147 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 10:01:55 +0000 Subject: [PATCH 1/7] Keep timelines out of fact-list conflicts, save fact rows between updates, add `HINBOX_PROFILE_UPDATE_APPROACH=facts` The compare step gets an `update` verdict for a later state of the same thing, written next to the old state; a `conflict` must also share most of its words with the row (`same_subject()`). Rows are saved as `profile["fact_rows_json"]` and reloaded by `load_rows()`, so source kinds and repeat counts survive. `update_profile(approach="facts")` runs it in the pipeline. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017KuZSfznoSE6bnm9t8KWMf --- CLAUDE.md | 2 +- README.md | 2 +- configs/guantanamo/prompts/fact_compare.md | 3 +- src/engine/fact_list.py | 100 +++++++++++++++--- src/engine/profiles.py | 63 +++++++++++- tests/test_fact_list.py | 112 +++++++++++++++++++++ 6 files changed, 262 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7b77216..c206ad7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,7 +55,7 @@ The CLI entry point `src/process_and_extract.py` coordinates the pipeline end to - **Name-variant fallback**: when no candidate clears the similarity threshold, a candidate with a spelling variant of the incoming name still goes to the match check (`names_are_variants()` in `src/utils/name_variants.py`, searched over every entity, not just the capped lexical shortlist). Variants: the same name after `normalize_for_match()` (which drops accents, so "Guantánamo Bay" = "Guantanamo Bay"), a configured equivalence group ("Gitmo"), for locations/organizations the same core name without a leading "the"/"U.S.", a trailing "County"/"City" or a trailing "in/at/near …" description, one word respelled as a plural, ordinal or shortened form ("Commission"/"Commissions", "11"/"11th", "Counterterror"/"Counterterrorism"), or middle words dropped with the same first and last word ("Guantánamo Navy base" / "Guantánamo Bay Navy Base"); for people the same first and last name ("Donald J. Trump"), the same last name with a first name shortened or one letter off ("Clay"/"Clayton", "Shelley"/"Shelly"), or the reverse, or a bare surname ("Trump"); for events the same words in any order ("Firing of Harvey Rishikof as convening authority"). Substring containment and derived acronyms don't count. On the merge gold set this sends 22 of 25 labelled-same pairs to the match check (was 12) and 1 of 166 labelled-different pairs (unchanged). If the match check rejects the closest candidate, the best name-variant candidate gets one more match check. - **Merge dispute agent**: When a match result falls in the "gray band" (similarity within ±`MERGE_GRAY_BAND_DELTA` of threshold) with confidence below `MERGE_UNCERTAIN_CONFIDENCE_CUTOFF`, `MergeDisputeAgent` provides a second-stage LLM analysis that can override the initial merge/skip decision. Deferred cases are appended to `{output_dir}/merge_review_queue.jsonl` for human review. When the final verdict is skip or defer, the incoming entity is created as a new entity rather than dropped (unless an entity with the same key already exists, which is never overwritten). - **Canonical name selection**: 5-layer deterministic scoring (`score_canonical_name()` in `src/utils/name_variants.py`) picks the best display name, penalizing acronyms, generic phrases, contextual suffixes, and ALL CAPS names (`is_shouting_name()`; a same-name-in-normal-case incoming entity replaces an ALL CAPS one). -9. **Profile Versioning**: `src/engine/profiles.py` maintains `VersionedProfile` history whenever entity content changes. Creation generates a first draft, then runs up to 3 reflect/regenerate rounds (`iterative_improve()`). Updates apply a list of edits from one model call (`src/engine/profile_edits.py`, no reflection); `HINBOX_PROFILE_UPDATE_APPROACH=rewrite` (or `update_profile(..., approach="rewrite")`) brings back the old whole-profile rewrite with reflection. +9. **Profile Versioning**: `src/engine/profiles.py` maintains `VersionedProfile` history whenever entity content changes. Creation generates a first draft, then runs up to 3 reflect/regenerate rounds (`iterative_improve()`). Updates apply a list of edits from one model call (`src/engine/profile_edits.py`, no reflection); `HINBOX_PROFILE_UPDATE_APPROACH=rewrite` (or `update_profile(..., approach="rewrite")`) brings back the old whole-profile rewrite with reflection. `HINBOX_PROFILE_UPDATE_APPROACH=facts` uses the fact-list updater (`src/engine/fact_list.py`) instead; its rows (source kinds and quotes, repeat counts, which row a conflict disputes) are saved as a JSON string under `profile["fact_rows_json"]` and reloaded on the next update (`load_rows()`, which drops saved rows whose sentence is no longer in the text). Its compare step has an `update` verdict for a later state of the same thing (moved camps, a decision after "still deciding"), written as a timeline entry next to the old one; a `conflict` must also pass `values_differ()` and `same_subject()` in code. 10. **Profile Grounding**: Post-processing verification (`verify_profile_grounding()`) extracts citation markers from profile text, looks up the cited source articles, and uses an LLM to check whether each claim is supported. The `GroundingReport` includes a grounding score and per-claim support levels. The pass (`src/engine/grounding_pass.py`) checks up to `grounding_workers` profiles at once after all merging. 11. **Persistence**: Batched Parquet writes per entity type via `write_entities_table()` avoid write amplification. Every row is given every key before writing, because Arrow takes its columns from the first row only. Article processing status is tracked in a sidecar JSON file (`ProcessingStatus`) rather than rewriting the articles Parquet. An article is marked processed only when every extraction and merge step worked: if an extraction call raised, a type's merge raised, or a profile create/update call failed (`incomplete_reasons()` in `src/process_and_extract.py`; the merger counts these in `MergeStats.failed_profiles`), it is recorded as `incomplete` and retried next run. A failed profile update leaves the article unlinked from that entity so the retry updates the profile. Entity tables are written before the status sidecar (`save_entities_then_status()`), so a failed entity write leaves articles unprocessed. Ctrl+C during article processing or the grounding pass saves the entities merged so far and exits with 130. diff --git a/README.md b/README.md index 411004f..013b34f 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,7 @@ Gold files live in `{output_dir}/eval/`. Each run writes its per-item prediction The pipeline's `update_profile` uses an edit-list updater (`--approach rewrite` scores the old whole-profile rewrite instead). Rather than writing the whole profile again, the model returns a short list of edits: add these sentences after this existing sentence (or under this heading, or at the end), or replace this one sentence with a sharper version. Code applies the edits to the old text, so a sentence the model never mentions stays exactly as it was, and a replaced sentence keeps its citations. The prompt (`profile_edit.cloud.md` for cloud models, `profile_edit.local.md` for local ones) also says to keep both versions, each with its source, when the article contradicts the profile, and to add little or nothing from an article that only mentions the entity in passing. On the dev and test cases it passes 0.89 on Luna against 0.29 for the rewrite, in about 2s per update instead of 21s. Set `HINBOX_PROFILE_UPDATE_APPROACH=rewrite` to run the pipeline with the old rewrite. -`--approach facts` scores a prototype for very long profiles (`src/engine/fact_list.py`), which no model call ever sees whole. The model lists the article's claims about the entity, each with an exact quote that code checks against the article. Code then finds the few existing sentences most like each claim, and the model says whether the claim is the same, new, or in conflict. A repeated claim is only counted, not cited again, unless it comes from a different kind of source (a court record where the profile rested on a newspaper). A conflict is written by code right after the old sentence, so both versions stay. New facts are placed one section at a time. It isn't used by the pipeline yet. +`--approach facts` scores a prototype for very long profiles (`src/engine/fact_list.py`), which no model call ever sees whole. The model lists the article's claims about the entity, each with an exact quote that code checks against the article. Code then finds the few existing sentences most like each claim, and the model says whether the claim is the same, new, or in conflict. A repeated claim is only counted, not cited again, unless it comes from a different kind of source (a court record where the profile rested on a newspaper). A conflict is written by code right after the old sentence, so both versions stay. New facts are placed one section at a time. Set `HINBOX_PROFILE_UPDATE_APPROACH=facts` to use it in `just process`; each profile then keeps its fact rows (who said what, which articles only repeated a fact) under `fact_rows_json` for the next update. A later state of the same thing (a transfer, a decision after "still deciding") is kept as a timeline, not called a conflict. **DSPy experiments** live in [`experiments/dspy_opt/`](experiments/dspy_opt/README.md), in their own environment, so DSPy never enters Hinbox's main dependencies, and LiteLLM is never installed. `just dspy-merge --model --optimizer mipro` optimises the merge check for one model and puts the result on the same scoreboard. `just dspy-profile --model --reflection-model openai/gpt-6-luna` runs GEPA on the edit-list profile updater. diff --git a/configs/guantanamo/prompts/fact_compare.md b/configs/guantanamo/prompts/fact_compare.md index 491b1c5..b0bbad1 100644 --- a/configs/guantanamo/prompts/fact_compare.md +++ b/configs/guantanamo/prompts/fact_compare.md @@ -6,9 +6,10 @@ You keep a list of sourced facts about the {entity_type} "{entity_name}". A new - `same`: one of the known facts already says this, possibly in other words. Give its id in `row`. Set `new_evidence` to true only when the claim comes from a different kind of source than the known fact and that matters to a historian: a court record, official document or first-hand witness where the known fact rested on a secondhand report, or the reverse. A second newspaper repeating the same thing is not new evidence. When the known fact's source is "not recorded", judge from its wording ("a court filing said", "according to his lawyer"); if it gives no source either, set `new_evidence` only for a court record, official document or first-hand witness. - `conflict`: one known fact gives a different value for the very same thing, so both can't be true: the same event on a different date, the same count with a different number. Give its id in `row`. +- `update`: the claim is a later state of the same thing as a known fact, so both were true at their own time: a detainee moved to another camp, a decision made after "still deciding", a new trial date, a new count for a later date, a new role. Give the known fact's id in `row`. Both stay in the profile as a timeline. - `new`: none of the known facts says this. Also use `new` when the claim adds real detail (a date, an outcome, a new step) to a known fact. -These are not conflicts (most are `new`): +These are not conflicts (they are `update` or `new`): - two different events or steps, such as charges filed in one month and a trial ordered in another; - an older report saying something hadn't happened yet; - a later development, such as a new role, a transfer, or a rescheduled hearing; diff --git a/src/engine/fact_list.py b/src/engine/fact_list.py index f6a4802..b8c28c3 100644 --- a/src/engine/fact_list.py +++ b/src/engine/fact_list.py @@ -30,6 +30,7 @@ were, word for word. """ +import json import re from difflib import SequenceMatcher from typing import Any, Callable, Dict, List, Literal, Optional, Tuple @@ -45,7 +46,9 @@ MAX_CANDIDATES = 4 # existing rows shown beside each new claim MIN_SHARED = 2 # word stems a row must share with a claim to be shown MAX_PASSING_CLAIMS = 2 # claims kept from an article not mainly about the entity -MIN_QUOTE_WORDS = 0.9 # share of a loose quote's words that must be in the article +MIN_QUOTE_WORDS = 0.9 +# Share of the shorter sentence's word stems a conflicting row must share +SAME_SUBJECT = 0.4 # share of a loose quote's words that must be in the article _CITATION = re.compile(r"\^\[([^\]]+)\]") _HEADING = re.compile(r"^#{1,6} .*$", re.MULTILINE) _NUMBER = re.compile(r"\b\d+(?:[.,]\d+)*\b") @@ -87,8 +90,10 @@ def _parse(cls, value: Any) -> Any: class ClaimVerdict(BaseModel): claim: int = Field(description="The claim's number") - verdict: Literal["same", "new", "conflict"] - row: str = Field(default="", description="The known fact's id, for same/conflict") + verdict: Literal["same", "new", "update", "conflict"] + row: str = Field( + default="", description="The known fact's id, for same/update/conflict" + ) new_evidence: bool = False @@ -153,7 +158,7 @@ def rows_from_profile(text: str) -> List[Dict[str, Any]]: continue rows.append( { - "id": f"f{len(rows) + 1}", + "id": f"f{_next_id(rows)}", "section": section, "claim": _CITATION.sub("", sentence).strip(), "text": sentence, @@ -164,6 +169,33 @@ def rows_from_profile(text: str) -> List[Dict[str, Any]]: return rows +FACT_ROWS_KEY = "fact_rows_json" # a JSON string, so Parquet sees one column + + +def load_rows(profile: Dict[str, Any]) -> List[Dict[str, Any]]: + """The profile's saved rows, plus rows for cited sentences they don't cover. + + Saved rows keep what the text can't: each source's kind and quote, the + articles that only repeated a fact, which row a conflict disputes. A saved + row whose sentence is no longer in the text (the profile was rewritten + another way) is dropped, so rows and text can't drift apart. + """ + text = profile.get("text", "") + try: + saved = json.loads(profile.get(FACT_ROWS_KEY) or "[]") + except (TypeError, ValueError): + saved = [] + rows = [r for r in saved if r.get("text") and r["text"] in text] + covered = {r["text"] for r in rows} + used = {int(r["id"][1:]) for r in rows if str(r.get("id", ""))[1:].isdigit()} + next_id = max(used, default=0) + 1 + for row in rows_from_profile(text): + if row["text"] not in covered: + rows.append({**row, "id": f"f{next_id}"}) + next_id += 1 + return rows + + # ---------------------------------------------------------------- checks by code @@ -258,6 +290,15 @@ def values_differ(claim: str, row_claim: str) -> bool: return bool(counts - r_counts and r_counts - counts and years & r_years) +def same_subject(claim: str, row_claim: str) -> bool: + """Code's second check on a ``conflict``: the two sentences are mostly + about the same thing, numbers aside. Two different events with different + years ("still deciding" in 2009, "decided" in 2011) share less.""" + mine, theirs = _stems(claim), _stems(row_claim) + smaller = min(len(mine), len(theirs)) or 1 + return len(mine & theirs) / smaller >= SAME_SUBJECT + + def _match_heading( section: str, headings: List[str], article: str = "", near: str = "" ) -> str: @@ -368,7 +409,7 @@ def update_from_facts( model = model or (OLLAMA_MODEL if local else CLOUD_MODEL) generate = generate or (local_generation if local else cloud_generation) text = existing_profile.get("text", "") - rows = rows_from_profile(text) + rows = load_rows(existing_profile) chunks = split_sections(text) headings = [h for h in (_heading(c) for c in chunks) if h] names = {"entity_type": entity_type, "entity_name": entity_name} @@ -413,7 +454,14 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: by_id = {r["id"]: r for r in rows} section_edits: Dict[str, List[ProfileEdit]] = {} new_facts: Dict[str, List[Tuple[int, ArticleClaim]]] = {} - tally = {"same": 0, "new_evidence": 0, "conflict": 0, "new": 0, "not_conflict": 0} + tally = { + "same": 0, + "new_evidence": 0, + "conflict": 0, + "not_conflict": 0, + "update": 0, + "new": 0, + } for n, (claim, rows_for) in enumerate(zip(claims, found), 1): verdict = verdicts.get(n) # A verdict naming a row it wasn't shown is a slip: treat it as new @@ -433,26 +481,40 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: section_edits.setdefault(row["section"], []).append( ProfileEdit(op="replace", anchor=row["text"], text=cited) ) + row["text"] = cited else: row["repeats"].append(article_id) continue if verdict and row in rows_for and verdict.verdict == "conflict": - if not values_differ(claim.claim, row["claim"]): + if not ( + values_differ(claim.claim, row["claim"]) + and same_subject(claim.claim, row["claim"]) + ): tally["not_conflict"] += 1 # written as a plain new fact below verdict = None + if verdict and row in rows_for and verdict.verdict == "update": + # A later state of the same thing (moved camps, decided after + # deciding): a new fact on the timeline, linked to the old row + tally["update"] += 1 + claim.section = row["section"] # next to the state it follows + verdict = None if verdict and row in rows_for and verdict.verdict == "conflict": # Written by code, right after the old sentence: both stay, each cited tally["conflict"] += 1 body = _CITATION.sub("", claim.claim).strip() + sentence = f"Another report differs: {body}^[{article_id}]" section_edits.setdefault(row["section"], []).append( - ProfileEdit( - op="add", - anchor=row["text"], - text=f"Another report differs: {body}^[{article_id}]", - ) + ProfileEdit(op="add", anchor=row["text"], text=sentence) ) rows.append( - _new_row(rows, row["section"], claim, article_id, disputes=row["id"]) + _new_row( + rows, + row["section"], + claim, + article_id, + text=sentence, + disputes=row["id"], + ) ) continue tally["new"] += 1 @@ -540,11 +602,21 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: **existing_profile, "text": new_text, "sources": sources + ([article_id] if article_id not in sources else []), + # Rows whose sentence made it into the text; saved for the next update + FACT_ROWS_KEY: json.dumps( + [r for r in rows if r.get("text") and r["text"] in new_text], + ensure_ascii=False, + ), "fact_rows": rows, "fact_report": report, } +def _next_id(rows: List[Dict[str, Any]]) -> int: + ids = [int(r["id"][1:]) for r in rows if str(r.get("id", ""))[1:].isdigit()] + return max(ids, default=0) + 1 + + def _new_row( rows: List[Dict[str, Any]], section: str, @@ -554,7 +626,7 @@ def _new_row( disputes: Optional[str] = None, ) -> Dict[str, Any]: row: Dict[str, Any] = { - "id": f"f{len(rows) + 1}", + "id": f"f{_next_id(rows)}", "section": section, "claim": claim.claim, "text": text, diff --git a/src/engine/profiles.py b/src/engine/profiles.py index d968375..a4207b6 100644 --- a/src/engine/profiles.py +++ b/src/engine/profiles.py @@ -517,6 +517,62 @@ def _edit_profile_update( }, [] +def _fact_profile_update( + entity_type: str, + entity_name: str, + existing_profile: Dict, + new_article_text: str, + new_article_id: str, + model_type: str = "gemini", + domain: str = "guantanamo", + model: Optional[str] = None, +) -> Tuple[Dict, list]: + """Update from the profile's fact list (``src/engine/fact_list.py``). + + Saves the rows under ``fact_rows_json`` for the next update; the step + report is logged, not stored. Falls back to the existing profile on failure. + """ + from src.engine.fact_list import FACT_ROWS_KEY, update_from_facts + + if _resolve_mode(model_type, model) == GenerationMode.LOCAL and not model: + model = OLLAMA_MODEL + try: + updated = update_from_facts( + entity_type, + entity_name, + existing_profile, + new_article_text, + new_article_id, + model=model, + domain=domain, + ) + except Exception as e: + log( + f"Fact-list update failed for {entity_name}, keeping existing profile: {e}", + level="warning", + ) + return existing_profile, [] + report = updated["fact_report"] + log( + f"Fact-list update for {entity_name}: {report.get('claims', 0)} claims, " + f"{report.get('new', 0)} new, {report.get('same', 0)} same, " + f"{report.get('update', 0)} timeline, {report.get('conflict', 0)} conflict", + level="debug", + ) + return { + **existing_profile, + "text": updated["text"], + "sources": updated["sources"], + FACT_ROWS_KEY: updated[FACT_ROWS_KEY], + }, [] + + +_UPDATERS = { + "edit": _edit_profile_update, + "facts": _fact_profile_update, +} + + @timed("profile.update") def update_profile( entity_type: str, @@ -533,14 +589,15 @@ def update_profile( """ Update profile and add version if versioning is enabled. - ``approach`` is "edit" or "rewrite" (default: ``PROFILE_UPDATE_APPROACH``). + ``approach`` is "edit", "facts" (the fact-list updater) or "rewrite" + (default: ``PROFILE_UPDATE_APPROACH``). Returns (updated_profile, versioned_profile, improvement_history); the edit approach has no reflection rounds, so its history is empty. """ approach = approach or PROFILE_UPDATE_APPROACH - if approach not in ("edit", "rewrite"): + if approach not in ("edit", "rewrite", "facts"): raise ValueError(f"Unknown profile update approach: {approach!r}") - update = _edit_profile_update if approach == "edit" else _update_profile_internal + update = _UPDATERS.get(approach, _update_profile_internal) updated_profile, history = update( entity_type, entity_name, diff --git a/tests/test_fact_list.py b/tests/test_fact_list.py index 75798f4..d9fc009 100644 --- a/tests/test_fact_list.py +++ b/tests/test_fact_list.py @@ -1,6 +1,9 @@ """The fact-list updater keeps old text, collapses repeats and keeps both sides.""" +import json + from src.engine.fact_list import ( + FACT_ROWS_KEY, ArticleClaim, ArticleClaims, ClaimVerdict, @@ -9,8 +12,10 @@ SectionPlacements, _match_heading, candidates, + load_rows, quote_found, rows_from_profile, + same_subject, split_sections, update_from_facts, values_differ, @@ -379,3 +384,110 @@ def test_a_loose_quote_must_match_one_stretch_in_order(): ) assert not quote_found("Nashiri was not captured in 2002.", article) assert quote_found("His lawyer said that he was not charged until 2011", article) + + +def test_a_later_state_is_a_timeline_entry_and_rows_survive_to_the_next_update(): + claim = ArticleClaim( + claim="Nashiri was moved from the secret sites to Guantánamo in 2006.", + section="## Other", + quote="judge ruled", + source="Pentagon statement", + ) + + def verdicts(messages): + return ClaimVerdicts( + verdicts=[ClaimVerdict(claim=1, verdict="update", row="f3")] + ) + + answers = { + "ArticleClaims": lambda m: ArticleClaims(claims=[claim]), + "ClaimVerdicts": verdicts, + "SectionPlacements": lambda m: SectionPlacements(), + } + generate, _ = _fake(answers) + new = update_from_facts( + "people", + "Nashiri", + {"text": PROFILE}, + ARTICLE, + "a9", + model="m", + generate=generate, + ) + assert new["fact_report"]["update"] == 1 + assert "Another report differs" not in new["text"] + # Goes in the section of the state it follows, not a new "## Other" + assert ( + "secret sites.^[a2]\nNashiri was moved from the secret sites to Guantánamo in 2006.^[a9]" + in new["text"] + ) + # The saved rows keep the new row's source kind for the next update + rows = load_rows({"text": new["text"], FACT_ROWS_KEY: new[FACT_ROWS_KEY]}) + moved = next(r for r in rows if "moved" in r["claim"]) + assert moved["sources"][0]["kind"] == "Pentagon statement" + assert len({r["id"] for r in rows}) == len(rows) == 6 + + +def test_saved_rows_whose_sentence_is_gone_are_dropped(): + saved = json.dumps( + [ + { + "id": "f9", + "section": "## Capture", + "claim": "Old", + "text": "Gone.^[a1]", + "sources": [{"article_id": "a1", "kind": "court filing"}], + "repeats": [], + }, + { + "id": "f2", + "section": "## Capture", + "claim": "He was held by the CIA at secret sites.", + "text": "He was held by the CIA at secret sites.^[a2]", + "sources": [{"article_id": "a2", "kind": "court filing"}], + "repeats": ["a7"], + }, + ] + ) + rows = load_rows({"text": PROFILE, FACT_ROWS_KEY: saved}) + assert "Gone.^[a1]" not in [r["text"] for r in rows] + held = next(r for r in rows if r["id"] == "f2") + assert held["repeats"] == ["a7"] and held["sources"][0]["kind"] == "court filing" + assert len(rows) == 5 # the other four sentences are read from the text + + +def test_a_conflict_must_be_about_the_same_thing(): + assert same_subject( + "The camps opened in January 2003.", "The prison camps opened in January 2002." + ) + assert not same_subject( + "Attorney General Eric Holder decided the defendants should face federal " + "trials in New York.", + "The Obama administration was still deciding where to stage the trials of " + "five alleged co-conspirators.", + ) + + +def test_update_profile_can_use_the_fact_list(monkeypatch): + from src.engine import fact_list, profiles + + def fake(*args, **kwargs): + return { + "text": PROFILE + "New.^[a9]\n", + "sources": ["a9"], + "fact_report": {}, + FACT_ROWS_KEY: "[]", + } + + monkeypatch.setattr(fact_list, "update_from_facts", fake) + profile, _, history = profiles.update_profile( + "people", + "Nashiri", + {"text": PROFILE, "sources": []}, + profiles.VersionedProfile(), + ARTICLE, + "a9", + approach="facts", + ) + assert profile["text"].endswith("New.^[a9]\n") and profile[FACT_ROWS_KEY] == "[]" + assert history == [] From 7eb586dedd7cc16613156e5d7b933fe4bc2d1264 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 10:21:53 +0000 Subject: [PATCH 2/7] Treat a fact-list `update` that moves a past event's date as a conflict Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017KuZSfznoSE6bnm9t8KWMf --- configs/guantanamo/prompts/fact_compare.md | 2 +- src/engine/fact_list.py | 31 +++++++++++++++++++--- tests/test_fact_list.py | 31 ++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/configs/guantanamo/prompts/fact_compare.md b/configs/guantanamo/prompts/fact_compare.md index b0bbad1..2553680 100644 --- a/configs/guantanamo/prompts/fact_compare.md +++ b/configs/guantanamo/prompts/fact_compare.md @@ -6,7 +6,7 @@ You keep a list of sourced facts about the {entity_type} "{entity_name}". A new - `same`: one of the known facts already says this, possibly in other words. Give its id in `row`. Set `new_evidence` to true only when the claim comes from a different kind of source than the known fact and that matters to a historian: a court record, official document or first-hand witness where the known fact rested on a secondhand report, or the reverse. A second newspaper repeating the same thing is not new evidence. When the known fact's source is "not recorded", judge from its wording ("a court filing said", "according to his lawyer"); if it gives no source either, set `new_evidence` only for a court record, official document or first-hand witness. - `conflict`: one known fact gives a different value for the very same thing, so both can't be true: the same event on a different date, the same count with a different number. Give its id in `row`. -- `update`: the claim is a later state of the same thing as a known fact, so both were true at their own time: a detainee moved to another camp, a decision made after "still deciding", a new trial date, a new count for a later date, a new role. Give the known fact's id in `row`. Both stay in the profile as a timeline. +- `update`: the claim is a later state of the same thing as a known fact, so both were true at their own time: a detainee moved to another camp, a decision made after "still deciding", a new trial date, a new count for a later date, a new role. Give the known fact's id in `row`. Both stay in the profile as a timeline. A completed past event can't be updated: a different date or place for the same bombing, capture, arrival or ruling is a `conflict`, not an `update`. - `new`: none of the known facts says this. Also use `new` when the claim adds real detail (a date, an outcome, a new step) to a known fact. These are not conflicts (they are `update` or `new`): diff --git a/src/engine/fact_list.py b/src/engine/fact_list.py index b8c28c3..3ff97cd 100644 --- a/src/engine/fact_list.py +++ b/src/engine/fact_list.py @@ -48,7 +48,10 @@ MAX_PASSING_CLAIMS = 2 # claims kept from an article not mainly about the entity MIN_QUOTE_WORDS = 0.9 # Share of the shorter sentence's word stems a conflicting row must share -SAME_SUBJECT = 0.4 # share of a loose quote's words that must be in the article +SAME_SUBJECT = 0.4 +# ...and the share above which an ``update`` with a different date is really +# the same event given a different date (a conflict) +SAME_EVENT = 0.6 # share of a loose quote's words that must be in the article _CITATION = re.compile(r"\^\[([^\]]+)\]") _HEADING = re.compile(r"^#{1,6} .*$", re.MULTILINE) _NUMBER = re.compile(r"\b\d+(?:[.,]\d+)*\b") @@ -290,13 +293,22 @@ def values_differ(claim: str, row_claim: str) -> bool: return bool(counts - r_counts and r_counts - counts and years & r_years) -def same_subject(claim: str, row_claim: str) -> bool: +def dates_differ(claim: str, row_claim: str) -> bool: + """Each sentence gives a year or month the other doesn't.""" + (years, months, _), (r_years, r_months, _) = _values(claim), _values(row_claim) + return bool( + (years - r_years and r_years - years) + or (months - r_months and r_months - months) + ) + + +def same_subject(claim: str, row_claim: str, threshold: float = 0.0) -> bool: """Code's second check on a ``conflict``: the two sentences are mostly about the same thing, numbers aside. Two different events with different years ("still deciding" in 2009, "decided" in 2011) share less.""" mine, theirs = _stems(claim), _stems(row_claim) smaller = min(len(mine), len(theirs)) or 1 - return len(mine & theirs) / smaller >= SAME_SUBJECT + return len(mine & theirs) / smaller >= (threshold or SAME_SUBJECT) def _match_heading( @@ -460,6 +472,7 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: "conflict": 0, "not_conflict": 0, "update": 0, + "update_to_conflict": 0, "new": 0, } for n, (claim, rows_for) in enumerate(zip(claims, found), 1): @@ -492,6 +505,18 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: ): tally["not_conflict"] += 1 # written as a plain new fact below verdict = None + if ( + verdict + and row in rows_for + and verdict.verdict == "update" + and dates_differ(claim.claim, row["claim"]) + and same_subject(claim.claim, row["claim"], SAME_EVENT) + ): + # A past event can't move: "the bombing was on Oct. 12, 1999" + # against "Oct. 12, 2000" in nearly the same words is a conflict. + # On test, Qwen and Luna called every planted date change an update + tally["update_to_conflict"] += 1 + verdict = verdict.model_copy(update={"verdict": "conflict"}) if verdict and row in rows_for and verdict.verdict == "update": # A later state of the same thing (moved camps, decided after # deciding): a new fact on the timeline, linked to the old row diff --git a/tests/test_fact_list.py b/tests/test_fact_list.py index d9fc009..3706a32 100644 --- a/tests/test_fact_list.py +++ b/tests/test_fact_list.py @@ -491,3 +491,34 @@ def fake(*args, **kwargs): ) assert profile["text"].endswith("New.^[a9]\n") and profile[FACT_ROWS_KEY] == "[]" assert history == [] + + +def test_a_new_date_for_the_same_past_event_is_a_conflict_not_an_update(): + claim = ArticleClaim( + claim="Nashiri was captured in the United Arab Emirates in November 2003.", + section="## Capture", + quote="captured in the United Arab Emirates in October 2002", + ) + + def verdicts(messages): + return ClaimVerdicts( + verdicts=[ClaimVerdict(claim=1, verdict="update", row="f2")] + ) + + answers = { + "ArticleClaims": lambda m: ArticleClaims(claims=[claim]), + "ClaimVerdicts": verdicts, + "SectionPlacements": lambda m: SectionPlacements(), + } + generate, _ = _fake(answers) + new = update_from_facts( + "people", + "Nashiri", + {"text": PROFILE}, + ARTICLE, + "a9", + model="m", + generate=generate, + ) + assert new["fact_report"]["update_to_conflict"] == 1 + assert "November 2002.^[a1] Another report differs:" in new["text"] From cc1158f899875052566a8b08f9936eb70af08ed9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 04:37:34 +0000 Subject: [PATCH 3/7] Write a fact-list `same` claim that adds a number or date the row lacks Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017KuZSfznoSE6bnm9t8KWMf --- src/engine/fact_list.py | 18 ++++++++++++++++++ tests/test_fact_list.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/src/engine/fact_list.py b/src/engine/fact_list.py index 3ff97cd..4d59ee3 100644 --- a/src/engine/fact_list.py +++ b/src/engine/fact_list.py @@ -293,6 +293,13 @@ def values_differ(claim: str, row_claim: str) -> bool: return bool(counts - r_counts and r_counts - counts and years & r_years) +def adds_values(claim: str, row_claim: str) -> bool: + """The claim has a year, month or number the row doesn't.""" + return any( + mine - theirs for mine, theirs in zip(_values(claim), _values(row_claim)) + ) + + def dates_differ(claim: str, row_claim: str) -> bool: """Each sentence gives a year or month the other doesn't.""" (years, months, _), (r_years, r_months, _) = _values(claim), _values(row_claim) @@ -473,12 +480,23 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: "not_conflict": 0, "update": 0, "update_to_conflict": 0, + "same_with_detail": 0, "new": 0, } for n, (claim, rows_for) in enumerate(zip(claims, found), 1): verdict = verdicts.get(n) # A verdict naming a row it wasn't shown is a slip: treat it as new row = by_id.get(verdict.row) if verdict else None + if ( + verdict + and row in rows_for + and verdict.verdict == "same" + and adds_values(claim.claim, row["claim"]) + ): + # "Same" but with a number or date the row lacks ("a 78-minute + # secret session"): the detail is new, so write it, don't drop it + tally["same_with_detail"] += 1 + verdict = None if verdict and row in rows_for and verdict.verdict == "same": tally["same"] += 1 if verdict.new_evidence: diff --git a/tests/test_fact_list.py b/tests/test_fact_list.py index 3706a32..9388bfd 100644 --- a/tests/test_fact_list.py +++ b/tests/test_fact_list.py @@ -522,3 +522,32 @@ def verdicts(messages): ) assert new["fact_report"]["update_to_conflict"] == 1 assert "November 2002.^[a1] Another report differs:" in new["text"] + + +def test_a_same_claim_with_a_new_number_is_written_not_dropped(): + claim = ArticleClaim( + claim="Nashiri was held by the CIA at secret sites for 4 years.", + section="## Capture", + quote="He was held by the CIA at secret sites.", + ) + + def verdicts(messages): + return ClaimVerdicts(verdicts=[ClaimVerdict(claim=1, verdict="same", row="f3")]) + + answers = { + "ArticleClaims": lambda m: ArticleClaims(claims=[claim]), + "ClaimVerdicts": verdicts, + "SectionPlacements": lambda m: SectionPlacements(), + } + generate, _ = _fake(answers) + new = update_from_facts( + "people", + "Nashiri", + {"text": PROFILE}, + ARTICLE, + "a9", + model="m", + generate=generate, + ) + assert new["fact_report"]["same_with_detail"] == 1 + assert "for 4 years.^[a9]" in new["text"] From 5795add00137e4c8603f9fcf2fff30a019dfd476 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 05:02:30 +0000 Subject: [PATCH 4/7] Turn fact-list `same` verdicts with a different year or month into conflicts; loosen the `update` check Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017KuZSfznoSE6bnm9t8KWMf --- src/engine/fact_list.py | 22 +++++++++++++++++++--- tests/test_fact_list.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/engine/fact_list.py b/src/engine/fact_list.py index 4d59ee3..58fdbb7 100644 --- a/src/engine/fact_list.py +++ b/src/engine/fact_list.py @@ -50,8 +50,9 @@ # Share of the shorter sentence's word stems a conflicting row must share SAME_SUBJECT = 0.4 # ...and the share above which an ``update`` with a different date is really -# the same event given a different date (a conflict) -SAME_EVENT = 0.6 # share of a loose quote's words that must be in the article +# the same event given a different date (a conflict). 0.6 never fired in 180+ +# updates on dev/test: claims are worded differently from the stored row +SAME_EVENT = 0.4 # share of a loose quote's words that must be in the article _CITATION = re.compile(r"\^\[([^\]]+)\]") _HEADING = re.compile(r"^#{1,6} .*$", re.MULTILINE) _NUMBER = re.compile(r"\b\d+(?:[.,]\d+)*\b") @@ -481,12 +482,26 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: "update": 0, "update_to_conflict": 0, "same_with_detail": 0, + "same_to_conflict": 0, "new": 0, } for n, (claim, rows_for) in enumerate(zip(claims, found), 1): verdict = verdicts.get(n) # A verdict naming a row it wasn't shown is a slip: treat it as new row = by_id.get(verdict.row) if verdict else None + trusted_match = False + if ( + verdict + and row in rows_for + and verdict.verdict == "same" + and dates_differ(claim.claim, row["claim"]) + ): + # The model says it's the same claim, yet the year or month + # differs: that is a conflict. On test Luna called "the bombing + # was on Oct. 12, 1999" the same as the row giving 2000 + tally["same_to_conflict"] += 1 + verdict = verdict.model_copy(update={"verdict": "conflict"}) + trusted_match = True if ( verdict and row in rows_for @@ -519,7 +534,7 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: if verdict and row in rows_for and verdict.verdict == "conflict": if not ( values_differ(claim.claim, row["claim"]) - and same_subject(claim.claim, row["claim"]) + and (trusted_match or same_subject(claim.claim, row["claim"])) ): tally["not_conflict"] += 1 # written as a plain new fact below verdict = None @@ -530,6 +545,7 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: and dates_differ(claim.claim, row["claim"]) and same_subject(claim.claim, row["claim"], SAME_EVENT) ): + trusted_match = True # A past event can't move: "the bombing was on Oct. 12, 1999" # against "Oct. 12, 2000" in nearly the same words is a conflict. # On test, Qwen and Luna called every planted date change an update diff --git a/tests/test_fact_list.py b/tests/test_fact_list.py index 9388bfd..801c925 100644 --- a/tests/test_fact_list.py +++ b/tests/test_fact_list.py @@ -551,3 +551,32 @@ def verdicts(messages): ) assert new["fact_report"]["same_with_detail"] == 1 assert "for 4 years.^[a9]" in new["text"] + + +def test_a_same_verdict_with_a_different_year_is_a_conflict(): + claim = ArticleClaim( + claim="The CIA held Nashiri, who was seized in November 2003.", + section="## Capture", + quote="captured in the United Arab Emirates in October 2002", + ) + + def verdicts(messages): + return ClaimVerdicts(verdicts=[ClaimVerdict(claim=1, verdict="same", row="f2")]) + + answers = { + "ArticleClaims": lambda m: ArticleClaims(claims=[claim]), + "ClaimVerdicts": verdicts, + "SectionPlacements": lambda m: SectionPlacements(), + } + generate, _ = _fake(answers) + new = update_from_facts( + "people", + "Nashiri", + {"text": PROFILE}, + ARTICLE, + "a9", + model="m", + generate=generate, + ) + assert new["fact_report"]["same_to_conflict"] == 1 + assert "November 2002.^[a1] Another report differs:" in new["text"] From 24d42adc5f4fc76a90f3b7b297f510382ca3abe9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 05:17:35 +0000 Subject: [PATCH 5/7] Revert the looser fact-list conflict conversion (5795add) On dev/test it brought back false "Another report differs" sentences (6 per test run) and still caught 0 of 4 planted disputes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017KuZSfznoSE6bnm9t8KWMf --- src/engine/fact_list.py | 22 +++------------------- tests/test_fact_list.py | 29 ----------------------------- 2 files changed, 3 insertions(+), 48 deletions(-) diff --git a/src/engine/fact_list.py b/src/engine/fact_list.py index 58fdbb7..4d59ee3 100644 --- a/src/engine/fact_list.py +++ b/src/engine/fact_list.py @@ -50,9 +50,8 @@ # Share of the shorter sentence's word stems a conflicting row must share SAME_SUBJECT = 0.4 # ...and the share above which an ``update`` with a different date is really -# the same event given a different date (a conflict). 0.6 never fired in 180+ -# updates on dev/test: claims are worded differently from the stored row -SAME_EVENT = 0.4 # share of a loose quote's words that must be in the article +# the same event given a different date (a conflict) +SAME_EVENT = 0.6 # share of a loose quote's words that must be in the article _CITATION = re.compile(r"\^\[([^\]]+)\]") _HEADING = re.compile(r"^#{1,6} .*$", re.MULTILINE) _NUMBER = re.compile(r"\b\d+(?:[.,]\d+)*\b") @@ -482,26 +481,12 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: "update": 0, "update_to_conflict": 0, "same_with_detail": 0, - "same_to_conflict": 0, "new": 0, } for n, (claim, rows_for) in enumerate(zip(claims, found), 1): verdict = verdicts.get(n) # A verdict naming a row it wasn't shown is a slip: treat it as new row = by_id.get(verdict.row) if verdict else None - trusted_match = False - if ( - verdict - and row in rows_for - and verdict.verdict == "same" - and dates_differ(claim.claim, row["claim"]) - ): - # The model says it's the same claim, yet the year or month - # differs: that is a conflict. On test Luna called "the bombing - # was on Oct. 12, 1999" the same as the row giving 2000 - tally["same_to_conflict"] += 1 - verdict = verdict.model_copy(update={"verdict": "conflict"}) - trusted_match = True if ( verdict and row in rows_for @@ -534,7 +519,7 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: if verdict and row in rows_for and verdict.verdict == "conflict": if not ( values_differ(claim.claim, row["claim"]) - and (trusted_match or same_subject(claim.claim, row["claim"])) + and same_subject(claim.claim, row["claim"]) ): tally["not_conflict"] += 1 # written as a plain new fact below verdict = None @@ -545,7 +530,6 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: and dates_differ(claim.claim, row["claim"]) and same_subject(claim.claim, row["claim"], SAME_EVENT) ): - trusted_match = True # A past event can't move: "the bombing was on Oct. 12, 1999" # against "Oct. 12, 2000" in nearly the same words is a conflict. # On test, Qwen and Luna called every planted date change an update diff --git a/tests/test_fact_list.py b/tests/test_fact_list.py index 801c925..9388bfd 100644 --- a/tests/test_fact_list.py +++ b/tests/test_fact_list.py @@ -551,32 +551,3 @@ def verdicts(messages): ) assert new["fact_report"]["same_with_detail"] == 1 assert "for 4 years.^[a9]" in new["text"] - - -def test_a_same_verdict_with_a_different_year_is_a_conflict(): - claim = ArticleClaim( - claim="The CIA held Nashiri, who was seized in November 2003.", - section="## Capture", - quote="captured in the United Arab Emirates in October 2002", - ) - - def verdicts(messages): - return ClaimVerdicts(verdicts=[ClaimVerdict(claim=1, verdict="same", row="f2")]) - - answers = { - "ArticleClaims": lambda m: ArticleClaims(claims=[claim]), - "ClaimVerdicts": verdicts, - "SectionPlacements": lambda m: SectionPlacements(), - } - generate, _ = _fake(answers) - new = update_from_facts( - "people", - "Nashiri", - {"text": PROFILE}, - ARTICLE, - "a9", - model="m", - generate=generate, - ) - assert new["fact_report"]["same_to_conflict"] == 1 - assert "November 2002.^[a1] Another report differs:" in new["text"] From 339ca4882d61fbee3dbcc8c9ef074256e2c7e99d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 05:21:12 +0000 Subject: [PATCH 6/7] Keep timeline updates as updates and save placed facts' sources in `fact_list` The update-to-conflict check never fired on the evals and could turn a rescheduled trial date into a false dispute, so it is removed. Facts the model placed now keep their sentence on their row, so their quote and source kind are saved in `fact_rows_json`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017KuZSfznoSE6bnm9t8KWMf --- src/engine/fact_list.py | 42 ++++++++++------------------------------- tests/test_fact_list.py | 39 +++++++++++++++++++++++++------------- 2 files changed, 36 insertions(+), 45 deletions(-) diff --git a/src/engine/fact_list.py b/src/engine/fact_list.py index 4d59ee3..8b4c836 100644 --- a/src/engine/fact_list.py +++ b/src/engine/fact_list.py @@ -49,9 +49,6 @@ MIN_QUOTE_WORDS = 0.9 # Share of the shorter sentence's word stems a conflicting row must share SAME_SUBJECT = 0.4 -# ...and the share above which an ``update`` with a different date is really -# the same event given a different date (a conflict) -SAME_EVENT = 0.6 # share of a loose quote's words that must be in the article _CITATION = re.compile(r"\^\[([^\]]+)\]") _HEADING = re.compile(r"^#{1,6} .*$", re.MULTILINE) _NUMBER = re.compile(r"\b\d+(?:[.,]\d+)*\b") @@ -300,22 +297,13 @@ def adds_values(claim: str, row_claim: str) -> bool: ) -def dates_differ(claim: str, row_claim: str) -> bool: - """Each sentence gives a year or month the other doesn't.""" - (years, months, _), (r_years, r_months, _) = _values(claim), _values(row_claim) - return bool( - (years - r_years and r_years - years) - or (months - r_months and r_months - months) - ) - - -def same_subject(claim: str, row_claim: str, threshold: float = 0.0) -> bool: +def same_subject(claim: str, row_claim: str) -> bool: """Code's second check on a ``conflict``: the two sentences are mostly about the same thing, numbers aside. Two different events with different years ("still deciding" in 2009, "decided" in 2011) share less.""" mine, theirs = _stems(claim), _stems(row_claim) smaller = min(len(mine), len(theirs)) or 1 - return len(mine & theirs) / smaller >= (threshold or SAME_SUBJECT) + return len(mine & theirs) / smaller >= SAME_SUBJECT def _match_heading( @@ -479,7 +467,6 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: "conflict": 0, "not_conflict": 0, "update": 0, - "update_to_conflict": 0, "same_with_detail": 0, "new": 0, } @@ -523,18 +510,6 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: ): tally["not_conflict"] += 1 # written as a plain new fact below verdict = None - if ( - verdict - and row in rows_for - and verdict.verdict == "update" - and dates_differ(claim.claim, row["claim"]) - and same_subject(claim.claim, row["claim"], SAME_EVENT) - ): - # A past event can't move: "the bombing was on Oct. 12, 1999" - # against "Oct. 12, 2000" in nearly the same words is a conflict. - # On test, Qwen and Luna called every planted date change an update - tally["update_to_conflict"] += 1 - verdict = verdict.model_copy(update={"verdict": "conflict"}) if verdict and row in rows_for and verdict.verdict == "update": # A later state of the same thing (moved camps, decided after # deciding): a new fact on the timeline, linked to the old row @@ -591,24 +566,27 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: placed = [] # One sentence may state several facts; any fact the model left out # is written as extracted, at the end of the section + # Each fact's row keeps the sentence that states it (shared when one + # sentence states several), so its quote and source kind are saved numbers = {n for n, _ in items} - written: set = set() + written: Dict[int, str] = {} for p in placed: facts = [n for n in p.facts if n in numbers and n not in written] if not facts or not p.sentence.strip(): continue - written.update(facts) + sentence = _clean_sentence(p.sentence, article_id) + written.update({n: sentence for n in facts}) after = p.after.strip() section_edits.setdefault(section, []).append( ProfileEdit( op="add", anchor=after if after and after != "END" else section, - text=_clean_sentence(p.sentence, article_id), + text=sentence, ) ) for n, claim in items: - sentence = "" if n in written else _clean_sentence(claim.claim, article_id) - if sentence: + sentence = written.get(n) or _clean_sentence(claim.claim, article_id) + if n not in written: section_edits.setdefault(section, []).append( ProfileEdit(op="add", anchor=section, text=sentence) ) diff --git a/tests/test_fact_list.py b/tests/test_fact_list.py index 9388bfd..5c8040f 100644 --- a/tests/test_fact_list.py +++ b/tests/test_fact_list.py @@ -493,21 +493,17 @@ def fake(*args, **kwargs): assert history == [] -def test_a_new_date_for_the_same_past_event_is_a_conflict_not_an_update(): +def test_a_rescheduled_date_stays_on_the_timeline(): claim = ArticleClaim( - claim="Nashiri was captured in the United Arab Emirates in November 2003.", - section="## Capture", - quote="captured in the United Arab Emirates in October 2002", + claim="The judge set a trial date for June 2016.", + section="## Legal Proceedings", + quote="judge ruled", ) - - def verdicts(messages): - return ClaimVerdicts( - verdicts=[ClaimVerdict(claim=1, verdict="update", row="f2")] - ) - answers = { "ArticleClaims": lambda m: ArticleClaims(claims=[claim]), - "ClaimVerdicts": verdicts, + "ClaimVerdicts": lambda m: ClaimVerdicts( + verdicts=[ClaimVerdict(claim=1, verdict="update", row="f5")] + ), "SectionPlacements": lambda m: SectionPlacements(), } generate, _ = _fake(answers) @@ -520,8 +516,25 @@ def verdicts(messages): model="m", generate=generate, ) - assert new["fact_report"]["update_to_conflict"] == 1 - assert "November 2002.^[a1] Another report differs:" in new["text"] + assert new["fact_report"]["update"] == 1 + assert "Another report differs" not in new["text"] + assert "February 2015.^[a3]" in new["text"] and "June 2016.^[a9]" in new["text"] + + +def test_placed_facts_keep_their_quote_and_source_kind(): + generate, _ = _fake(ANSWERS) + new = update_from_facts( + "people", + "Nashiri", + {"text": PROFILE}, + ARTICLE, + "a9", + model="m", + generate=generate, + ) + saved = json.loads(new[FACT_ROWS_KEY]) + placed = next(r for r in saved if "inadmissible" in r["text"]) + assert placed["sources"][0]["quote"].startswith("a military judge ruled") def test_a_same_claim_with_a_new_number_is_written_not_dropped(): From b52b25d24d104acd9775b8c239a3309093a973af Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 05:24:45 +0000 Subject: [PATCH 7/7] Keep rows that share a sentence when new evidence re-cites it in `fact_list` Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017KuZSfznoSE6bnm9t8KWMf --- src/engine/fact_list.py | 4 +++ tests/test_fact_list.py | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/engine/fact_list.py b/src/engine/fact_list.py index 9f88920..280c7e8 100644 --- a/src/engine/fact_list.py +++ b/src/engine/fact_list.py @@ -503,6 +503,10 @@ def ask(name: str, user: str, schema: Any, **values: str) -> Any: section_edits.setdefault(row["section"], []).append( ProfileEdit(op="replace", anchor=row["text"], text=cited) ) + # Rows that share this sentence follow it, or they'd be lost + for other in rows: + if other is not row and other.get("text") == row["text"]: + other["text"] = cited row["text"] = cited else: row["repeats"].append(article_id) diff --git a/tests/test_fact_list.py b/tests/test_fact_list.py index 5c8040f..7dcd24e 100644 --- a/tests/test_fact_list.py +++ b/tests/test_fact_list.py @@ -564,3 +564,57 @@ def verdicts(messages): ) assert new["fact_report"]["same_with_detail"] == 1 assert "for 4 years.^[a9]" in new["text"] + + +def test_new_evidence_keeps_rows_that_share_its_sentence(): + shared = "Nashiri was held by the CIA at secret sites in Thailand.^[a2]" + profile = PROFILE.replace("He was held by the CIA at secret sites.^[a2]", shared) + rows = [ + { + "id": "f1", + "section": "## Capture", + "claim": "Nashiri was held by the CIA at secret sites.", + "text": shared, + "sources": [{"article_id": "a2", "kind": "official"}], + "repeats": [], + }, + { + "id": "f2", + "section": "## Capture", + "claim": "Nashiri was held at secret sites in Thailand.", + "text": shared, + "sources": [{"article_id": "a2", "kind": "leak"}], + "repeats": [], + }, + ] + claim = ArticleClaim( + claim="Nashiri was held by the CIA at secret sites.", + section="## Capture", + quote="He was held by the CIA at secret sites.", + source="court filing", + ) + answers = { + "ArticleClaims": lambda m: ArticleClaims(claims=[claim]), + "ClaimVerdicts": lambda m: ClaimVerdicts( + verdicts=[ + ClaimVerdict(claim=1, verdict="same", row="f1", new_evidence=True) + ] + ), + "SectionPlacements": lambda m: SectionPlacements(), + } + generate, _ = _fake(answers) + new = update_from_facts( + "people", + "Nashiri", + {"text": profile, FACT_ROWS_KEY: json.dumps(rows)}, + ARTICLE, + "a9", + model="m", + generate=generate, + ) + saved = {r["claim"]: r for r in json.loads(new[FACT_ROWS_KEY])} + assert "Nashiri was held by the CIA at secret sites." in saved + assert "Nashiri was held at secret sites in Thailand." in saved + assert saved["Nashiri was held at secret sites in Thailand."]["sources"] == [ + {"article_id": "a2", "kind": "leak"} + ]