Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions packages/ai/src/checks.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,32 @@ describe('claim extraction', () => {
test('ignores common words even when capitalised', () => {
expect(extractClaims('the thing is Thanks matters')).not.toContain('Thanks');
});

test('does not read a first-person contraction as a name', () => {
// "I'd" is capitalised mid-sentence but asserts nothing. Extracting it
// withheld every first-person draft, because normalising the apostrophe
// leaves "i d" and no evidence can contain that.
expect(extractClaims("fees are brutal, and I'd guess settlement is worse")).toEqual([]);
expect(extractClaims('fees are brutal, and I’d guess settlement is worse')).toEqual([]);
expect(extractClaims("we shipped it and I've seen the same thing since")).toEqual([]);
});

test('reads a possessive as the name it possesses', () => {
// "Stripe's" must match evidence that says "Stripe", not read as invented.
expect(extractClaims('the Stripe’s fees are brutal')).toContain('Stripe');
expect(extractClaims("the Stripe's fees are brutal")).toContain('Stripe');
});

test('does not treat apostrophes as quotation marks', () => {
// Two contractions used to bracket the prose between them into a
// "quotation" that nothing could support.
const claims = extractClaims("I've seen this. Curious what you'd try next.");
expect(claims).toEqual([]);
});

test('still picks out a genuinely quoted phrase', () => {
expect(extractClaims('they said “fees are brutal” last week')).toContain('fees are brutal');
});
});

describe('flattery (PRD §13.3)', () => {
Expand Down Expand Up @@ -304,4 +330,40 @@ describe('findUnsupportedClaims', () => {

expect(unsupported).toHaveLength(0);
});

test('accepts a first-person draft that only cites the evidence', () => {
// The shape the composer actually produces. Every one of these was
// rejected as an unsupported claim, so the approvals queue answered
// "the wording kept asserting things nothing supports — I'd".
const { unsupported } = findUnsupportedClaims(
"You said fees are brutal and settlement takes days. I'd guess the delay is the " +
"batching rather than Stripe's rails, but I've been wrong about that before.",
GROUNDING,
);

expect(unsupported).toHaveLength(0);
});

test('a first-person draft passes every gate', () => {
const report = runChecks(
input({
body:
"You said settlement takes days and fees are brutal. I'd guess the batching " +
"is the harder half — I've seen that pattern before.",
}),
);

expect(report.passed).toBe(true);
expect(report.unsupported).toHaveLength(0);
});

test('still catches an invented specific inside first-person prose', () => {
const { unsupported } = findUnsupportedClaims(
"I'd say Fluxwire cut our settlement time by 40%.",
GROUNDING,
);

expect(unsupported).toContain('Fluxwire');
expect(unsupported).toContain('40%');
});
});
26 changes: 23 additions & 3 deletions packages/ai/src/checks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,20 @@ const STOPWORDS = new Set([
'way',
]);

/**
* Contraction and possessive endings, removed before a capitalised word is
* read as a name.
*
* "I'd" is not a proper noun, but it is capitalised mid-sentence and so was
* extracted as one — and nothing could ever support it, because `normalize`
* turns the apostrophe into a space and leaves the untraceable token "i d".
* Any draft written in the first person failed its own grounding gate, twice,
* and was withheld. Stripping the ending also fixes the opposite error:
* "Stripe's" normalised to "stripe s" and did not match stored evidence
* reading "Stripe", so a correctly grounded name was rejected as invented.
*/
const CONTRACTION_SUFFIX = /['’](?:s|d|m|t|ll|ve|re)$/i;

/**
* Distinct content words a draft must share with the evidence to count as
* grounded. Two is enough to rule out generic outreach ("would you be open to
Expand Down Expand Up @@ -377,8 +391,13 @@ export function findUnsupportedClaims(
export function extractClaims(body: string): string[] {
const claims: string[] = [];

// Quoted phrases: an explicit claim about what someone said.
for (const match of body.matchAll(/[""']([^""']{4,120})[""']/g)) {
// Quoted phrases: an explicit claim about what someone said. Only double
// quotes delimit one. An apostrophe is not a quote mark, and treating it as
// one made every pair of contractions in a draft look like a quotation: "I've
// seen this. Curious what you'd try" yielded the claim "ve seen this. Curious
// what you", which no evidence can support. Single quotes are left out for
// the same reason — they are indistinguishable from the apostrophe in "I'd".
for (const match of body.matchAll(/["“”]([^"“”]{4,120})["“”]/g)) {
if (match[1]) claims.push(match[1]);
}

Expand All @@ -396,7 +415,8 @@ export function extractClaims(body: string): string[] {
for (const sentence of sentences) {
const words = sentence.trim().split(/\s+/);
for (let i = 1; i < words.length; i += 1) {
const word = words[i]!.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '');
const trimmed = words[i]!.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, '');
const word = trimmed.replace(CONTRACTION_SUFFIX, '');
if (word.length < 3) continue;
if (!/^\p{Lu}/u.test(word)) continue;
if (STOPWORDS.has(word.toLowerCase())) continue;
Expand Down
Loading