Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions packages/providers/src/site/extract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,39 @@ export function networkForUrl(url: string): Network | undefined {
}

/** `https://github.com/octocat/repo` → `octocat`. */
/**
* Path roots that carry the handle in the *next* segment rather than this one.
*/
const NESTED_ROOTS = /^(company|school|showcase|in|pub|c|channel|user|r|profile)$/i;

/**
* Path roots that are content, search or site furniture — never an account.
*
* Found by crawling stripe.com: a footer link to a YouTube video produced the
* "handle" `watch`, which would have been resolved as if it were somebody's
* account. A junk identity is worse than a missing one, because the resolver
* treats it as a claim worth weighing.
*/
const NOT_HANDLES =
/^(watch|playlist|embed|shorts|results|feed|hashtag|search|explore|about|legal|terms|privacy|help|login|signup|share|intent|home|posts|status|p|reel|stories|tv|jobs|pricing|blog|docs|features|events|groups|topics|orgs|sponsors|marketplace|pulse|learning|today|new|trending)$/i;

function handleFromUrl(url: string): string | undefined {
try {
const segments = new URL(url).pathname.split('/').filter(Boolean);
const parsed = new URL(url);
const segments = parsed.pathname.split('/').filter(Boolean);
const first = segments[0];
if (!first) return undefined;
// Company pages and static sections are not handles.
if (/^(company|school|showcase|in|pub|c|channel|user|r)$/i.test(first)) return segments[1];
return first;

const candidate = NESTED_ROOTS.test(first) ? segments[1] : first;
if (!candidate) return undefined;
if (NOT_HANDLES.test(candidate)) return undefined;

// A handle is one path segment of plausible characters. Anything carrying
// a dot-extension or punctuation beyond the usual is a file or a page.
if (!/^[@\w][\w.-]{0,63}$/.test(candidate)) return undefined;
if (/\.(html?|php|aspx?|jsp)$/i.test(candidate)) return undefined;

return candidate;
} catch {
return undefined;
}
Expand Down
19 changes: 19 additions & 0 deletions packages/providers/src/site/site.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,25 @@ describe('deterministic extraction', () => {
expect(company.identities[0]!.handle).toBe('loopwright');
});

test('a link to content is not mistaken for an account', () => {
// Found by crawling stripe.com: a footer link to a YouTube video yielded
// the "handle" `watch`, which the resolver would then weigh as a real
// claim. A junk identity is worse than a missing one.
const company = extractCompany(
`<a href="https://www.youtube.com/watch?v=abc123">video</a>
<a href="https://www.youtube.com/@stripe">channel</a>
<a href="https://twitter.com/search?q=stripe">search</a>
<a href="https://github.com/stripe">code</a>`,
'https://stripe.com/',
);

const handles = company.identities.map((i) => i.handle);
expect(handles).not.toContain('watch');
expect(handles).not.toContain('search');
expect(handles).toContain('@stripe');
expect(handles).toContain('stripe');
});

test('a page with no structured data names nobody', () => {
const result = extractSite('<html><body><p>We are a team.</p></body></html>', 'https://a.io/');
expect(result.people).toHaveLength(0);
Expand Down
Loading