feat: rule-engine roboauto parity + premium AI-Recognition section

Free tier (rule engine):
- Title length: tier 30–60 = pass, <30 warn, >70 warn. Also detects
  error-page titles (502 Bad Gateway etc.) so a misread of a downed
  site doesn't silently produce a clean report.
- Meta description length: 50–160 = pass, otherwise warn.
- Twitter Card detection (card/title/description/image).
- Content volume / word count: 300+ = pass, 100–300 warn, <100 fail.
- Heading hierarchy: counts h1/h2/h3, warns if total <3.
- Internal vs external link count: warns if <5 internal links.
- Recommendations now include copy-pasteable code snippets for OG,
  Twitter Card, canonical, lang, security.txt, and Organization
  JSON-LD — matches roboauto's "AI Visibility Fixes" pattern.

Premium tier (LLM engines):
- New "AI Recognition" section in SECTIONS list.
- Claude system prompt instructs the model to honestly self-report
  whether the target appears in its training data BEFORE any web
  search, with a "recognition.training_data" finding. Priority 3
  (informational), not a defect.
- Markdown report template reflows: AI Recognition slots in after
  LLM Accessibility, pushing other sections up one number.

The non-Claude engines (openai/gemini/qwen/kimi/deepseek) use the
same SECTIONS list via shared prompt.ts, so the section is offered
to all paid engines — though prompt-side enforcement remains
Anthropic-only for now.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: ralyodio

lib/audit/checks/homepage.ts

@@ -94,8 +94,10 @@ export function checkHomepage(ctx: CrawlContext): Finding[] {
     });
   }
 
-  // Title
+  // Title — flag both too-short and too-long. Search/answer engines truncate
+  // around 60 chars and AI bots use short titles as a weak signal of intent.
   const title = $("title").first().text().trim();
+  const titleLooksLikeError = /\b(502|503|504|bad gateway|service unavailable|gateway timeout|not found)\b/i.test(title);
   if (!title) {
     out.push({
       section: "Homepage Audit",
@@ -105,13 +107,33 @@ export function checkHomepage(ctx: CrawlContext): Finding[] {
       detail: "The `<title>` tag is required for search and answer engines.",
       priority: 1,
     });
-  } else if (title.length < 15) {
+  } else if (titleLooksLikeError) {
+    out.push({
+      section: "Homepage Audit",
+      check_key: "homepage.title",
+      status: "fail",
+      title: "`<title>` looks like an error page",
+      detail: `Title is "${title}" — the crawl may have hit a server error or downtime. Re-scan when the site is healthy.`,
+      evidence: { title },
+      priority: 1,
+    });
+  } else if (title.length < 30) {
+    out.push({
+      section: "Homepage Audit",
+      check_key: "homepage.title",
+      status: "warn",
+      title: `Short \`<title>\` (${title.length} chars)`,
+      detail: "Aim for 30–60 chars. Lead with brand or product, then value prop.",
+      evidence: { title },
+      priority: 3,
+    });
+  } else if (title.length > 70) {
     out.push({
       section: "Homepage Audit",
       check_key: "homepage.title",
       status: "warn",
-      title: "Very short `<title>`",
-      detail: `Title is only ${title.length} chars.`,
+      title: `Long \`<title>\` (${title.length} chars)`,
+      detail: "Engines and AI snippets truncate titles around 60–70 chars. Trim to keep the key phrase visible.",
       evidence: { title },
       priority: 3,
     });
@@ -120,13 +142,14 @@ export function checkHomepage(ctx: CrawlContext): Finding[] {
       section: "Homepage Audit",
       check_key: "homepage.title",
       status: "pass",
-      title: "`<title>` present",
+      title: `\`<title>\` present (${title.length} chars)`,
       evidence: { title },
       priority: 5,
     });
   }
 
-  // Meta description
+  // Meta description — length tiers (Google truncates ~155, AI bots use it
+  // as the default snippet).
   const description = $("meta[name='description']").attr("content")?.trim();
   if (!description) {
     out.push({
@@ -137,12 +160,32 @@ export function checkHomepage(ctx: CrawlContext): Finding[] {
       detail: "Add a `<meta name=\"description\">` to control the snippet AI/SERP show.",
       priority: 2,
     });
+  } else if (description.length < 50) {
+    out.push({
+      section: "Homepage Audit",
+      check_key: "homepage.description",
+      status: "warn",
+      title: `Short meta description (${description.length} chars)`,
+      detail: "Aim for 50–160 chars. Restate your value prop in plain language.",
+      evidence: { description },
+      priority: 3,
+    });
+  } else if (description.length > 160) {
+    out.push({
+      section: "Homepage Audit",
+      check_key: "homepage.description",
+      status: "warn",
+      title: `Long meta description (${description.length} chars)`,
+      detail: "Snippets truncate around 160 chars. Tighten to keep the key sentence visible.",
+      evidence: { description },
+      priority: 3,
+    });
   } else {
     out.push({
       section: "Homepage Audit",
       check_key: "homepage.description",
       status: "pass",
-      title: "Meta description present",
+      title: `Meta description present (${description.length} chars)`,
       evidence: { description },
       priority: 5,
     });
@@ -179,6 +222,32 @@ export function checkHomepage(ctx: CrawlContext): Finding[] {
     priority: ogMissing.length === 0 ? 5 : 3,
   });
 
+  // Twitter Card — AI link previews and social agents both consume these.
+  const twitter = {
+    card: $("meta[name='twitter:card']").attr("content"),
+    title: $("meta[name='twitter:title']").attr("content"),
+    description: $("meta[name='twitter:description']").attr("content"),
+    image: $("meta[name='twitter:image']").attr("content"),
+  };
+  const twitterMissing = Object.entries(twitter).filter(([, v]) => !v).map(([k]) => k);
+  out.push({
+    section: "Homepage Audit",
+    check_key: "homepage.twitter",
+    status: twitterMissing.length === 0 ? "pass" : twitterMissing.length === 4 ? "warn" : "warn",
+    title:
+      twitterMissing.length === 0
+        ? "Twitter Card tags complete"
+        : twitterMissing.length === 4
+          ? "No Twitter Card tags"
+          : `Twitter Card: missing ${twitterMissing.join(", ")}`,
+    detail:
+      twitterMissing.length === 0
+        ? undefined
+        : "Add twitter:card, twitter:title, twitter:description, twitter:image for richer previews in social and AI agent surfaces.",
+    evidence: twitter as Record<string, unknown>,
+    priority: twitterMissing.length === 0 ? 5 : 3,
+  });
+
   // JS-rendered content check
   if (home.renderedText && home.renderedBytes) {
     const rawText = $.root().text().replace(/\s+/g, " ").trim();
@@ -232,5 +301,73 @@ export function checkHomepage(ctx: CrawlContext): Finding[] {
     });
   }
 
+  // Content volume — AI models need enough substance to summarize. Count
+  // visible text in <body>, ignoring scripts/styles/templates.
+  const $body = $("body").clone();
+  $body.find("script, style, noscript, template").remove();
+  const visibleText = $body.text().replace(/\s+/g, " ").trim();
+  const wordCount = visibleText ? visibleText.split(/\s+/).length : 0;
+  out.push({
+    section: "Homepage Audit",
+    check_key: "homepage.word_count",
+    status: wordCount >= 300 ? "pass" : wordCount >= 100 ? "warn" : "fail",
+    title: `Content volume: ${wordCount} words`,
+    detail:
+      wordCount >= 300
+        ? "Substantive content — AI models have enough to summarize and recommend."
+        : wordCount >= 100
+          ? "Thin content. Aim for 300+ words on the homepage so AI models can extract a useful description."
+          : "Very thin content. AI models need substantive text to understand and recommend your site.",
+    evidence: { wordCount },
+    priority: wordCount >= 300 ? 5 : wordCount >= 100 ? 3 : 1,
+  });
+
+  // Heading hierarchy — AI uses h1/h2/h3 to chunk content. A flat document
+  // with one heading is harder to parse than one with sectioned structure.
+  const h2Count = $("h2").length;
+  const h3Count = $("h3").length;
+  const totalHeadings = h1s.length + h2Count + h3Count;
+  out.push({
+    section: "Homepage Audit",
+    check_key: "homepage.heading_structure",
+    status: totalHeadings >= 3 ? "pass" : totalHeadings >= 1 ? "warn" : "fail",
+    title: `Heading structure: ${totalHeadings} (h1:${h1s.length}, h2:${h2Count}, h3:${h3Count})`,
+    detail:
+      totalHeadings >= 3
+        ? "Multiple headings help AI chunk and outline your page."
+        : "Few headings make it hard for AI to understand sectioning. Use h2/h3 to label each section.",
+    evidence: { h1: h1s.length, h2: h2Count, h3: h3Count },
+    priority: totalHeadings >= 3 ? 5 : 3,
+  });
+
+  // Internal link count — AI crawlers follow links to discover the rest of
+  // the site. Need at least a nav-worth.
+  const allLinks = $("a[href]").get();
+  let internalLinks = 0;
+  let externalLinks = 0;
+  for (const a of allLinks) {
+    const href = $(a).attr("href") ?? "";
+    if (!href || href.startsWith("#") || href.startsWith("mailto:") || href.startsWith("tel:")) continue;
+    try {
+      const u = new URL(href, ctx.target);
+      if (u.hostname === ctx.host) internalLinks++;
+      else externalLinks++;
+    } catch {
+      // skip invalid hrefs
+    }
+  }
+  out.push({
+    section: "Homepage Audit",
+    check_key: "homepage.internal_links",
+    status: internalLinks >= 5 ? "pass" : internalLinks >= 1 ? "warn" : "fail",
+    title: `Internal links: ${internalLinks}`,
+    detail:
+      internalLinks >= 5
+        ? `${internalLinks} internal + ${externalLinks} external links help crawlers navigate.`
+        : "Few internal links. Add a nav/footer with links to your key pages so AI crawlers can discover them.",
+    evidence: { internal: internalLinks, external: externalLinks },
+    priority: internalLinks >= 5 ? 5 : 3,
+  });
+
   return out;
 }

lib/audit/claude-engine.ts

@@ -58,6 +58,15 @@ For each finding:
 - detail: one sentence with the WHY plus the specific evidence (e.g. the actual H1 text, the actual robots.txt line)
 - priority: 1 (critical, breaks AEO entirely) to 5 (polish)
 
+For the "AI Recognition" section, BEFORE doing any web_fetch / web_search, answer honestly from your own training data:
+- Do you recognize this company / domain from training? (status: "pass" if yes with substantive recall, "warn" if vague, "fail" if not at all, "unknown" if uncertain)
+- check_key: "recognition.training_data"
+- title: "Familiar with <domain>" or "No prior knowledge of <domain>"
+- detail: One sentence. If you recognize it, name 1–2 concrete things you remember (what they do, who they serve, notable products). If you don't, say so plainly — do NOT speculate or paraphrase web search results.
+- priority: 3 (informational signal, not a defect)
+
+This section is a signal of training-data presence, not a quality judgment. Sites that don't appear in training can still be excellent — but the user should know whether they're invisible to the dominant AI surfaces.
+
 For score: weigh critical fails heavily. A site missing schema, blocking GPTBot in robots.txt, or hiding all content behind JS should score below 50. A clean, well-instrumented site should score 80+.
 
 For summary.pass/warn/fail/unknown: count each across all findings.
@@ -76,10 +85,12 @@ For markdown: produce the complete report in Markdown with these exact section h
 ## 4. Schema / Structured Data Audit
 ## 5. robots.txt and sitemap.xml Audit
 ## 6. LLM / AI Crawler Accessibility
-## 7. Positioning Clarity
-## 8. Missing or Hard-to-Find Information
-## 9. Recommended Fixes
-## 10. Priority To-Do Checklist
+## 7. AI Recognition
+(One paragraph stating whether you recognize this site from training data, and what specifically you remember if so. No fluff.)
+## 8. Positioning Clarity
+## 9. Missing or Hard-to-Find Information
+## 10. Recommended Fixes
+## 11. Priority To-Do Checklist
 
 Use ✅ / ⚠️ / ❌ / ❓ status emojis on each bullet. Quote actual content from the site. Section 10 must be reusable checkboxes ("- [ ] **P1** Add JSON-LD Organization schema").
 

lib/audit/prompt.ts

@@ -137,6 +137,7 @@ export const SECTIONS = [
   "Schema / Structured Data Audit",
   "robots.txt and sitemap.xml Audit",
   "LLM / AI Crawler Accessibility",
+  "AI Recognition",
   "Positioning Clarity",
   "Missing or Hard-to-Find Information",
   "Recommended Fixes",

lib/audit/recommendations.ts

@@ -8,19 +8,67 @@ const RECS: Record<string, { title: string; how: string }> = {
   },
   "homepage.title": {
     title: "Set a meaningful `<title>`",
-    how: "50–60 chars. Lead with the brand or product, then the value prop: 'CrawlProof — AEO audits for AI crawlers.'",
+    how:
+      "30–60 chars. Lead with the brand or product, then the value prop.\n\n" +
+      "```html\n<title>CrawlProof — AEO audits for AI crawlers</title>\n```",
   },
   "homepage.description": {
     title: "Add a meta description",
-    how: "150–160 chars. Repeat your core value prop in plain language; this often becomes the AI snippet.",
+    how:
+      "50–160 chars. Repeat your core value prop in plain language; this often becomes the AI snippet.\n\n" +
+      "```html\n<meta name=\"description\" content=\"CrawlProof shows you exactly how AI crawlers see your site, then tells you what to fix.\" />\n```",
   },
   "homepage.canonical": {
     title: "Add a canonical link",
-    how: "`<link rel=\"canonical\" href=\"https://yoursite.com/\">` on every page to prevent dup-content drift.",
+    how:
+      "Prevents dup-content drift and tells AI crawlers which URL is authoritative.\n\n" +
+      "```html\n<link rel=\"canonical\" href=\"https://yoursite.com/\" />\n```",
   },
   "homepage.og": {
     title: "Complete Open Graph tags",
-    how: "Set og:title, og:description, og:image. AI bots also use OG for fast disambiguation.",
+    how:
+      "AI bots use OG for fast disambiguation. Add all four:\n\n" +
+      "```html\n" +
+      "<meta property=\"og:title\" content=\"Your Page Title\" />\n" +
+      "<meta property=\"og:description\" content=\"50–160 char description of this page.\" />\n" +
+      "<meta property=\"og:image\" content=\"https://yoursite.com/og-image.jpg\" />\n" +
+      "<meta property=\"og:url\" content=\"https://yoursite.com/\" />\n" +
+      "<meta property=\"og:type\" content=\"website\" />\n" +
+      "<meta property=\"og:site_name\" content=\"YourSite\" />\n" +
+      "```",
+  },
+  "homepage.twitter": {
+    title: "Add Twitter Card meta tags",
+    how:
+      "Used by social platforms and AI agents for richer previews.\n\n" +
+      "```html\n" +
+      "<meta name=\"twitter:card\" content=\"summary_large_image\" />\n" +
+      "<meta name=\"twitter:title\" content=\"Your Page Title\" />\n" +
+      "<meta name=\"twitter:description\" content=\"50–160 char description.\" />\n" +
+      "<meta name=\"twitter:image\" content=\"https://yoursite.com/og-image.jpg\" />\n" +
+      "```",
+  },
+  "homepage.lang": {
+    title: "Declare a language on the <html> tag",
+    how:
+      "Helps AI engines serve localized results and confirms content language.\n\n" +
+      "```html\n<html lang=\"en\">\n```",
+  },
+  "homepage.load_time": {
+    title: "Speed up homepage rendering",
+    how: "AI crawlers commonly time out around 3s. Cache the HTML, ship less JS for the first paint, and pre-render the hero section server-side.",
+  },
+  "homepage.word_count": {
+    title: "Add more substantive homepage content",
+    how: "AI models need 300+ words of visible body text to summarize and recommend a site. Add a value-prop paragraph, a short FAQ, and a 'how it works' section.",
+  },
+  "homepage.heading_structure": {
+    title: "Add structured headings",
+    how: "Use h2 for each section and h3 for sub-points. AI uses these to outline and chunk the page.",
+  },
+  "homepage.internal_links": {
+    title: "Add internal navigation links",
+    how: "Top nav + footer with links to /pricing, /docs, /about, /contact gives AI crawlers an entry point to the rest of the site.",
   },
   "homepage.js_rendered": {
     title: "Server-render critical homepage content",
@@ -78,6 +126,37 @@ const RECS: Record<string, { title: string; how: string }> = {
     title: "Add /skill.md",
     how: "Describe what an agent can do with your site (e.g., 'Search docs', 'Look up pricing'). Useful for agentic flows.",
   },
+  "security_txt": {
+    title: "Publish /.well-known/security.txt",
+    how:
+      "A security contact builds trust with crawlers and researchers. Minimal example:\n\n" +
+      "```\n" +
+      "Contact: mailto:security@yourdomain.com\n" +
+      "Expires: 2027-01-01T00:00:00.000Z\n" +
+      "Preferred-Languages: en\n" +
+      "```",
+  },
+  "schema.json_ld_organization": {
+    title: "Add Schema.org Organization JSON-LD",
+    how:
+      "Helps AI engines resolve your brand entity:\n\n" +
+      "```html\n" +
+      "<script type=\"application/ld+json\">\n" +
+      "{\n" +
+      "  \"@context\": \"https://schema.org\",\n" +
+      "  \"@type\": \"Organization\",\n" +
+      "  \"name\": \"Your Company\",\n" +
+      "  \"url\": \"https://yoursite.com\",\n" +
+      "  \"logo\": \"https://yoursite.com/logo.png\",\n" +
+      "  \"description\": \"What you do, in one sentence.\",\n" +
+      "  \"sameAs\": [\n" +
+      "    \"https://twitter.com/yourhandle\",\n" +
+      "    \"https://linkedin.com/company/yourcompany\"\n" +
+      "  ]\n" +
+      "}\n" +
+      "</script>\n" +
+      "```",
+  },
   "positioning.who": {
     title: "Make your About/Team page reachable",
     how: "Add a top-nav or footer link to /about or /team so LLMs can identify the entity behind the site.",