feat: rule-engine — security.txt probe, lang attr, page load grade

First slice of roboauto.ai parity for the free tier:

- Probe /.well-known/security.txt and emit a finding under LLM
  Accessibility. Crawlers and security researchers expect it; AI
  systems use its presence as a trust signal.
- Check <html lang="…"> — engines use it to serve localized results.
  Missing => warn with example.
- Grade homepage load time: <1s pass, <3s warn, >=3s fail. GPTBot and
  similar tend to time out around 3s.

More additions (title-length tiers, meta-description length, Twitter
Card, word count, heading hierarchy, internal-link count, and copy-
pasteable code-snippet recommendations) coming in a follow-up.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: ralyodio

lib/audit/checks/homepage.ts

@@ -26,8 +26,41 @@ export function checkHomepage(ctx: CrawlContext): Finding[] {
     priority: 5,
   });
 
+  // Page load time grading. Crawlers (including AI bots) deprioritize slow
+  // pages — anything over ~3s tends to time out for GPTBot.
+  const loadSec = home.fetchMs / 1000;
+  out.push({
+    section: "Homepage Audit",
+    check_key: "homepage.load_time",
+    status: loadSec < 1 ? "pass" : loadSec < 3 ? "warn" : "fail",
+    title: `Page load time: ${loadSec.toFixed(2)}s`,
+    detail:
+      loadSec < 1
+        ? "Fast — well within AI crawler budgets."
+        : loadSec < 3
+          ? "Acceptable — consider optimizing for faster crawl times."
+          : "Too slow — AI crawlers commonly time out around 3s.",
+    evidence: { fetchMs: home.fetchMs },
+    priority: loadSec < 1 ? 5 : loadSec < 3 ? 3 : 2,
+  });
+
   const $ = cheerio.load(home.rawHtml);
 
+  // <html lang="…"> — helps engines serve localized results and confirms
+  // content language without needing to NLP-detect it.
+  const lang = $("html").attr("lang")?.trim();
+  out.push({
+    section: "Homepage Audit",
+    check_key: "homepage.lang",
+    status: lang ? "pass" : "warn",
+    title: lang ? `<html lang="${lang}"> declared` : "Missing <html lang> attribute",
+    detail: lang
+      ? undefined
+      : 'Add a lang attribute on the root <html> tag (e.g. <html lang="en">) so AI engines serve language-appropriate results.',
+    evidence: lang ? { lang } : undefined,
+    priority: lang ? 5 : 3,
+  });
+
   // H1
   const h1s = $("h1").map((_, el) => $(el).text().trim()).get().filter(Boolean);
   if (h1s.length === 0) {

lib/audit/checks/robots.ts

@@ -200,5 +200,19 @@ export function checkRobotsAndSitemap(ctx: CrawlContext): Finding[] {
     });
   }
 
+  const sec = ctx.wellKnown.securityTxt;
+  out.push({
+    section: "LLM / AI Crawler Accessibility",
+    check_key: "security_txt",
+    status: sec && sec.status === 200 ? "pass" : "warn",
+    title: sec && sec.status === 200
+      ? "/.well-known/security.txt present"
+      : "/.well-known/security.txt missing",
+    detail: sec && sec.status === 200
+      ? "Security contact published — builds trust with crawlers and security researchers."
+      : "Publish a /.well-known/security.txt with at least a Contact: line. Crawlers and security researchers expect it; AI systems use it as a trust signal.",
+    priority: sec && sec.status === 200 ? 5 : 3,
+  });
+
   return out;
 }

lib/audit/engine.ts

@@ -95,13 +95,14 @@ export async function runAudit(
   // 1. Fetch homepage.
   const homepage = await fetchPage(target);
   // 2. Probe well-known files in parallel.
-  const [robots, sitemap, llmsTxt, llmsFullTxt, skillMd, aiPlugin] = await Promise.all([
+  const [robots, sitemap, llmsTxt, llmsFullTxt, skillMd, aiPlugin, securityTxt] = await Promise.all([
     probeText(`${origin}/robots.txt`),
     probeText(`${origin}/sitemap.xml`),
     probeText(`${origin}/llms.txt`),
     probeText(`${origin}/llms-full.txt`),
     probeText(`${origin}/skill.md`),
     probeText(`${origin}/.well-known/ai-plugin.json`),
+    probeText(`${origin}/.well-known/security.txt`),
   ]);
 
   // 3. Render homepage with Playwright (best effort).
@@ -137,6 +138,7 @@ export async function runAudit(
       llmsFullTxt: llmsFullTxt,
       skillMd: skillMd,
       aiPlugin: aiPlugin,
+      securityTxt: securityTxt,
     },
     findings: [],
   };

lib/audit/types.ts

@@ -38,6 +38,7 @@ export type CrawlContext = {
     llmsFullTxt?: { content: string; status: number };
     skillMd?: { content: string; status: number };
     aiPlugin?: { content: string; status: number };
+    securityTxt?: { content: string; status: number };
   };
   findings: Finding[];
 };