Repository navigation
Final Enterprise Verification #7
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
60fd5da
2c90fe7
e897481
3d25faf
8022dc0
dc0c091
58b310e
6f66604
ebd00af
fdbca31
8fb417b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,10 @@ name: CI + LLM Review | |
| on: | ||
| pull_request: | ||
| types: [opened, synchronize, reopened, ready_for_review] | ||
| permissions: | ||
| pull-requests: write | ||
| contents: read | ||
| checks: write | ||
| jobs: | ||
| llm-review: | ||
| runs-on: ubuntu-latest | ||
|
|
@@ -26,7 +30,8 @@ jobs: | |
| env: | ||
| LLM_API_URL: ${{ secrets.LLM_API_URL }} | ||
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| LLM_EMAIL: ${{ secrets.LLM_EMAIL }} | ||
| GITHUB_TOKEN: ${{ github.token }} | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Duplicated
|
||
| GITHUB_REPOSITORY: ${{ github.repository }} | ||
| PR_NUMBER: ${{ github.event.pull_request.number }} | ||
| run: | | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,3 +43,5 @@ function getInternalStatus() { | |
| } | ||
|
|
||
| module.exports = { processData, startDaemon }; | ||
|
|
||
| // Final verification trigger | ||
| Original file line number | Diff line number | Diff line change | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1 @@ | ||||||||||||||
| async function test() { items.forEach(async i => { await save(i); }); } // Ignored Promise Bug | ||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The
|
||||||||||||||
| async function test() { items.forEach(async i => { await save(i); }); } // Ignored Promise Bug | |
| async function test() { | |
| for (const i of items) { | |
| await save(i); | |
| } | |
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,25 +1,74 @@ | ||
| // ESM-compatible imports supported via dynamic import() inside the function | ||
| /** | ||
| * Annotates a git patch with (Line: X) markers for the AI. | ||
| */ | ||
| function annotatePatch(patch) { | ||
| const lines = patch.split('\n'); | ||
| let annotated = []; | ||
| let currentNewLine = 0; | ||
|
|
||
| for (const line of lines) { | ||
| if (line.startsWith('@@')) { | ||
| // Parse hunk header: @@ -oldStart,oldCount +newStart,newCount @@ | ||
| const match = line.match(/\+(\d+)/); | ||
| if (match) currentNewLine = parseInt(match[1], 10); | ||
| annotated.push(line); | ||
| } else if (line.startsWith('+')) { | ||
| annotated.push(`(Line: ${currentNewLine}) ${line}`); | ||
| currentNewLine++; | ||
| } else if (line.startsWith('-')) { | ||
| annotated.push(line); // Deletions don't contribute to NEW line numbers in suggestions | ||
| } else if (line.startsWith(' ')) { | ||
| annotated.push(`(Line: ${currentNewLine}) ${line}`); | ||
| currentNewLine++; | ||
| } else { | ||
| annotated.push(line); | ||
| } | ||
| } | ||
| return annotated.join('\n'); | ||
| } | ||
|
|
||
| async function performReview({ patch, geminiKey, octokit, repo, prNumber, severity = 'Medium' }) { | ||
| // Dynamic import for node-fetch (ESM only package) | ||
| const { default: fetch } = await import('node-fetch'); | ||
| console.log(`[DEBUG] Starting performReview for ${repo} #${prNumber}`); | ||
|
|
||
| // Dynamic imports | ||
| let fetch; | ||
| try { | ||
| const fetchMod = await import('node-fetch'); | ||
| fetch = fetchMod.default || fetchMod; | ||
| } catch (e) { | ||
| throw new Error(`Failed to load node-fetch: ${e.message}`); | ||
| } | ||
|
|
||
| if (!octokit) { | ||
| try { | ||
| const { Octokit } = await import('@octokit/rest'); | ||
| octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }); | ||
| } catch (e) { | ||
| throw new Error(`Failed to load Octokit: ${e.message}`); | ||
| } | ||
| } | ||
|
|
||
| const [owner, repoName] = repo.split('/'); | ||
|
|
||
| // Extract file names and clean the patch (remove binary diff noise) | ||
| // Extract file names and clean the patch | ||
| const filesInDiff = [...patch.matchAll(/^diff --git a\/(.*) b\/(.*)$/gm)].map(m => m[1]); | ||
| const fileList = filesInDiff.join(', '); | ||
| const cleanedPatch = patch.replace(/Binary files [\s\S]*?differ\n/g, ''); | ||
| const annotatedPatch = annotatePatch(patch.replace(/Binary files [\s\S]*?differ\n/g, '')); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The
|
||
|
|
||
| const prompt = ` | ||
| You are a senior code reviewer at a top tech company. | ||
| Files in this PR: ${fileList} | ||
|
|
||
| CRITICAL REASONING GOALS: | ||
| 1. CROSS-FILE: Analyze relationships between these files (e.g., a vulnerability in file A demonstrated by an exploit in file B). | ||
| 2. TEST QUALITY: Look for ignored Futures, background threads, infinite loops, swallowed exceptions, @SuppressWarnings abuse, or flaky patterns. | ||
| 3. STYLE CONSISTENCY: Ensure changes match patterns used in adjacent code and prefer existing project helpers over new ones. | ||
| 1. CROSS-FILE: Analyze architectural relationships. | ||
| 2. TEST QUALITY: Detect ignored Futures, threading bugs, and swallowed exceptions. | ||
| 3. STYLE: Match project patterns. | ||
|
|
||
| CONTEXT: | ||
| The patch below has been annotated with (Line: X) markers. When reporting an issue, you MUST use the provided Line number for 'start_line' and 'end_line'. 'end_line' must be the line where the issue exists in the NEW version of the file. | ||
|
|
||
| You must output JSON matching this schema: | ||
| You must output JSON: | ||
| { | ||
| "summary": string, | ||
| "findings": [ | ||
|
|
@@ -30,50 +79,38 @@ You must output JSON matching this schema: | |
| "issue": string, | ||
| "severity": "INFO"|"LOW"|"MEDIUM"|"HIGH"|"CRITICAL", | ||
| "confidence": float (0-1), | ||
| "impact": string (what could go wrong), | ||
| "why": string (reasoning behind this finding), | ||
| "impact": string, | ||
| "why": string, | ||
| "suggestion": string, | ||
| "suggested_code": string (optional, a literal code replacement for the lines from start_line to end_line) | ||
| "suggested_code": string | ||
| } | ||
| ] | ||
| } | ||
|
|
||
| Return an empty findings list if nothing to report. | ||
| Return ONLY JSON. Return empty findings list if nothing to report. | ||
|
|
||
| Patch below: | ||
| ----- | ||
| ${cleanedPatch} | ||
| ${annotatedPatch} | ||
| ----- | ||
|
|
||
| CRITICAL RULES: | ||
| 1. 'file' MUST be exactly one of [${fileList}]. | ||
| 2. Line numbers MUST exist within the specified diff hunks (1-indexed based on the new file content). | ||
| 3. 'suggested_code' MUST be the EXACT replacement for the line range. ONLY provide the code that replaces lines ${'${start_line}'} through ${'${end_line}'}. ZERO prose. | ||
| 4. Provide ONLY JSON. | ||
| `; | ||
|
|
||
| const baseUrl = process.env.GEMINI_BASE_URL || 'https://generativelanguage.googleapis.com/v1beta'; | ||
| const model = process.env.GEMINI_MODEL || 'gemini-2.0-flash'; | ||
| const apiUrl = `${baseUrl}/models/${model}:generateContent`; | ||
|
|
||
| console.log(`[DEBUG] Calling Gemini API...`); | ||
| const res = await fetch(`${apiUrl}?key=${geminiKey}`, { | ||
| method: "POST", | ||
| headers: { | ||
| "Content-Type": "application/json", | ||
| "x-goog-api-key": geminiKey, | ||
| }, | ||
| headers: { "Content-Type": "application/json", "x-goog-api-key": geminiKey }, | ||
| body: JSON.stringify({ | ||
| contents: [{ parts: [{ text: prompt }] }], | ||
| generationConfig: { | ||
| temperature: 0, | ||
| response_mime_type: "application/json" | ||
| } | ||
| generationConfig: { temperature: 0, response_mime_type: "application/json" } | ||
| }), | ||
| }); | ||
|
|
||
| if (!res.ok) { | ||
| const errorBody = await res.text(); | ||
| throw new Error(`Gemini API failed with status ${res.status}: ${errorBody}`); | ||
| throw new Error(`Gemini API failed (${res.status}): ${errorBody}`); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Inconsistent error message format for Gemini API failure.Severity: LOW | Confidence: 70% Why: The error message format in this line differs from the format used in other error messages in the file. Advice: Standardize the error message format. |
||
| } | ||
|
|
||
| const resJson = await res.json(); | ||
|
|
@@ -86,48 +123,47 @@ CRITICAL RULES: | |
| } catch (e) { | ||
| const jsonMatch = llmRaw.match(/```json\n([\s\S]*?)\n```/) || llmRaw.match(/\{[\s\S]*\}/); | ||
| if (jsonMatch) { | ||
| try { | ||
| result = JSON.parse(jsonMatch[jsonMatch.length - 1].trim()); | ||
| } catch (innerError) { | ||
| console.error("Gemini Parsing Failed:", llmRaw.substring(0, 500)); | ||
| throw new Error(`Failed to parse extracted JSON block: ${innerError.message}`); | ||
| } | ||
| result = JSON.parse(jsonMatch[jsonMatch.length - 1].trim()); | ||
| } else { | ||
| throw new Error(`No JSON found in Gemini response: ${e.message}`); | ||
| throw new Error(`No JSON found in Gemini response`); | ||
| } | ||
| } | ||
|
|
||
| // Post-AI Validation: filter out hallucinations | ||
| result.findings = (result.findings || []).filter(f => filesInDiff.includes(f.file)); | ||
| console.log(`[DEBUG] Gemini returned ${result.findings.length} findings`); | ||
|
|
||
| // 1. ALWAYS post the Summary first (General Comment) | ||
| // This ensures visibility even if the detailed review/annotations fail. | ||
| if (result.summary) { | ||
| try { | ||
| console.log("[DEBUG] Posting summary comment..."); | ||
| await octokit.rest.issues.createComment({ | ||
| owner, | ||
| repo: repoName, | ||
| issue_number: Number(prNumber), | ||
| body: `## **Gemini AI Review Summary**\n\n${result.summary}` | ||
| }); | ||
| } catch (e) { | ||
| console.warn(`[WARN] Failed to post summary comment: ${e.message}`); | ||
| } | ||
| } | ||
|
|
||
| // 2. Post detailed findings as a PR Review if possible | ||
| if (result.findings && result.findings.length > 0) { | ||
| const annotations = []; | ||
| const reviewComments = []; | ||
| const seenFindings = new Set(); | ||
|
|
||
| for (const f of result.findings) { | ||
| // Deduplication: hash by file + issue type + line | ||
| const findingHash = `${f.file}:${f.issue}:${f.end_line}`; | ||
| if (seenFindings.has(findingHash)) continue; | ||
| seenFindings.add(findingHash); | ||
|
|
||
| const isHighSignal = (f.severity === 'HIGH' || f.severity === 'CRITICAL') && (f.confidence >= 0.75); | ||
|
|
||
| annotations.push({ | ||
| path: f.file, | ||
| start_line: f.start_line, | ||
| end_line: f.end_line, | ||
| annotation_level: f.severity === 'CRITICAL' || f.severity === 'HIGH' ? 'failure' : (f.severity === 'MEDIUM' ? 'warning' : 'notice'), | ||
| message: `${f.issue}\n\nWhy: ${f.why}\nImpact: ${f.impact}`, | ||
| title: `Gemini: ${f.issue}`, | ||
| raw_details: f.suggestion | ||
| }); | ||
|
|
||
| let body = `### **${f.issue}**\n\n**Severity**: ${f.severity} | **Confidence**: ${Math.round(f.confidence * 100)}%\n\n**Why**: ${f.why}`; | ||
|
|
||
| if (isHighSignal && f.suggested_code) { | ||
| const codeMatch = f.suggested_code.match(/```(?:javascript|js)?\n?([\s\S]*?)```/) || [null, f.suggested_code]; | ||
| const pureCode = codeMatch[1].trim(); | ||
| const pureCode = f.suggested_code.replace(/```(?:javascript|js)?\n?([\s\S]*?)```/g, '$1').trim(); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Inconsistent code extraction from LLM response.Severity: LOW | Confidence: 60% Why: The code extraction logic uses a regex replace instead of the previous match. This could lead to unexpected behavior if the LLM response format changes. Advice: Use the previous match logic for consistency. |
||
| body += `\n\n**Suggested Fix**:\n\`\`\`suggestion\n${pureCode}\n\`\`\``; | ||
| } else if (f.suggestion) { | ||
| body += `\n\n**Advice**: ${f.suggestion}`; | ||
|
|
@@ -142,52 +178,20 @@ CRITICAL RULES: | |
| } | ||
|
|
||
| try { | ||
| // 1. Post to Checks API (Formal Annotations) - Optional | ||
| try { | ||
| const headShaRes = await octokit.rest.pulls.get({ owner, repo: repoName, pull_number: Number(prNumber) }); | ||
| const headSha = headShaRes.data.head.sha; | ||
|
|
||
| await octokit.rest.checks.create({ | ||
| owner, | ||
| repo: repoName, | ||
| name: "Gemini AI Analysis", | ||
| head_sha: headSha, | ||
| status: "completed", | ||
| conclusion: result.findings.some(f => f.severity === 'CRITICAL') ? "failure" : "neutral", | ||
| output: { | ||
| title: "Gemini AI Analysis", | ||
| summary: result.summary || "AI analysis completed.", | ||
| annotations: annotations.slice(0, 50) | ||
| } | ||
| }); | ||
| } catch (e) { | ||
| console.warn(`Checks API skipped: ${e.message}`); | ||
| } | ||
|
|
||
| // 2. Post as a Unified Review with Suggestions | ||
| console.log(`[DEBUG] Attempting to post detailed review (${reviewComments.length} comments)...`); | ||
| await octokit.rest.pulls.createReview({ | ||
| owner, | ||
| repo: repoName, | ||
| pull_number: Number(prNumber), | ||
| event: "COMMENT", | ||
| body: `## **Gemini AI Review Summary**\n\n${result.summary || "Analysis completed."}\n\n*Review consolidated to reduce noise.*`, | ||
| comments: reviewComments.slice(0, 50) | ||
| }); | ||
| console.log("Successfully posted consolidated review."); | ||
| } catch (e) { | ||
| console.warn(`Failed to post PR Review: ${e.message}`); | ||
| // Non-fatal, just log it | ||
| } | ||
| } else if (result.summary) { | ||
| try { | ||
| await octokit.rest.issues.createComment({ | ||
| owner, | ||
| repo: repoName, | ||
| issue_number: Number(prNumber), | ||
| body: `## **Gemini AI Review Summary**\n\n${result.summary}` | ||
| body: `*Detailed AI findings follow below.*`, | ||
| comments: reviewComments.slice(0, 30) // Limit to 30 to stay within API safety | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hardcoded limit of 30 comments for detailed review.Severity: MEDIUM | Confidence: 70% Why: The code limits the number of comments in the detailed review to 30, which might not be sufficient for larger PRs with many findings. Advice: Make the comment limit configurable or implement a more dynamic approach to prioritize comments. |
||
| }); | ||
| console.log("[DEBUG] Successfully posted consolidated review."); | ||
| } catch (e) { | ||
| console.warn(`Failed to post summary: ${e.message}`); | ||
| console.warn(`[WARN] Failed to post detailed Review (likely invalid line numbers): ${e.message}`); | ||
| // If the unified review fails, try posting high-priority items as individual comments? | ||
| // For now, the summary is already posted, so the user has the main value. | ||
| } | ||
| } | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Adding
email-service.jswithout clear context. Is this a new dependency or part of existing functionality?Severity: INFO | Confidence: 70%
Why: The purpose and integration of
email-service.jsare not immediately obvious from the surrounding code. It's important to understand how it fits into the overall architecture.Advice: Add a comment explaining the purpose and usage of
email-service.js.