diff --git a/graphify/extractors/sql.py b/graphify/extractors/sql.py index 602f89b0b4..75e2f8aafb 100644 --- a/graphify/extractors/sql.py +++ b/graphify/extractors/sql.py @@ -32,11 +32,17 @@ # 'SELECT AUTOCREATE PROCEDURE x FROM t;' in an error-bearing file minted a # phantom routine x() (delimited identifiers are span-skipped at the scan # site, but a bare word has no span). +# A backtick-quoted name is also accepted: _debracket_tsql rewrites every +# T-SQL bracket-quoted identifier in the source to a backtick-quoted one +# before parsing, so by the time an ERROR-bearing file reaches this scan a +# bracket-named routine has no brackets left to match — only a backtick +# name. Without this alternative a bracketed routine name went from a +# recovered (if ugly) node to no node at all. _ROUTINE_RECOVERY_RX = re.compile( r"\bCREATE\s+(?:OR\s+(?:REPLACE|ALTER)\s+)?(?:FUNCTION|PROC(?:EDURE)?)\s+" r"(?:IF\s+NOT\s+EXISTS\s+)?" - r"((?:\"(?:[^\"\n]|\"\")+\"|\[(?:[^\]\n]|\]\])+\]|[\w$]+)" - r"(?:\s*\.\s*(?:\"(?:[^\"\n]|\"\")+\"|\[(?:[^\]\n]|\]\])+\]|[\w$]+))*)", + r"((?:\"(?:[^\"\n]|\"\")+\"|`(?:[^`\n]|``)+`|\[(?:[^\]\n]|\]\])+\]|[\w$]+)" + r"(?:\s*\.\s*(?:\"(?:[^\"\n]|\"\")+\"|`(?:[^`\n]|``)+`|\[(?:[^\]\n]|\]\])+\]|[\w$]+))*)", re.IGNORECASE, ) @@ -81,9 +87,11 @@ def _scan_sql(text: str) -> tuple[str, list[tuple[int, int]]]: One output character per input character: non-newline characters inside a blanked span become spaces and newlines are kept, so positions and line numbers computed against the masked text are valid against the - original. Double-quoted and bracket-delimited identifiers are preserved - verbatim (they carry recoverable routine names); single-quoted strings, - line comments, and (nesting-aware) block comments are blanked. Used by + original. Double-quoted, bracket-delimited, and backtick-delimited + identifiers are preserved verbatim (they carry recoverable routine + names — a backtick-quoted one is what _debracket_tsql rewrote a T-SQL + bracket-quoted name into before parsing); single-quoted strings, line + comments, and (nesting-aware) block comments are blanked. Used by the routine-recovery scan so CREATE PROCEDURE/FUNCTION DDL reachable only through a comment or a single-quoted string cannot fabricate a routine node when an unrelated parse error arms recovery. @@ -173,7 +181,7 @@ def _blank_tail_and_carry(start: int) -> int: break j += 1 i = _blank(j) - elif c == '"' or c == "[": + elif c == '"' or c == "[" or c == "`": # Delimited identifier: preserve verbatim. Doubled closers are # escapes. A span is DISTRUSTED when it is unterminated (no # closer before the newline) or would swallow a comment opener @@ -202,7 +210,7 @@ def _blank_tail_and_carry(start: int) -> int: # same /, an irreducible divergence whose only closure would be # blanking to EOF on every */* sequence (accepted, documented # limitation; the token sequence appears in no dialect's idiom). - closer = '"' if c == '"' else "]" + closer = '"' if c == '"' else ("]" if c == "[" else "`") j = i + 1 closed = False while j < n and text[j] != "\n": @@ -269,6 +277,236 @@ def _norm_ident(name: str) -> str: return ".".join(parts) +# PostgreSQL dollar-quoted string/body delimiter: $$ or $tag$, tag optional. +_DOLLAR_QUOTE_TAG_RX = re.compile(rb"\$([A-Za-z_][A-Za-z0-9_]*)?\$") + + +def _debracket_tsql(source: bytes) -> tuple[bytes, list[tuple[int, int]]]: + """Rewrite T-SQL bracket quoted identifiers to backtick quoted ones. + + tree_sitter_sql has no grammar rule for a bracket delimited identifier + ([dbo].[Orders]): every one lands in an ERROR node, and the surrounding + statement can misparse or drop entirely, mangling labels (#2712) and + silently losing a whole table behind a broken foreign key (#2713). + Backtick quoting (MySQL's dialect) is a token the grammar already + recognizes, so rewriting the source before parsing lets the normal AST + path handle these statements instead of falling into error recovery. + + Only a bracket span that reads as an identifier is rewritten. A trailing + [] pair is also used as an array type or array literal marker in other + dialects: int[], numeric(10)[3], ARRAY[1,2,3]. Those are told apart from + a quoted identifier by what comes right before the bracket. A quoted + identifier starts a name, so it is preceded by whitespace, `.`, `,`, `(`, + or the start of the file; an array marker is preceded by the identifier + or keyword it subscripts (ARRAY[, col[, the closing `)` of a type's + precision/scale list), with no separating punctuation -- except the + ARRAY keyword itself, which PostgreSQL also allows any whitespace + (including a newline) before its bracket (ARRAY [1, 2, 3], or + ARRAY\n[1, 2, 3]); that specific case is checked for past a run of + whitespace so it is not mistaken for the start of a quoted name. + Content that is empty or purely numeric is also excluded, since neither + is a legal bare T-SQL identifier, and content holding a comment opener + (`--`, `/*`) is + distrusted the same way _scan_sql treats it: a genuine identifier never + contains one, so a bracket that appears to swallow one is more likely an + unrelated stray `[` racing ahead to some later statement's real closing + `]`. Rewriting it anyway would erase the comment before _scan_sql's own + line-scoped distrust handling ever sees it. + + Content holding a literal `.` or a literal backtick is excluded too, + but for a different reason: tree_sitter_sql's own grammar cannot + correctly parse either one inside a backtick span. A dot splits the + token regardless of quoting, so [My.Table] would still come out mangled + after rewriting to `My.Table` (the grammar reads it as `My`, a bare `.` + qualifier, `Table`, and a dangling stray backtick). A literal backtick + is escaped by doubling it when this function writes a rewritten span + ([Foo`Bar] -> `Foo``Bar`, matching MySQL's own escaping convention), + but the grammar treats the first backtick of that doubled pair as the + closing delimiter instead of an escape, truncating the identifier to + `Foo` and leaving `Bar` behind as an unrelated ERROR node. No amount of + escaping on this end can round-trip either shape; leaving the span as a + plain, un-rewritten bracket is the fallback that existed before this + function did. + + A single or double quoted string is scanned to its real closing quote + even across embedded newlines, matching how the engines actually read + one: stopping at the first line break would leave the remainder of a + still-open multi-line string scanned as ordinary source, so bracket-like + text inside it (dynamic SQL split across lines, say) would be mistaken + for a real identifier and rewritten, corrupting the string's content. + A string that in fact never closes just consumes the rest of the file + this way, which only means nothing past it gets debracketed -- the same + fallback behavior as before this function existed. + + A PostgreSQL dollar-quoted span ($$ ... $$ or $tag$ ... $tag$, as used + for a PL/pgSQL function body) is skipped the same way: its content is + opaque body text, not SQL to debracket, and bracket-like text inside it + (an array subscript expression, say) must not be rewritten just because + it happens to sit between a stray pair of square brackets. + + A genuinely backtick-quoted MySQL identifier is skipped for the same + reason: without a dedicated branch its content was scanned character by + character like ordinary source, so a bracket-like substring inside one + (a name like [weird]name written between backticks) was mistaken for a + real T-SQL bracket identifier and rewritten right there inside the + existing backtick span, producing doubled and orphaned backticks. A + doubled backtick inside the span is honored as an escaped literal + backtick, the same escaping convention this function itself uses when + it writes a rewritten span, so the scan does not stop early on one. + + Returns the rewritten source plus the byte-range spans, in the rewritten + source's own coordinates, of every backtick pair this function inserted. + A downstream reader uses those spans to un-rewrite ONLY the identifiers + that came from a bracket: a genuinely backtick-quoted MySQL name sitting + anywhere else in the same file must not be touched just because the file + also happened to need debracketing somewhere (#2721 follow-up). + """ + out = bytearray() + i, n = 0, len(source) + spans: list[tuple[int, int]] = [] + while i < n: + c = source[i] + if c == ord("'"): + j = i + 1 + while j < n: + if source[j] == ord("'"): + if j + 1 < n and source[j + 1] == ord("'"): + j += 2 + continue + j += 1 + break + j += 1 + out += source[i:j] + i = j + elif c == ord('"'): + j = i + 1 + while j < n: + if source[j] == ord('"'): + if j + 1 < n and source[j + 1] == ord('"'): + j += 2 + continue + j += 1 + break + j += 1 + out += source[i:j] + i = j + elif c == ord("`"): + j = i + 1 + while j < n: + if source[j] == ord("`"): + if j + 1 < n and source[j + 1] == ord("`"): + j += 2 + continue + j += 1 + break + j += 1 + out += source[i:j] + i = j + elif c == ord("$") and (m := _DOLLAR_QUOTE_TAG_RX.match(source, i)): + tag = m.group(0) + close = source.find(tag, m.end()) + j = close + len(tag) if close != -1 else n + out += source[i:j] + i = j + elif c == ord("-") and i + 1 < n and source[i + 1] == ord("-"): + j = i + while j < n and source[j] != ord("\n"): + j += 1 + out += source[i:j] + i = j + elif c == ord("/") and i + 1 < n and source[i + 1] == ord("*"): + depth, j = 1, i + 2 + while j < n and depth: + if source[j] == ord("/") and j + 1 < n and source[j + 1] == ord("*"): + depth += 1 + j += 2 + elif source[j] == ord("*") and j + 1 < n and source[j + 1] == ord("/"): + depth -= 1 + j += 2 + else: + j += 1 + out += source[i:j] + i = j + elif c == ord("["): + prev = source[i - 1] if i > 0 else None + subscript_like = prev is not None and ( + chr(prev).isalnum() or chr(prev) in "_$)]" + ) + if not subscript_like and prev in (0x20, 0x09, 0x0A, 0x0D): + # PostgreSQL allows whitespace -- including a newline, since + # SQL treats all whitespace between tokens the same way -- + # between the ARRAY keyword and its bracket constructor + # (ARRAY [1, 2, 3], or ARRAY\n[1, 2, 3]), so a plain + # "preceding char" check misses it: the char right before + # '[' is whitespace, not the identifier/keyword the marker + # actually subscripts. Look back past the run of whitespace + # for a bare ARRAY word instead. + k = i - 1 + while k > 0 and source[k - 1] in (0x20, 0x09, 0x0A, 0x0D): + k -= 1 + word_start = k - 5 + if word_start >= 0 and source[word_start:k].upper() == b"ARRAY" and ( + word_start == 0 + or not ( + chr(source[word_start - 1]).isalnum() + or source[word_start - 1] in (0x5F, 0x24) + ) + ): + subscript_like = True + j = i + 1 + closed = False + while j < n and source[j] != ord("\n"): + if source[j] == ord("]"): + if j + 1 < n and source[j + 1] == ord("]"): + j += 2 + continue + j += 1 + closed = True + break + j += 1 + content = source[i + 1:j - 1] if closed else b"" + looks_like_ident = ( + closed + and content + and not subscript_like + and not content.strip().isdigit() + and b"--" not in content + and b"/*" not in content + and b"." not in content + and b"`" not in content + ) + if looks_like_ident: + escaped = content.replace(b"]]", b"]").replace(b"`", b"``") + span_start = len(out) + out += b"`" + escaped + b"`" + spans.append((span_start, len(out))) + i = j + else: + out.append(c) + i += 1 + else: + out.append(c) + i += 1 + return bytes(out), spans + + +def _strip_backtick_parts(name: str) -> str: + """Undo _debracket_tsql's rewrite for display labels and recovered names. + + The caller is responsible for only invoking this on a name it has + confirmed overlaps a span _debracket_tsql actually rewrote (see + _clean_name) -- called unconditionally, this would also strip a + genuinely backtick-quoted MySQL name sitting anywhere else in the file. + """ + parts = [] + for part in name.split("."): + p = part.strip() + if len(p) >= 2 and p[0] == "`" and p[-1] == "`": + p = p[1:-1].replace("``", "`") + parts.append(p) + return ".".join(parts) + + def extract_sql(path: Path, content: str | bytes | None = None) -> dict: """Extract tables, views, functions, and relationships from .sql files via tree-sitter.""" try: @@ -295,6 +533,7 @@ def extract_sql(path: Path, content: str | bytes | None = None) -> dict: else content if content is not None else path.read_bytes() ) + source, debracket_spans = _debracket_tsql(source) tree = parser.parse(source) root = tree.root_node except Exception as e: @@ -313,10 +552,27 @@ def extract_sql(path: Path, content: str | bytes | None = None) -> dict: def _read(n) -> str: return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") + def _overlaps_debracketed_span(start_byte: int, end_byte: int) -> bool: + return any(s < end_byte and start_byte < e for s, e in debracket_spans) + + def _clean_name(n) -> str: + """Read n's text, un-rewriting it ONLY if _debracket_tsql touched it. + + A node whose byte range never overlaps a rewritten span is returned + exactly as read -- in particular, a genuinely backtick-quoted MySQL + name elsewhere in a file that also needed T-SQL debracketing is left + with its own backticks intact, not stripped just because the file as + a whole went through the rewrite (#2721 follow-up). + """ + text = _read(n) + if not debracket_spans or not _overlaps_debracketed_span(n.start_byte, n.end_byte): + return text + return _strip_backtick_parts(text) + def _obj_name(n) -> str | None: for c in n.children: if c.type == "object_reference": - return _read(c) + return _clean_name(c) return None def _add_node(nid: str, label: str, line: int) -> None: @@ -380,7 +636,7 @@ def walk(node) -> None: if cc.type == "keyword_references": found_ref = True elif found_ref and cc.type == "object_reference": - ref_name = _read(cc) + ref_name = _clean_name(cc) break if ref_name: ref_nid = table_nids.get(_norm_ident(ref_name)) or _ref_stub(ref_name) @@ -397,7 +653,7 @@ def walk(node) -> None: if cc.type == "keyword_references": found_ref = True elif found_ref and cc.type == "object_reference": - ref_name = _read(cc) + ref_name = _clean_name(cc) break if ref_name: ref_nid = table_nids.get(_norm_ident(ref_name)) or _ref_stub(ref_name) @@ -458,7 +714,7 @@ def walk(node) -> None: if ccc.type == "keyword_references": found_ref = True elif found_ref and ccc.type == "object_reference": - ref_name = _read(ccc) + ref_name = _clean_name(ccc) break if ref_name: ref_nid = (table_nids.get(_norm_ident(ref_name)) @@ -474,11 +730,11 @@ def walk(node) -> None: if c.type == "keyword_trigger": after_trigger = True elif after_trigger and not trig_name and c.type == "object_reference": - trig_name = _read(c) + trig_name = _clean_name(c) elif c.type == "keyword_for": after_for = True elif after_for and not tbl_name and c.type == "object_reference": - tbl_name = _read(c) + tbl_name = _clean_name(c) if trig_name: trig_nid = _make_id(stem, trig_name) _add_node(trig_nid, trig_name, line) @@ -583,7 +839,7 @@ def _walk_from_refs(node, caller_nid: str, line: int, if c.type == "relation": for cc in c.children: if cc.type == "object_reference": - tbl = _read(cc) + tbl = _clean_name(cc) if _norm_ident(tbl) in cte_names: continue tbl_nid = table_nids.get(_norm_ident(tbl)) or _ref_stub(tbl) @@ -635,7 +891,44 @@ def _collect_defined_names(node) -> None: # (e.g. Firebird COMPUTED BY columns push constraints out of the tree entirely). # Snapshot after tree walk so we don't re-emit edges already captured above. emitted = {(e["source"], e["target"]) for e in edges if e["relation"] == "references"} - src_text = source.decode("utf-8", errors="replace") + # surrogateescape, not replace: _clean_regex_name reconstructs a byte + # offset by re-encoding a src_text slice, which only works if decoding + # was lossless. errors="replace" collapses each invalid byte to a + # single U+FFFD that re-encodes to 3 bytes, permanently inflating every + # offset computed past it (confirmed: 200 invalid bytes shifted a + # routine name's computed span 400 bytes off, past every real + # debracket_spans entry, so a debracketed name that should have had + # its backticks stripped kept them in the final label instead). + # surrogateescape maps each invalid byte to its own lone surrogate + # codepoint and back losslessly, so encode(decode(source)) == source + # exactly and every offset stays correct. + src_text = source.decode("utf-8", errors="surrogateescape") + + def _clean_regex_name(text: str, char_start: int, char_end: int) -> str: + """_clean_name's counterpart for a name captured by a regex match + against src_text (a decoded str) rather than read off a tree-sitter + node -- the whole-file ERROR-recovery fallback has no node to ask + for a byte range, so this converts the match's character offsets to + byte offsets (src_text decodes the same, already-debracketed + `source` debracket_spans was computed against) before doing the same + overlap check _clean_name does. + + A delimited name's content can itself carry one of src_text's + surrogate-escaped bytes (a quoted identifier's negated character + class does not exclude it the way a bare [\\w$]+ name already does). + Sanitized on the way out, past the point the surrogate was needed + for correct offset math, so a name never carries one into a label — + encoding a lone surrogate is a hard error everywhere except this + one special mode, and a label is not meant to be decoded back with + it. + """ + if not debracket_spans: + return text.encode("utf-8", errors="replace").decode("utf-8") + byte_start = len(src_text[:char_start].encode("utf-8", errors="surrogateescape")) + byte_end = len(src_text[:char_end].encode("utf-8", errors="surrogateescape")) + if not _overlaps_debracketed_span(byte_start, byte_end): + return text.encode("utf-8", errors="replace").decode("utf-8") + return _strip_backtick_parts(text).encode("utf-8", errors="replace").decode("utf-8") for m in re.finditer(r"CREATE\s+TABLE\s+([\w$]+)\s*\(", src_text, re.IGNORECASE): tbl_name = m.group(1) tbl_nid = table_nids.get(_norm_ident(tbl_name)) @@ -687,7 +980,7 @@ def _collect_defined_names(node) -> None: for m in _ROUTINE_RECOVERY_RX.finditer(masked_src): if any(s <= m.start() < e for s, e in ident_spans): continue - fn_name = m.group(1) + fn_name = _clean_regex_name(m.group(1), m.start(1), m.end(1)) fn_line = src_text[: m.start()].count("\n") + 1 _add_node(_make_id(stem, fn_name), f"{fn_name}()", fn_line) diff --git a/tests/test_multilang.py b/tests/test_multilang.py index 8701f2bff6..f25f09af9b 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -528,7 +528,7 @@ def test_sql_tsql_bracketed_procedure_is_recovered(tmp_path): ) r = extract_sql(p) routine = [n["label"] for n in r["nodes"] if n["label"] != "proc.sql"] - assert routine == ["[dbo].[usp_LoadDebtors]()"], routine + assert routine == ["dbo.usp_LoadDebtors()"], routine def test_sql_tsql_create_or_alter_procedure_is_recovered(tmp_path): @@ -542,7 +542,7 @@ def test_sql_tsql_create_or_alter_procedure_is_recovered(tmp_path): ) r = extract_sql(p) labels = sorted(n["label"] for n in r["nodes"] if n["label"] != "proc.sql") - assert labels == ["[Utils].[ValidateSourceView]()", "usp_Bare()"], labels + assert labels == ["Utils.ValidateSourceView()", "usp_Bare()"], labels def test_sql_escaped_closing_bracket_in_routine_name_is_consumed(tmp_path): @@ -555,7 +555,7 @@ def test_sql_escaped_closing_bracket_in_routine_name_is_consumed(tmp_path): p.write_text("CREATE PROCEDURE [dbo].[a]]b]\nAS\nBEGIN\n SELECT 1;\nEND;\n") r = extract_sql(p) routine = [n["label"] for n in r["nodes"] if n["label"] != "proc.sql"] - assert routine == ["[dbo].[a]]b]()"], routine + assert routine == ["dbo.a]b()"], routine def test_sql_recovery_sites_agree_on_the_captured_name(tmp_path): @@ -573,7 +573,7 @@ def test_sql_recovery_sites_agree_on_the_captured_name(tmp_path): p.write_text("CREATE PROCEDURE dbo.[usp_Mixed] AS\nBEGIN\n SELECT 1;\nEND;\n") r = extract_sql(p) routine = [n["label"] for n in r["nodes"] if n["label"] != "proc.sql"] - assert routine == ["dbo.[usp_Mixed]()"], routine + assert routine == ["dbo.usp_Mixed()"], routine def test_sql_tsql_proc_shorthand_is_recovered(tmp_path): @@ -588,7 +588,7 @@ def test_sql_tsql_proc_shorthand_is_recovered(tmp_path): ) r = extract_sql(p) labels = sorted(n["label"] for n in r["nodes"] if n["label"] != "proc.sql") - assert labels == ["[dbo].[usp_Short]()", "usp_BareShort()"], labels + assert labels == ["dbo.usp_Short()", "usp_BareShort()"], labels def test_sql_commented_ddl_is_not_fabricated_by_error_recovery(tmp_path): @@ -611,7 +611,7 @@ def test_sql_commented_ddl_is_not_fabricated_by_error_recovery(tmp_path): ) r = extract_sql(p) labels = [n["label"] for n in r["nodes"] if n["label"] != "broken.sql"] - assert "[dbo].[usp_Real]()" in labels, labels + assert "dbo.usp_Real()" in labels, labels assert not any("Comment" in l for l in labels), ( f"commented-out DDL fabricated a node: {labels}" ) @@ -644,8 +644,8 @@ def test_sql_comment_openers_inside_string_literals_do_not_hide_ddl(tmp_path): ) r = extract_sql(p) labels = [n["label"] for n in r["nodes"] if n["label"] != "strings.sql"] - assert "[dbo].[usp_AfterLineString]()" in labels, labels - assert "[dbo].[usp_AfterBlockString]()" in labels, labels + assert "dbo.usp_AfterLineString()" in labels, labels + assert "dbo.usp_AfterBlockString()" in labels, labels def test_mask_sql_comments_literal_and_comment_handling(): @@ -965,7 +965,7 @@ def test_sql_dynamic_sql_and_unclosed_comment_do_not_fabricate_routines(tmp_path ) r = extract_sql(p) labels = [n["label"] for n in r["nodes"] if n["label"] != "dynamic.sql"] - assert "[dbo].[usp_Real]()" in labels, labels + assert "dbo.usp_Real()" in labels, labels fabricated = [ label for label in labels if any(k in label for k in ("Dynamic", "Nested", "Bracketed", "Unterminated")) @@ -992,7 +992,7 @@ def test_sql_ddl_keywords_inside_delimited_identifiers_do_not_fabricate(tmp_path ) r = extract_sql(p) labels = [n["label"] for n in r["nodes"] if n["label"] != "alias.sql"] - assert labels == ["[dbo].[usp_Real]()"], labels + assert labels == ["dbo.usp_Real()"], labels def test_sql_create_inside_a_bare_word_does_not_fabricate(tmp_path): @@ -1011,7 +1011,7 @@ def test_sql_create_inside_a_bare_word_does_not_fabricate(tmp_path): ) r = extract_sql(p) labels = [n["label"] for n in r["nodes"] if n["label"] != "bare.sql"] - assert labels == ["[dbo].[usp_Real]()"], labels + assert labels == ["dbo.usp_Real()"], labels def test_sql_escaped_double_quote_in_routine_name_is_consumed(tmp_path): @@ -1259,6 +1259,304 @@ def test_sql_schema_qualified_alter_fk(): assert e["source"] in node_ids, f"dangling source: {e['source']}" assert e["target"] in node_ids, f"dangling target: {e['target']}" +def test_sql_tsql_bracket_identifiers_produce_clean_labels(tmp_path): + """#2712: [dbo].[Alpha] must label as dbo.Alpha, not the delimiter-mangled + `dbo].[Alpha` tree-sitter-sql's grammar produces for bracket quoting (it has + no token for `[...]`, so each bracket lands as its own one-byte ERROR node, + one character short of the real pair).""" + pytest.importorskip("tree_sitter_sql") + p = tmp_path / "schema.sql" + p.write_text( + "CREATE TABLE [dbo].[Alpha] (\n" + " [Id] INT NOT NULL PRIMARY KEY\n" + ");\n" + "GO\n" + "CREATE TABLE [dbo].[Beta] (\n" + " [Id] INT NOT NULL PRIMARY KEY\n" + ");\n" + "GO\n" + ) + r = extract_sql(p) + labels = [n["label"] for n in r["nodes"]] + assert "dbo.Alpha" in labels, f"got {labels}" + assert "dbo.Beta" in labels, f"got {labels}" + assert not any("]" in l or "[" in l for l in labels), ( + f"a bracket fragment leaked into a label: {labels}" + ) + +def test_sql_tsql_bracket_reference_resolves_by_clean_name(tmp_path): + """#2712: a bracket-quoted FOREIGN KEY ... REFERENCES [dbo].[Alpha] must + resolve onto the real Alpha table node, not dangle or mint a stub keyed by + the mangled `Alpha` text.""" + pytest.importorskip("tree_sitter_sql") + p = tmp_path / "schema.sql" + p.write_text( + "CREATE TABLE [dbo].[Alpha] ([Id] INT NOT NULL PRIMARY KEY);\n" + "GO\n" + "CREATE TABLE [dbo].[Beta] (\n" + " [Id] INT NOT NULL PRIMARY KEY,\n" + " [AlphaId] INT NOT NULL,\n" + " CONSTRAINT [FK_Beta_Alpha] FOREIGN KEY ([AlphaId])\n" + " REFERENCES [dbo].[Alpha] ([Id])\n" + ");\n" + "GO\n" + ) + r = extract_sql(p) + nid = {n["label"]: n["id"] for n in r["nodes"]} + assert "dbo.Alpha" in nid and "dbo.Beta" in nid + refs = {(e["source"], e["target"]) for e in r["edges"] if e["relation"] == "references"} + assert (nid["dbo.Beta"], nid["dbo.Alpha"]) in refs, f"got {refs}" + +def test_sql_tsql_bracket_debracketing_does_not_corrupt_array_types(tmp_path): + """#2712 follow-up: the bracket->backtick rewrite must not misfire on + Postgres/MySQL array-type syntax (`text[]`, `numeric(10)[3]`), which uses + `[...]` for something other than a T-SQL quoted identifier.""" + pytest.importorskip("tree_sitter_sql") + p = tmp_path / "schema.sql" + p.write_text("CREATE TABLE t (id INT, tags text[], scores numeric(10,2)[3]);\n") + r = extract_sql(p) + labels = [n["label"] for n in r["nodes"]] + assert any(l == "t" for l in labels), f"table extraction broke on array types: {labels}" + for l in labels: + assert "`" not in l, f"a synthetic backtick leaked into a label: {labels}" + +def test_sql_tsql_bracket_debracketing_does_not_corrupt_array_literal(tmp_path): + """#2712 follow-up: an ARRAY[...] literal must not be misread as a + bracket-quoted identifier just because its content is not purely numeric.""" + pytest.importorskip("tree_sitter_sql") + p = tmp_path / "schema.sql" + p.write_text( + "CREATE TABLE t (id INT, tags INT[] DEFAULT ARRAY[1,2,3]);\n" + ) + r = extract_sql(p) + labels = [n["label"] for n in r["nodes"]] + assert any(l == "t" for l in labels), f"table extraction broke on ARRAY literal: {labels}" + for l in labels: + assert "`" not in l, f"a synthetic backtick leaked into a label: {labels}" + +def test_sql_tsql_bracket_does_not_strip_unrelated_mysql_backticks(tmp_path): + """#2721 follow-up: a genuinely backtick-quoted MySQL name elsewhere in the + same file must keep its backticks even though the file also needed T-SQL + debracketing for an unrelated statement -- the un-rewrite must only touch + the specific spans _debracket_tsql actually rewrote, not every + backtick-quoted name in a file that happened to need debracketing at all.""" + pytest.importorskip("tree_sitter_sql") + p = tmp_path / "mixed.sql" + p.write_text( + "CREATE TABLE [dbo].[Orders] (Id INT);\n" + "CREATE TABLE `mysql_table` (Id INT);\n" + ) + r = extract_sql(p) + labels = [n["label"] for n in r["nodes"]] + assert "dbo.Orders" in labels, f"got {labels}" + assert "`mysql_table`" in labels, ( + f"genuine MySQL backtick name stripped just because the file also had a " + f"T-SQL bracket span elsewhere: {labels}" + ) + +def test_sql_tsql_bracket_with_dot_is_left_unrewritten(tmp_path): + """tree_sitter_sql's grammar splits on a literal `.` even inside a + backtick span, so a bracket name like [My.Table] cannot be represented + by rewriting it to a backtick pair -- the grammar would still mangle it + into `My` + a bare `.` + `Table` + a stray dangling backtick. Debracketing + must refuse to rewrite such a span and leave it as plain bracket text + instead of producing a corrupted label.""" + pytest.importorskip("tree_sitter_sql") + p = tmp_path / "dotted.sql" + p.write_text("CREATE TABLE [My.Table] (Id INT);\n") + r = extract_sql(p) + labels = [n["label"] for n in r["nodes"]] + assert "My.Table" in labels, f"got {labels}" + for l in labels: + assert "`" not in l, f"a synthetic backtick leaked into a label: {labels}" + +def test_sql_tsql_debracket_does_not_corrupt_multiline_string(tmp_path): + """_debracket_tsql used to stop scanning a single quoted string at the + first newline regardless of whether it was actually closed there, so + bracket-like text on a later line of a still-open multi-line string + (dynamic SQL split across lines, say) was mistaken for a real bracket + identifier and rewritten to backtick form, corrupting the string's + content. The scan must follow the string to its real closing quote.""" + from graphify.extractors.sql import _debracket_tsql + + src = b"SET @sql = 'first line\n[NotAnIdentifier] second line';\n" + new_src, spans = _debracket_tsql(src) + assert new_src == src, f"string content was rewritten: {new_src!r}" + assert not spans + +def test_sql_tsql_debracket_does_not_corrupt_dollar_quoted_body(tmp_path): + """A PostgreSQL dollar-quoted PL/pgSQL body ($$ ... $$ or $tag$ ... $tag$) + is opaque function body text, not SQL to debracket. _debracket_tsql had + no handling for it at all, so bracket-like text anywhere inside a + dollar-quoted body was mistaken for a real bracket identifier and + rewritten to backtick form, corrupting the body's content.""" + from graphify.extractors.sql import _debracket_tsql + + src = ( + b"CREATE FUNCTION foo() RETURNS void AS $$\n" + b"BEGIN\n" + b" SELECT [NotReallyBracketIdent];\n" + b"END;\n" + b"$$ LANGUAGE plpgsql;\n" + ) + new_src, spans = _debracket_tsql(src) + assert new_src == src, f"dollar-quoted body was rewritten: {new_src!r}" + assert not spans + + tagged = ( + b"CREATE FUNCTION foo() RETURNS void AS $body$\n" + b" SELECT [NotAnIdentifier];\n" + b"$body$ LANGUAGE plpgsql;\n" + ) + new_tagged, tagged_spans = _debracket_tsql(tagged) + assert new_tagged == tagged, f"tagged dollar-quoted body was rewritten: {new_tagged!r}" + assert not tagged_spans + +def test_sql_tsql_debracket_does_not_corrupt_backtick_identifier(tmp_path): + """A genuine MySQL backtick-quoted identifier that happens to contain + bracket-like text (`[weird]name`) had no dedicated scan branch, so its + content was walked character by character like ordinary source: the + embedded [weird] read as a real T-SQL bracket identifier and got + rewritten right there inside the existing backtick span, producing + doubled and orphaned backticks. A doubled backtick inside the span + (an escaped literal backtick) must not be mistaken for the close + either.""" + from graphify.extractors.sql import _debracket_tsql + + src = b"CREATE TABLE `[weird]name` (Id INT);\n" + new_src, spans = _debracket_tsql(src) + assert new_src == src, f"backtick identifier was rewritten: {new_src!r}" + assert not spans + + escaped = b"CREATE TABLE `a``b` (Id INT);\n" + new_escaped, escaped_spans = _debracket_tsql(escaped) + assert new_escaped == escaped, f"escaped backtick corrupted: {new_escaped!r}" + assert not escaped_spans + +def test_sql_tsql_debracket_leaves_bracket_content_with_backtick_alone(tmp_path): + """A T-SQL bracket identifier containing a literal backtick ([Foo`Bar], + a legal T-SQL name since only ] needs escaping inside a bracket) cannot + be represented by rewriting to backtick form: the grammar treats the + first backtick of the doubled escape this function would write + (`Foo``Bar`) as the closing delimiter rather than an escape, truncating + the identifier to Foo and leaving `Bar` as a stray ERROR node. Must be + left as a plain, un-rewritten bracket instead.""" + from graphify.extractors.sql import _debracket_tsql + + src = b"CREATE TABLE [Foo`Bar] (Id INT);\n" + new_src, spans = _debracket_tsql(src) + assert new_src == src, f"bracket with backtick content was rewritten: {new_src!r}" + assert not spans + +def test_sql_tsql_debracket_does_not_rewrite_array_constructor_with_space(tmp_path): + """PostgreSQL allows whitespace between the ARRAY keyword and its + bracket constructor (ARRAY [1, 2, 3]). The preceding-character check + that tells an array marker apart from a bracket-quoted identifier only + looked at the single character right before '[', which is a space in + this shape rather than the keyword -- so the array literal was + mistaken for an identifier and rewritten to `1, 2, 3`.""" + from graphify.extractors.sql import _debracket_tsql + + src = b"SELECT ARRAY [1, 2, 3];\n" + new_src, spans = _debracket_tsql(src) + assert new_src == src, f"array constructor was rewritten: {new_src!r}" + assert not spans + + # A bracket identifier preceded by ordinary whitespace (not ARRAY) must + # still be rewritten -- this is the common, unaffected case. + ident = b"CREATE TABLE [Foo] (Id INT);\n" + new_ident, ident_spans = _debracket_tsql(ident) + assert new_ident == b"CREATE TABLE `Foo` (Id INT);\n", new_ident + assert ident_spans + +def test_sql_tsql_debracket_does_not_rewrite_array_constructor_after_newline(tmp_path): + """Same ARRAY-keyword gap as the space case above, but with a newline + (or a mix of whitespace kinds) between ARRAY and its bracket -- SQL + treats all whitespace between tokens the same way, so ARRAY\\n[1, 2, 3] + is just as valid as ARRAY [1, 2, 3], and the lookback must skip a + newline too, not just spaces and tabs.""" + from graphify.extractors.sql import _debracket_tsql + + src = b"SELECT ARRAY\n[1, 2, 3];\n" + new_src, spans = _debracket_tsql(src) + assert new_src == src, f"array constructor was rewritten: {new_src!r}" + assert not spans + + mixed = b"SELECT ARRAY \t\n [1, 2, 3];\n" + new_mixed, mixed_spans = _debracket_tsql(mixed) + assert new_mixed == mixed, f"array constructor was rewritten: {new_mixed!r}" + assert not mixed_spans + +def test_sql_regex_recovery_survives_invalid_utf8_bytes_earlier_in_file(tmp_path): + """_clean_regex_name reconstructs a byte offset by re-encoding a + src_text slice, which only works if the original decode was lossless. + src_text used to decode with errors="replace", which collapses every + invalid byte to one U+FFFD that re-encodes to 3 bytes -- permanently + inflating every offset computed past an invalid byte anywhere earlier + in the file. A routine recovered only through the regex fallback (its + AS BEGIN...END body has no grammar rule at all, so it can never reach + _clean_name's tree-sitter-node path) then had its overlap check miss + every real debracket span, so a name debracketing DID touch kept its + backticks in the final label instead of having them stripped.""" + pytest.importorskip("tree_sitter_sql") + p = tmp_path / "schema.sql" + invalid_run = b"\xff" * 200 + p.write_bytes( + b"-- padding comment with invalid bytes " + invalid_run + b"\n" + b"CREATE TABLE [dbo].[Customer] (Id INT NOT NULL PRIMARY KEY);\n" + b"GO\n" + b"CREATE PROCEDURE [dbo].[GetCustomer] @Id INT\n" + b"AS\n" + b"BEGIN\n" + b" SELECT Id FROM dbo.Customer WHERE Id = @Id;\n" + b"END\n" + b"GO\n" + ) + r = extract_sql(p) + labels = [n["label"] for n in r["nodes"]] + assert "dbo.GetCustomer()" in labels, f"got {labels}" + assert not any("`" in l for l in labels), f"backtick leaked into a label: {labels}" + +def test_sql_tsql_bracketed_fk_does_not_drop_child_table_or_fabricate_self_loop(tmp_path): + """#2713: a bracket-quoted FOREIGN KEY ... REFERENCES clause used to confuse + the parser badly enough that the whole child table (Invoice) — FK + constraint included — landed as bogus nested content inside the PARENT + table's (Customer) own subtree. That dropped Invoice from the graph + entirely and fabricated a Customer -> Customer self-referencing edge + tagged EXTRACTED (highest confidence) where no such reference exists in + the source. Fixed as a side effect of #2712's debracketing: with clean + identifier tokens the two CREATE TABLE statements parse as separate + top-level statements again.""" + pytest.importorskip("tree_sitter_sql") + p = tmp_path / "s.sql" + p.write_text( + "CREATE TABLE [dbo].[Customer] (\n" + " [CustomerId] INT NOT NULL PRIMARY KEY,\n" + " [Name] NVARCHAR(100) NULL\n" + ");\n" + "GO\n" + "\n" + "CREATE TABLE [dbo].[Invoice] (\n" + " [InvoiceId] INT NOT NULL PRIMARY KEY,\n" + " [CustomerId] INT NOT NULL,\n" + " CONSTRAINT [FK_Invoice_Customer] FOREIGN KEY ([CustomerId])\n" + " REFERENCES [dbo].[Customer] ([CustomerId])\n" + ");\n" + "GO\n" + ) + r = extract_sql(p) + nid = {n["label"]: n["id"] for n in r["nodes"]} + assert "dbo.Customer" in nid, "Customer table missing from the graph" + assert "dbo.Invoice" in nid, "Invoice table was dropped from the graph (#2713)" + + refs = {(e["source"], e["target"]) for e in r["edges"] if e["relation"] == "references"} + assert (nid["dbo.Customer"], nid["dbo.Customer"]) not in refs, ( + "fabricated Customer -> Customer self-loop present (#2713)" + ) + assert (nid["dbo.Invoice"], nid["dbo.Customer"]) in refs, ( + f"expected Invoice -> Customer reference edge, got {refs}" + ) + def test_sql_plpgsql_functions_survive_parse_errors(): """PL/pgSQL bodies make tree-sitter-sql emit ERROR nodes; the functions must still be extracted (#1910), without cascading into later statements."""