From d72d4c1c35b6dddec7311f75296d1b139d1e80f3 Mon Sep 17 00:00:00 2001 From: Steve Dignam Date: Sun, 30 Aug 2026 23:18:36 -0400 Subject: [PATCH] fmt: avoid quoting things that don't need quoting --- crates/squawk_fmt/Cargo.toml | 2 +- crates/squawk_fmt/src/fmt.rs | 37 +++++--- crates/squawk_fmt/src/lib.rs | 1 + crates/squawk_fmt/src/main.rs | 18 +++- crates/squawk_fmt/src/token_compare.rs | 90 ++++++++++++++++++ crates/squawk_fmt/tests/after/copy.snap | 2 + .../tests/after/create_table_options.snap | 2 + .../after/create_text_search_parser.snap | 6 +- .../squawk_fmt/tests/after/select_expr.snap | 8 ++ crates/squawk_fmt/tests/after/types.snap | 2 +- crates/squawk_fmt/tests/after/update.snap | 2 +- .../squawk_fmt/tests/after/xml_functions.snap | 8 +- crates/squawk_fmt/tests/before/copy.sql | 2 + .../tests/before/create_table_options.sql | 2 + .../squawk_fmt/tests/before/select_expr.sql | 2 + crates/squawk_fmt/tests/before/update.sql | 2 +- .../squawk_fmt/tests/before/xml_functions.sql | 8 +- crates/squawk_fmt/tests/tests.rs | 94 +++---------------- 18 files changed, 179 insertions(+), 109 deletions(-) create mode 100644 crates/squawk_fmt/src/token_compare.rs diff --git a/crates/squawk_fmt/Cargo.toml b/crates/squawk_fmt/Cargo.toml index 179092ab..c0479aaa 100644 --- a/crates/squawk_fmt/Cargo.toml +++ b/crates/squawk_fmt/Cargo.toml @@ -22,6 +22,7 @@ tiny_pretty.workspace = true itertools.workspace = true squawk-syntax.workspace = true squawk-line-index.workspace = true +squawk-lexer.workspace = true rowan.workspace = true clap.workspace = true anyhow.workspace = true @@ -32,7 +33,6 @@ annotate-snippets.workspace = true insta.workspace = true dir-test.workspace = true camino.workspace = true -squawk-lexer.workspace = true [lints] workspace = true diff --git a/crates/squawk_fmt/src/fmt.rs b/crates/squawk_fmt/src/fmt.rs index 23a7345e..8c4fe003 100644 --- a/crates/squawk_fmt/src/fmt.rs +++ b/crates/squawk_fmt/src/fmt.rs @@ -1331,7 +1331,7 @@ fn build_accessor<'a>(accessor: ast::Accessor) -> Doc<'a> { } else if let Some(name) = field.composite_field_ref() { doc = doc .append(leading_comments(name.syntax())) - .append(build_name(name.syntax())); + .append(build_column_label(name.syntax())); } doc } @@ -3225,7 +3225,7 @@ fn build_copy_option_list<'a>(list: ast::CopyOptionList) -> Doc<'a> { fn build_copy_option<'a>(option: ast::CopyOption) -> Doc<'a> { let mut doc = option .copy_option_key() - .map(|key| build_keyword_node(key.syntax())) + .map(|key| build_column_label(key.syntax())) .unwrap_or_else(Doc::nil); if let Some(value) = option.copy_option_value() { doc = doc @@ -8727,18 +8727,31 @@ fn build_path_parts<'a>( .append(build_path_ref(&qualifier)) .append(trailing_comments(qualifier.syntax())); } - if dot.is_some() { + let is_qualified = dot.is_some(); + if is_qualified { doc = doc.append(Doc::text(".")); } if let Some(segment) = segment { doc = doc .append(leading_comments(segment.syntax())) - .append(build_name(segment.syntax())); + .append(if is_qualified { + build_column_label(segment.syntax()) + } else { + build_name(segment.syntax()) + }); } doc } fn build_name<'a>(node: &SyntaxNode) -> Doc<'a> { + build_name_with(node, quote_ident) +} + +fn build_column_label<'a>(node: &SyntaxNode) -> Doc<'a> { + build_name_with(node, quote_column_alias) +} + +fn build_name_with<'a>(node: &SyntaxNode, quote: fn(&str) -> String) -> Doc<'a> { let mut tokens = node .children_with_tokens() .filter_map(|el| el.into_token()) @@ -8763,7 +8776,7 @@ fn build_name<'a>(node: &SyntaxNode) -> Doc<'a> { return doc; } - Doc::text(quote_ident(&normalize_name_node(node))) + Doc::text(quote(&normalize_name_node(node))) } fn is_unicode_escape(text: &str) -> bool { @@ -9553,7 +9566,7 @@ fn build_attribute_list_with_layout<'a>(list: &ast::AttributeList, multiline: bo let items = list.attribute_options().map(|option| { let mut item = option .namespace() - .map(|namespace| build_name(namespace.syntax())) + .map(|namespace| build_column_label(namespace.syntax())) .unwrap_or_else(Doc::nil); if let Some(dot) = option.dot_token() { item = item.append(comments_before(dot)).append(Doc::text(".")); @@ -9564,7 +9577,7 @@ fn build_attribute_list_with_layout<'a>(list: &ast::AttributeList, multiline: bo } else if name.join_token().is_some() { Doc::text("join") } else { - build_name(name.syntax()) + build_column_label(name.syntax()) }; item = item .append(leading_comments(name.syntax())) @@ -19385,7 +19398,7 @@ fn build_field_expr<'a>(field_expr: ast::FieldExpr) -> Doc<'a> { } else if let Some(field) = field_expr.field() { doc = doc .append(leading_comments(field.syntax())) - .append(build_name(field.syntax())); + .append(build_column_label(field.syntax())); } doc @@ -19992,7 +20005,7 @@ fn build_xml_element_fn<'a>(xml_element_fn: ast::XmlElementFn) -> Doc<'a> { body = body .append(Doc::space()) .append(leading_comments(tag.syntax())) - .append(build_name(tag.syntax())); + .append(build_column_label(tag.syntax())); let mut items = Vec::new(); if let Some(attrs) = xml_element_fn.expr_as_xml_attr_list() { @@ -20049,7 +20062,7 @@ fn build_expr_as_xml_attr_list<'a>(attrs: ast::ExprAsXmlAttrList) -> Doc<'a> { item = item .append(Doc::space()) .append(leading_comments(name.syntax())) - .append(build_name(name.syntax())); + .append(build_column_label(name.syntax())); } ( leading_comments(attr.syntax()).append(item), @@ -20146,7 +20159,7 @@ fn build_expr_as_element_tag_list<'a>(list: ast::ExprAsElementTagList) -> Doc<'a item_doc = item_doc .append(Doc::space()) .append(leading_comments(tag.syntax())) - .append(build_name(tag.syntax())); + .append(build_column_label(tag.syntax())); } ( leading_comments(item.syntax()).append(item_doc), @@ -20212,7 +20225,7 @@ fn build_xml_pi_fn<'a>(xml_pi_fn: ast::XmlPiFn) -> Doc<'a> { body = body .append(Doc::space()) .append(leading_comments(target.syntax())) - .append(build_name(target.syntax())); + .append(build_column_label(target.syntax())); } if let Some(expr) = xml_pi_fn.expr() { if let Some(comma) = xml_pi_fn.comma_token() { diff --git a/crates/squawk_fmt/src/lib.rs b/crates/squawk_fmt/src/lib.rs index 69247dd8..4a9b0a23 100644 --- a/crates/squawk_fmt/src/lib.rs +++ b/crates/squawk_fmt/src/lib.rs @@ -1,3 +1,4 @@ mod fmt; +pub mod token_compare; pub use fmt::{fmt, fmt_str}; diff --git a/crates/squawk_fmt/src/main.rs b/crates/squawk_fmt/src/main.rs index 23d5c2c5..f4b4aa22 100644 --- a/crates/squawk_fmt/src/main.rs +++ b/crates/squawk_fmt/src/main.rs @@ -5,6 +5,7 @@ use std::process::ExitCode; use annotate_snippets::{AnnotationKind, Level, Renderer, Snippet, renderer::DecorStyle}; use anyhow::Result; use clap::Parser; +use squawk_fmt::token_compare::assert_no_dropped_tokens; use squawk_syntax::SourceFile; #[derive(Parser)] @@ -51,6 +52,21 @@ fn main() -> Result { return Ok(ExitCode::FAILURE); } - write!(io::stdout().lock(), "{}", squawk_fmt::fmt_str(&input)?)?; + let formatted = squawk_fmt::fmt_str(&input)?; + assert_no_dropped_tokens(&input, &formatted); + + let reparse = SourceFile::parse(&formatted); + assert!( + reparse.errors().is_empty(), + "formatted output has syntax errors:\n{}\n\nformatted output:\n{formatted}", + reparse + .errors() + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + ); + + write!(io::stdout().lock(), "{formatted}")?; Ok(ExitCode::SUCCESS) } diff --git a/crates/squawk_fmt/src/token_compare.rs b/crates/squawk_fmt/src/token_compare.rs new file mode 100644 index 00000000..782e4597 --- /dev/null +++ b/crates/squawk_fmt/src/token_compare.rs @@ -0,0 +1,90 @@ +use squawk_lexer::{Token, TokenKind, tokenize}; + +fn meaningful_tokens(text: &str) -> Vec<(TokenKind, &str)> { + let mut tokens = Vec::new(); + let mut offset = 0; + for Token { kind, len } in tokenize(text) { + let len = len as usize; + if kind != TokenKind::Eof && kind != TokenKind::Whitespace { + tokens.push((kind, &text[offset..offset + len])); + } + offset += len; + } + tokens +} + +fn tokens_equivalent(before: (TokenKind, &str), after: (TokenKind, &str)) -> bool { + let (before_kind, before_text) = before; + let (after_kind, after_text) = after; + + if before_kind == after_kind { + return before_text.eq_ignore_ascii_case(after_text); + } + + // The formatter removes unnecessary identifier quotes, so compare quoted + // and unquoted identifier tokens by their contents. + fn unquote<'a>(kind: &TokenKind, text: &'a str) -> Option<&'a str> { + match kind { + TokenKind::QuotedIdent { .. } => text + .strip_prefix('"') + .and_then(|text| text.strip_suffix('"')), + TokenKind::Ident => Some(text), + _ => None, + } + } + + match ( + unquote(&before_kind, before_text), + unquote(&after_kind, after_text), + ) { + (Some(before), Some(after)) => before.eq_ignore_ascii_case(after), + _ => false, + } +} + +pub fn assert_no_dropped_tokens(before: &str, after: &str) { + let before_tokens = meaningful_tokens(before); + let after_tokens = meaningful_tokens(after); + + let before_len = before_tokens.len(); + let after_len = after_tokens.len(); + + for (index, (&before, &after)) in before_tokens.iter().zip(&after_tokens).enumerate() { + assert!( + tokens_equivalent(before, after), + "token mismatch at position {index}:\n before: {:?} {:?}\n after: {:?} {:?}", + before.0, + before.1, + after.0, + after.1 + ); + } + + assert!( + before_len == after_len, + "token count mismatch: before has {before_len} tokens, after has {after_len} tokens\n {}", + if before_len > after_len { + let dropped = &before_tokens[after_len..]; + format!( + "dropped {} token(s): {}", + dropped.len(), + dropped + .iter() + .map(|(kind, text)| format!("{kind:?} {text:?}")) + .collect::>() + .join(", ") + ) + } else { + let extra = &after_tokens[before_len..]; + format!( + "extra {} token(s): {}", + extra.len(), + extra + .iter() + .map(|(kind, text)| format!("{kind:?} {text:?}")) + .collect::>() + .join(", ") + ) + } + ); +} diff --git a/crates/squawk_fmt/tests/after/copy.snap b/crates/squawk_fmt/tests/after/copy.snap index 80ee8d87..8f99b616 100644 --- a/crates/squawk_fmt/tests/after/copy.snap +++ b/crates/squawk_fmt/tests/after/copy.snap @@ -8,6 +8,8 @@ copy foo (id, name) to stdout with (format csv, header true, delimiter ',', null '', encoding 'UTF8'); +copy foo to stdout (select 'x'); + copy ( select id, a_very_long_column_name, another_very_long_column_name from a_very_long_schema_name.a_very_long_table_name diff --git a/crates/squawk_fmt/tests/after/create_table_options.snap b/crates/squawk_fmt/tests/after/create_table_options.snap index 658d408e..2cd612e2 100644 --- a/crates/squawk_fmt/tests/after/create_table_options.snap +++ b/crates/squawk_fmt/tests/after/create_table_options.snap @@ -11,6 +11,8 @@ create temporary table if not exists t (id int) on commit delete rows tablespace fast; +create table label_options (id int) with (select = 1, from.where = 2); + create global temporary table global_temp (id int) on commit preserve rows; diff --git a/crates/squawk_fmt/tests/after/create_text_search_parser.snap b/crates/squawk_fmt/tests/after/create_text_search_parser.snap index 27d48d5c..cb39ec54 100644 --- a/crates/squawk_fmt/tests/after/create_text_search_parser.snap +++ b/crates/squawk_fmt/tests/after/create_text_search_parser.snap @@ -5,14 +5,14 @@ input_file: crates/squawk_fmt/tests/before/create_text_search_parser.sql create text search parser public.default_parser ( start = public.parser_start, gettoken = public.parser_gettoken, - "end" = public.parser_end, + end = public.parser_end, lextypes = public.parser_lextypes ); create text search parser extraordinarily_long_schema_name.extraordinarily_long_parser_name ( start = public.extraordinarily_long_parser_start_function_name, gettoken = public.extraordinarily_long_parser_gettoken_function_name, - "end" = public.extraordinarily_long_parser_end_function_name, + end = public.extraordinarily_long_parser_end_function_name, lextypes = public.extraordinarily_long_parser_lextypes_function_name, headline = public.extraordinarily_long_parser_headline_function_name ); @@ -20,6 +20,6 @@ create text search parser extraordinarily_long_schema_name.extraordinarily_long_ create /* text keyword */ text /* search keyword */ search /* parser keyword */ parser /* name */ public /* dot */./* name segment */ commented_parser /* left parenthesis */ ( /* first option */ start /* equals */ = /* value */ public.parser_start /* comma */, /* second option */ gettoken = /* second value */ public.parser_gettoken /* comma two */, - /* third option */ "end" = public.parser_end, + /* third option */ end = public.parser_end, lextypes = public.parser_lextypes /* right parenthesis */ )/* semicolon */; diff --git a/crates/squawk_fmt/tests/after/select_expr.snap b/crates/squawk_fmt/tests/after/select_expr.snap index 480cb8f1..5eb3aa11 100644 --- a/crates/squawk_fmt/tests/after/select_expr.snap +++ b/crates/squawk_fmt/tests/after/select_expr.snap @@ -799,6 +799,14 @@ select a_very_long_function_name( third_very_long_argument_name ); +select + foo.select, + foo.left, + foo.array, + foo.filter, + foo."Mixed", + foo."has space"; + select json_object( 'a': 1, 'b' value 2 format json diff --git a/crates/squawk_fmt/tests/after/types.snap b/crates/squawk_fmt/tests/after/types.snap index 97e612c4..8bdcd579 100644 --- a/crates/squawk_fmt/tests/after/types.snap +++ b/crates/squawk_fmt/tests/after/types.snap @@ -186,7 +186,7 @@ select numeric(10, 2) '10', foo(1) '100', foo.bar(10, 2) '100', - foo."select" 'x'; + foo.select 'x'; select varchar e'a', diff --git a/crates/squawk_fmt/tests/after/update.snap b/crates/squawk_fmt/tests/after/update.snap index 3c8093af..0782354f 100644 --- a/crates/squawk_fmt/tests/after/update.snap +++ b/crates/squawk_fmt/tests/after/update.snap @@ -14,7 +14,7 @@ update foo set (a, b) = row(1, default), (c, d) = (select x, y from bar), - payload.field[1][2:3] = 4; + payload.select[1][2:3] = 4; update a_very_long_schema_name.a_very_long_table_name set diff --git a/crates/squawk_fmt/tests/after/xml_functions.snap b/crates/squawk_fmt/tests/after/xml_functions.snap index 991d2745..74020e52 100644 --- a/crates/squawk_fmt/tests/after/xml_functions.snap +++ b/crates/squawk_fmt/tests/after/xml_functions.snap @@ -3,9 +3,9 @@ source: crates/squawk_fmt/tests/tests.rs input_file: crates/squawk_fmt/tests/before/xml_functions.sql --- select - xmlelement(name foo), + xmlelement(name select), xmlelement(name foo, 1, 2), - xmlelement(name foo, xmlattributes(a, b as c)), + xmlelement(name foo, xmlattributes(a as from, b as c)), xmlelement(name foo, xmlattributes(a as attr), x, y), xmlelement( name a_very_long_element_name, @@ -42,7 +42,7 @@ select /* before document */ doc /* before second by */ by /* before value */ value /* before closing paren */ ) /* after exists */, - xmlforest(a, b as foo), + xmlforest(a, b as where), xmlforest( first_very_long_expression as first_very_long_element_name, second_very_long_expression as second_very_long_element_name @@ -63,7 +63,7 @@ select /* before expression */ value /* before preserve */ preserve /* before whitespace */ whitespace /* before closing paren */ ) /* after parse */, - xmlpi(name php), + xmlpi(name select), xmlpi(name php, 'echo'), xmlpi( name a_very_long_xml_processing_instruction_target, diff --git a/crates/squawk_fmt/tests/before/copy.sql b/crates/squawk_fmt/tests/before/copy.sql index 555cdd0c..c2bb184d 100644 --- a/crates/squawk_fmt/tests/before/copy.sql +++ b/crates/squawk_fmt/tests/before/copy.sql @@ -2,6 +2,8 @@ COPY foo FROM '/tmp/foo.csv'; copy foo (id, name) to stdout with (format csv, header true, delimiter ',', null '', encoding 'UTF8'); +copy foo to stdout ("select" 'x'); + copy (select id, a_very_long_column_name, another_very_long_column_name from a_very_long_schema_name.a_very_long_table_name) to program 'gzip > /tmp/a_very_long_output_file_name.csv' with (format csv, header on); copy binary foo from stdin binary freeze csv header json delimiter as ',' null as '' quote as '"' escape as '\\' encoding 'UTF8' force not null id,name force quote * force null description where id > 0; diff --git a/crates/squawk_fmt/tests/before/create_table_options.sql b/crates/squawk_fmt/tests/before/create_table_options.sql index 25def3d3..11533199 100644 --- a/crates/squawk_fmt/tests/before/create_table_options.sql +++ b/crates/squawk_fmt/tests/before/create_table_options.sql @@ -1,5 +1,7 @@ create temporary table if not exists t (id int) inherits (parent, archive.parent) partition by hash (id) using heap with (fillfactor=70) on commit delete rows tablespace fast; +create table label_options (id int) with ("select" = 1, "from"."where" = 2); + create global temporary table global_temp (id int) on commit preserve rows; create local temp table local_temp (id int) on commit drop; diff --git a/crates/squawk_fmt/tests/before/select_expr.sql b/crates/squawk_fmt/tests/before/select_expr.sql index 5e425bc7..9dbd2272 100644 --- a/crates/squawk_fmt/tests/before/select_expr.sql +++ b/crates/squawk_fmt/tests/before/select_expr.sql @@ -332,6 +332,8 @@ select select a_very_long_function_name(first_very_long_argument_name, second_very_long_argument_name, third_very_long_argument_name); +select foo."select", foo."left", foo."array", foo."filter", foo."Mixed", foo."has space"; + select json_object( 'a': 1, 'b' value 2 format json diff --git a/crates/squawk_fmt/tests/before/update.sql b/crates/squawk_fmt/tests/before/update.sql index 5e0561b1..f23d6953 100644 --- a/crates/squawk_fmt/tests/before/update.sql +++ b/crates/squawk_fmt/tests/before/update.sql @@ -2,7 +2,7 @@ update foo set a = 1; UPDATE ONLY (foo) AS f SET a = 1, b = DEFAULT FROM bar WHERE f.id = bar.id RETURNING f.*; -UPDATE foo SET (a, b) = ROW (1, DEFAULT), (c, d) = (SELECT x, y FROM bar), payload.field[1][2:3] = 4; +UPDATE foo SET (a, b) = ROW (1, DEFAULT), (c, d) = (SELECT x, y FROM bar), payload."select"[1][2:3] = 4; UPDATE a_very_long_schema_name.a_very_long_table_name SET a_very_long_first_column_name = 'a very long replacement value', a_very_long_second_column_name = 'another very long replacement value' WHERE organization_id = 12345 AND status = 'active' RETURNING id, a_very_long_first_column_name; diff --git a/crates/squawk_fmt/tests/before/xml_functions.sql b/crates/squawk_fmt/tests/before/xml_functions.sql index 68cca217..e3493bc6 100644 --- a/crates/squawk_fmt/tests/before/xml_functions.sql +++ b/crates/squawk_fmt/tests/before/xml_functions.sql @@ -1,7 +1,7 @@ select - XMLELEMENT(NAME foo), + XMLELEMENT(NAME "select"), XMLELEMENT(NAME foo, 1, 2), - XMLELEMENT(NAME foo, XMLATTRIBUTES(a, b AS c)), + XMLELEMENT(NAME foo, XMLATTRIBUTES(a AS "from", b AS c)), XMLELEMENT(NAME foo, XMLATTRIBUTES(a AS attr), x, y), XMLELEMENT(NAME a_very_long_element_name, XMLATTRIBUTES(a_very_long_xml_attribute_expression AS a_very_long_xml_attribute_name, a_second_very_long_xml_attribute_expression AS a_second_very_long_xml_attribute_name), first_very_long_content_expression, second_very_long_content_expression, third_very_long_content_expression), /* before element */ XMLELEMENT /* before outer opening paren */ ( /* before name */ NAME /* before tag */ "tag" /* before attributes comma */, /* before xmlattributes */ XMLATTRIBUTES /* before attributes opening paren */ ( /* before first attribute */ a /* before as */ AS /* before attribute name */ "attr" /* before attribute comma */, /* before second attribute */ b /* before attributes closing paren */ ) /* before content comma */, /* before content */ x /* before outer closing paren */ ) /* after element */, @@ -10,14 +10,14 @@ select XMLEXISTS('/foo' PASSING doc BY VALUE), XMLEXISTS(a_very_long_xml_exists_path_expression PASSING BY REF a_very_long_xml_exists_document_expression BY VALUE), /* before exists */ XMLEXISTS /* before opening paren */ ( /* before row */ '/foo' /* before passing */ PASSING /* before first by */ BY /* before ref */ REF /* before document */ doc /* before second by */ BY /* before value */ VALUE /* before closing paren */ ) /* after exists */, - XMLFOREST(a, b AS foo), + XMLFOREST(a, b AS "where"), XMLFOREST(first_very_long_expression AS first_very_long_element_name, second_very_long_expression AS second_very_long_element_name), /* before forest */ XMLFOREST /* before opening paren */ ( /* before first expression */ a /* before as */ AS /* before tag */ "first" /* before comma */, /* before second expression */ b /* before closing paren */ ) /* after forest */, XMLPARSE(DOCUMENT '' PRESERVE WHITESPACE), XMLPARSE(CONTENT value STRIP WHITESPACE), XMLPARSE(DOCUMENT a_very_long_xml_parse_document_expression_that_forces_the_xml_parse_node_to_wrap PRESERVE WHITESPACE), /* before parse */ XMLPARSE /* before opening paren */ ( /* before kind */ DOCUMENT /* before expression */ value /* before preserve */ PRESERVE /* before whitespace */ WHITESPACE /* before closing paren */ ) /* after parse */, - XMLPI(NAME php), + XMLPI(NAME "select"), XMLPI(NAME php, 'echo'), XMLPI(NAME a_very_long_xml_processing_instruction_target, a_very_long_xml_processing_instruction_expression_that_forces_the_xml_pi_node_to_wrap), /* before pi */ XMLPI /* before opening paren */ ( /* before name */ NAME /* before target */ php /* before comma */, /* before expression */ 'echo' /* before closing paren */ ) /* after pi */, diff --git a/crates/squawk_fmt/tests/tests.rs b/crates/squawk_fmt/tests/tests.rs index cb055bde..20280c69 100644 --- a/crates/squawk_fmt/tests/tests.rs +++ b/crates/squawk_fmt/tests/tests.rs @@ -1,7 +1,7 @@ use camino::Utf8Path; use dir_test::{Fixture, dir_test}; use insta::{assert_snapshot, with_settings}; -use squawk_lexer::{Token, TokenKind, tokenize}; +use squawk_fmt::token_compare::assert_no_dropped_tokens; #[dir_test( dir: "$CARGO_MANIFEST_DIR/tests/before", @@ -18,6 +18,7 @@ fn fmt(fixture: Fixture<&str>) { let formatted = squawk_fmt::fmt_str(content).unwrap(); assert_no_dropped_tokens(content, &formatted); + assert_parses(&formatted); with_settings!({ omit_expression => true, @@ -45,6 +46,7 @@ fn fmt_with_line_ending(line_ending: &str) -> String { match squawk_fmt::fmt_str(&sql) { Ok(formatted) => { assert_no_dropped_tokens(&sql, &formatted); + assert_parses(&formatted); formatted.replace('\r', "") } Err(err) => format!("error: {err}"), @@ -86,86 +88,16 @@ fn fmt_cr_line_endings() { assert_snapshot!(fmt_with_line_ending("\r"), @"-- a commentselect 1;/* a comment * spanning lines */select 'a', 'really long string ';"); } -fn meaningful_tokens(text: &str) -> Vec<(TokenKind, &str)> { - let mut tokens: Vec<(TokenKind, &str)> = vec![]; - let mut offset = 0; - for Token { kind, len } in tokenize(text) { - let len = len as usize; - if kind != TokenKind::Eof && kind != TokenKind::Whitespace { - tokens.push((kind, &text[offset..offset + len])); - } - offset += len; - } - tokens -} - -fn tokens_equivalent(before: (TokenKind, &str), after: (TokenKind, &str)) -> bool { - let (bkind, btext) = before; - let (akind, atext) = after; - - if bkind == akind { - return btext.eq_ignore_ascii_case(atext); - } - - // We convert `select 1 "foo"` to `select 1 foo` so we need to do some quote - // munging - fn unquote<'a>(kind: &TokenKind, text: &'a str) -> Option<&'a str> { - match kind { - TokenKind::QuotedIdent { .. } => { - text.strip_prefix('"').and_then(|t| t.strip_suffix('"')) - } - TokenKind::Ident => Some(text), - _ => None, - } - } - - match (unquote(&bkind, btext), unquote(&akind, atext)) { - (Some(b), Some(a)) => b.eq_ignore_ascii_case(a), - _ => false, - } -} - -fn assert_no_dropped_tokens(before: &str, after: &str) { - let before_tokens = meaningful_tokens(before); - let after_tokens = meaningful_tokens(after); - - let before_len = before_tokens.len(); - let after_len = after_tokens.len(); - - for (i, (&(bkind, btext), &(akind, atext))) in - before_tokens.iter().zip(after_tokens.iter()).enumerate() - { - assert!( - tokens_equivalent((bkind, btext), (akind, atext)), - "token mismatch at position {i}:\n before: {bkind:?} {btext:?}\n after: {akind:?} {atext:?}" - ); - } - +fn assert_parses(formatted: &str) { + let parse = squawk_syntax::ast::SourceFile::parse(formatted); assert!( - before_len == after_len, - "token count mismatch: before has {before_len} tokens, after has {after_len} tokens\n {}", - if before_len > after_len { - let dropped = &before_tokens[after_len..]; - format!( - "dropped {} token(s): {}", - dropped.len(), - dropped - .iter() - .map(|(k, t)| format!("{k:?} {t:?}")) - .collect::>() - .join(", ") - ) - } else { - let extra = &after_tokens[before_len..]; - format!( - "extra {} token(s): {}", - extra.len(), - extra - .iter() - .map(|(k, t)| format!("{k:?} {t:?}")) - .collect::>() - .join(", ") - ) - } + parse.errors().is_empty(), + "formatted output has syntax errors:\n{}\n\nformatted output:\n{formatted}", + parse + .errors() + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") ); }