From 4095677accd6934bd7351c90ebb6e972cb996624 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:02:34 +0000 Subject: [PATCH] fix: read Databricks and DuckDB strings with their own escape rules SQLBuild's lexical rules gave Databricks and DuckDB one shared entry with no backslash escapes, no raw strings and flat block comments. SQLQuack and both adapters read them differently: Databricks honours a backslash in '...' and "..." outside r'...' raw strings, DuckDB honours it inside E'...', and both nest block comments. The expression-macro dedent in sqb format reads the formatted body with these rules. A multi-line string after a backslash-escaped quote looked closed early, so its next line lost two spaces on every run and the string's value changed. Databricks now has its own entry, and DuckDB uses the PostgreSQL rules its adapter already declares. The same rules decide quoted text for semantic-validation normalisation, fast lineage, dynamic pivot, SQL tree nodes and query-facts shapes. --- .../src/lint_text/tests/test_lint_text.rs | 22 ++++++++-- .../sql_scan/main/dialect_lexical_syntax.rs | 5 ++- .../src/sql_scan/tests/test_dialect_scan.rs | 42 +++++++++++++++++++ .../sqlbuild/cli/commands/main/_test_types.py | 3 +- .../commands/main/test_format_preservation.py | 26 +++++++++++- 5 files changed, 91 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index 1e180c514..8f347b19f 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -35,6 +35,10 @@ const CROSS_JOIN_FILE: &str = "select o.order_id\nfrom orders as o\n-- sqb: igno const NORMALISE_NAME_MACRO: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(\n @column_name,\n '[[:space:]]+',\n ' '\n ))),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n"; const ESCAPED_QUOTE_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(\n @v,\n 'O\\'Brien\n kept indent',\n 'tail'\n)\n"; const ESCAPED_QUOTE_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(@v, 'O\\'Brien\n kept indent', 'tail')\n"; +const E_STRING_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(\n @v,\n E'O\\'Brien\n kept indent',\n 'tail'\n)\n"; +const E_STRING_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCONCAT(@v, E'O\\'Brien\n kept indent', 'tail')\n"; +const NESTED_COMMENT_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* outer /* inner */ it's\n kept */\n 'x'\n)\n"; +const NESTED_COMMENT_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* outer /* inner */ it's\n kept */ 'x'\n)\n"; const BLOCK_COMMENT_MACRO: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* first\n second */\n 'x'\n)\n"; const BLOCK_COMMENT_FORMATTED: &str = "MACRO (arguments (v VARCHAR), returns VARCHAR);\n\nCOALESCE(\n @v, /* first\n second */ 'x'\n)\n"; const NORMALISE_NAME_FORMATTED: &str = "MACRO (\n description \"Normalise a name\",\n arguments (column_name VARCHAR),\n returns VARCHAR,\n);\n\nREGEXP_REPLACE(\n TRIM(\n UPPER(\n -- Collapse repeated spaces\n REGEXP_REPLACE(@column_name, '[[:space:]]+', ' ')\n )\n ),\n '(^| )([A-Z]) ',\n '\\\\1\\\\2'\n)\n"; @@ -1011,22 +1015,34 @@ fn given_expression_macro_bodies_when_formatting_twice_then_every_dialect_lays_t let test_cases = [ ExpressionMacroFormatTestCase { description: "a body with a comment between nested calls", - dialects: &["duckdb", "snowflake"], + dialects: &["duckdb", "snowflake", "databricks"], contents: NORMALISE_NAME_MACRO.to_string(), expected_formatted: NORMALISE_NAME_FORMATTED.to_string(), }, ExpressionMacroFormatTestCase { description: "a body nesting fifteen calls, authored flush left", - dialects: &["duckdb", "snowflake"], + dialects: &["duckdb", "snowflake", "databricks"], contents: nested_macro(15, ""), expected_formatted: nested_macro(15, " "), }, ExpressionMacroFormatTestCase { description: "a multi-line string after a backslash-escaped quote", - dialects: &["snowflake"], + dialects: &["snowflake", "databricks"], contents: ESCAPED_QUOTE_MACRO.to_string(), expected_formatted: ESCAPED_QUOTE_FORMATTED.to_string(), }, + ExpressionMacroFormatTestCase { + description: "a multi-line E-string after a backslash-escaped quote", + dialects: &["duckdb"], + contents: E_STRING_MACRO.to_string(), + expected_formatted: E_STRING_FORMATTED.to_string(), + }, + ExpressionMacroFormatTestCase { + description: "a nested block comment holding an apostrophe", + dialects: &["duckdb", "databricks"], + contents: NESTED_COMMENT_MACRO.to_string(), + expected_formatted: NESTED_COMMENT_FORMATTED.to_string(), + }, ExpressionMacroFormatTestCase { description: "a block comment with indented lines", dialects: &["duckdb", "snowflake"], diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs index 3028fc6db..09a09777d 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/dialect_lexical_syntax.rs @@ -24,7 +24,10 @@ pub fn dialect_lexical_syntax(dialect: &str) -> LexicalSyntax { line_comment_prefixes: lines(&["--", "# ", "#!", "//"]), ..LexicalSyntax::default() }, - "databricks" | "duckdb" => LexicalSyntax { + "databricks" => LexicalSyntax { + backslash_escape_quotes: strings(&["'", "\""]), + raw_string_prefix: true, + nested_block_comments: true, line_comment_prefixes: lines(&["--"]), ..LexicalSyntax::default() }, diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs index 24354b62d..40a103128 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_dialect_scan.rs @@ -200,6 +200,48 @@ fn given_dialect_lexical_syntax_when_scanning_non_code_then_dialect_decides_boun syntax: single_quote_backslashes(), expected_first_range: Some((0, 4)), }, + DialectFirstNonCodeRangeTestCase { + description: "a Databricks string continues past a backslash-escaped quote", + sql: r"'a\'b' x", + syntax: dialect_lexical_syntax("databricks"), + expected_first_range: Some((0, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "Databricks backslash escapes hold in double-quoted strings", + sql: r#""a\"b" x"#, + syntax: dialect_lexical_syntax("databricks"), + expected_first_range: Some((0, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a Databricks raw string ends at the quote after a backslash", + sql: r"r'a\' x", + syntax: dialect_lexical_syntax("databricks"), + expected_first_range: Some((1, 5)), + }, + DialectFirstNonCodeRangeTestCase { + description: "Databricks block comments nest", + sql: "/* a /* b */ c */ x", + syntax: dialect_lexical_syntax("databricks"), + expected_first_range: Some((0, 17)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a DuckDB E-string continues past a backslash-escaped quote", + sql: r"E'a\'b' x", + syntax: dialect_lexical_syntax("duckdb"), + expected_first_range: Some((1, 7)), + }, + DialectFirstNonCodeRangeTestCase { + description: "a plain DuckDB string ends at the quote after a backslash", + sql: r"'a\' x", + syntax: dialect_lexical_syntax("duckdb"), + expected_first_range: Some((0, 4)), + }, + DialectFirstNonCodeRangeTestCase { + description: "DuckDB block comments nest", + sql: "/* a /* b */ c */ x", + syntax: dialect_lexical_syntax("duckdb"), + expected_first_range: Some((0, 17)), + }, ]; for test_case in test_cases { let ranges: Vec<(usize, usize)> = dialect_non_code_ranges(test_case.sql, &test_case.syntax); diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index d6ab08535..0c995342a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -924,13 +924,14 @@ class SqlReadabilityTestCase: @dataclass(frozen=True) class ExpressionMacroFormatTestCase: - """A Snowflake expression macro in one location, and the file formatting leaves.""" + """An expression macro in one location, and the file formatting leaves on an adapter.""" description: str macro_directory: str consumer_folders: tuple[str, ...] authored_macro: str expected_macro: str + adapter: str = "snowflake" @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py index 611a807e8..e4a353483 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py @@ -348,13 +348,35 @@ def test_given_multibyte_text_before_restorations_when_formatting_then_output_is _NORMALISE_NAME_HEADER + "CONCAT(@column_name, 'O\\'Brien\n kept indent', 'tail')\n", ), + ExpressionMacroFormatTestCase( + "Databricks multi-line string after a backslash-escaped quote", + "models/staging/_sqlbuild/_macros/sql", + ("staging",), + _NORMALISE_NAME_HEADER + + "CONCAT(\n @column_name,\n 'O\\'Brien\n kept indent',\n 'tail'\n)\n", + _NORMALISE_NAME_HEADER + + "CONCAT(@column_name, 'O\\'Brien\n kept indent', 'tail')\n", + adapter="databricks", + ), + ExpressionMacroFormatTestCase( + "DuckDB multi-line E-string after a backslash-escaped quote", + "models/staging/_sqlbuild/_macros/sql", + ("staging",), + _NORMALISE_NAME_HEADER + + "CONCAT(\n @column_name,\n E'O\\'Brien\n kept indent',\n 'tail'\n)\n", + _NORMALISE_NAME_HEADER + + "CONCAT(@column_name, E'O\\'Brien\n kept indent', 'tail')\n", + adapter="duckdb", + ), ], ids=lambda case: case.description, ) -def test_given_snowflake_expression_macro_when_formatting_then_body_is_laid_out_once( +def test_given_expression_macro_when_formatting_then_body_is_laid_out_once( test_case: ExpressionMacroFormatTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{test_case.adapter}"\n' + ) macro: Path = tmp_path / test_case.macro_directory / "normalise_name.sql" macro.parent.mkdir(parents=True) macro.write_text(test_case.authored_macro)