From dc5505cdc009a8f9550577ebdf2568a6b04fd45a Mon Sep 17 00:00:00 2001 From: Jin Xu Date: Fri, 6 Nov 2026 22:09:54 +0800 Subject: [PATCH 1/4] Preserve semicolons inside skipped DDL constructs Keep character-level skipping lenient while respecting quoted and commented semicolons. Reuse the existing escape and comment rules without validating skipped tokens or changing partial DDL acceptance. --- parse_test.go | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++ token.go | 35 +++++++++++++++-- 2 files changed, 139 insertions(+), 3 deletions(-) diff --git a/parse_test.go b/parse_test.go index 1426e90..a3f7227 100644 --- a/parse_test.go +++ b/parse_test.go @@ -19,6 +19,7 @@ package sqlparser import ( "bytes" "fmt" + "io" "strings" "testing" ) @@ -726,6 +727,112 @@ func TestInsertComments(t *testing.T) { } } +func TestDDLTailSemicolons(t *testing.T) { + for _, prefix := range []string{"create index i on t(", "create table t (列 text check ("} { + for _, expression := range []string{ + "'a;b'", `"a;b"`, "`a;b`", "[a;b]", + "'a'';b'", `"a"";b"`, "`a``;b`", + "a -- a;b\n", "a # a;b\n", "a // a;b\n", + "a /* a;b */", "a /*! 列;a */", "'a/*;*/--#;b'", "'a;select 2 --'", + } { + first := prefix + expression + ")" + if strings.HasPrefix(prefix, "create table") { + first += ")" + } + t.Run(first, func(t *testing.T) { + testDDLTail(t, first+"; select 1", []string{first, "select 1"}, false) + }) + } + } +} + +func TestDDLTailQuoteModes(t *testing.T) { + for _, quote := range []string{"'", `"`} { + // Without backslash escaping, the quote after the backslash ends it. + first := "create index i on t(a) where a = " + quote + "a\\" + quote + testDDLTail(t, first+"; select 1", []string{first, "select 1"}, false) + // With escaping enabled, that quote and the following semicolon stay inside. + first += ";b" + quote + testDDLTail(t, first+"; select 1", []string{first, "select 1"}, true) + } + // Backticks use doubled delimiters, regardless of the backslash mode. + first := "create index i on t(`a\\`)" + testDDLTail(t, first+"; select 1", []string{first, "select 1"}, true) +} + +func TestDDLTailLeniency(t *testing.T) { + for _, input := range []string{ + "create index i on t(列1a)", + "create index i on t(a$b)", + "create index i on t(a) /*! 列 */", + "CREATE TABLE t (列 TEXT)", + "CREATE TABLE t ([a] INTEGER)", + "CREATE TABLE t(a INT", + } { + t.Run(input, func(t *testing.T) { + testDDLTail(t, input, []string{input}, false) + }) + } +} + +func TestDDLTailUnterminated(t *testing.T) { + for _, tail := range []string{"'a;b", `"a;b`, "`a;b", "[a;b", "/* a;b", "-- a;b", "# a;b", "// a;b"} { + input := "create index i on t(a) " + tail + t.Run(tail, func(t *testing.T) { + testDDLTail(t, input, []string{input}, false) + }) + } +} + +func testDDLTail(t *testing.T, input string, want []string, backslash bool) { + t.Helper() + if !backslash { + // Parse intentionally retains master's lenient handling of partial DDL batches. + if stmt, err := Parse(input); err != nil { + t.Errorf("Parse(%q): %v", input, err) + } else if _, ok := stmt.(*DDL); !ok { + t.Errorf("Parse(%q): got %T, want DDL", input, stmt) + } + } + tokenizer := NewStringTokenizer(input) + tokenizer.AllowBackSlashEscape = backslash + queries, statements, err := ParseMultiple(tokenizer) + if err != nil { + t.Errorf("ParseMultiple(%q): %v", input, err) + } else if len(queries) != len(want) || len(statements) != len(want) { + t.Errorf("ParseMultiple(%q): got %v and %d statements, want %v", input, queries, len(statements), want) + } else { + for i, query := range queries { + if query != want[i] { + t.Errorf("ParseMultiple(%q): query %d = %q, want %q", input, i, query, want[i]) + } + } + } + for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} { + tokenizer.AllowBackSlashEscape = backslash + for i := range want { + stmt, err := ParseNext(tokenizer) + if err != nil { + t.Errorf("ParseNext(%q), statement %d: %v", input, i, err) + break + } + if i == 0 { + if _, ok := stmt.(*DDL); !ok { + t.Errorf("ParseNext(%q): got %T, want DDL", input, stmt) + } + if tokenizer.Position != len(want[0])+1 { + t.Errorf("ParseNext(%q): stopped at %d, want %d", input, tokenizer.Position, len(want[0])+1) + } + } else if String(stmt) != "select 1" { + t.Errorf("ParseNext(%q): second statement = %q, want select 1", input, String(stmt)) + } + } + if _, err := ParseNext(tokenizer); err != io.EOF { + t.Errorf("ParseNext(%q): got %v after last statement, want EOF", input, err) + } + } +} + func TestCaseSensitivity(t *testing.T) { validSQL := []struct { input string diff --git a/token.go b/token.go index a21c9b0..b2eb924 100644 --- a/token.go +++ b/token.go @@ -509,10 +509,39 @@ scanAgain: // skipStatement scans until the EOF, or end of statement is encountered. func (tkn *Tokenizer) skipStatement() { - ch := tkn.lastChar - for ch != ';' && ch != eofChar { + for tkn.lastChar != ';' && tkn.lastChar != eofChar { + ch := tkn.lastChar tkn.next() - ch = tkn.lastChar + // Only locate boundaries; skipped DDL need not be valid tokens. + switch ch { + case '\'', '"': + tkn.scanString(ch, STRING) + case '`': + tkn.scanLiteralIdentifier() + case '[': + for tkn.lastChar != ']' && tkn.lastChar != eofChar { + tkn.next() + } + if tkn.lastChar == ']' { + tkn.next() + } + case '#': + tkn.scanCommentType1("#") + case '-': + if tkn.lastChar == '-' { + tkn.next() + tkn.scanCommentType1("--") + } + case '/': + switch tkn.lastChar { + case '*': + tkn.next() + tkn.scanCommentType2() + case '/': + tkn.next() + tkn.scanCommentType1("//") + } + } } } From df2362aa4d69f9bef800d941a7107675b6d08dcf Mon Sep 17 00:00:00 2001 From: Jin Xu Date: Fri, 6 Nov 2026 22:38:21 +0800 Subject: [PATCH 2/4] Preserve consumed delimiters during statement recovery The lexer could return an error after consuming a bracket opener or entering a malformed HEX literal. Keep that delimiter until recovery finishes the construct, so following statements remain visible. Cover bracket tails, EXPLAIN, ParseNext recovery and Reader buffer boundaries. Treat an unterminated HEX literal like other unclosed quotes. --- parse_test.go | 93 +++++++++++++++++++++++++++++++++++++++++++++++++-- token.go | 29 ++++++++++++---- 2 files changed, 114 insertions(+), 8 deletions(-) diff --git a/parse_test.go b/parse_test.go index ca5e1c4..5125361 100644 --- a/parse_test.go +++ b/parse_test.go @@ -842,6 +842,94 @@ func TestDDLTailUnterminated(t *testing.T) { } } +func TestDDLRecoveryOpeners(t *testing.T) { + for _, first := range []string{ + "CREATE TABLE t ([a;b] INTEGER)", + "CREATE TABLE t ([a'b] INTEGER)", + "CREATE TABLE t ([a--b] INTEGER)", + "CREATE TABLE t(a TEXT DEFAULT X'g;h')", + // These errors are already past a closing delimiter, or inside a + // fully consumed executable comment, and must not reopen it. + "CREATE TABLE t '['", + "CREATE TABLE t(a TEXT DEFAULT X'a')", + "CREATE TABLE t (`` INT)", + "CREATE TABLE t (/*! [ */ a INT)", + } { + t.Run(first, func(t *testing.T) { + testDDLTail(t, first+"; select 1", []string{first, "select 1"}, false) + }) + } +} + +func TestExplainTailSemicolons(t *testing.T) { + first := "EXPLAIN SELECT 'a;b'" + input := first + "; SELECT 1" + if _, err := Parse(input); err != nil { + t.Fatal(err) + } + queries, statements, err := ParseMultiple(NewStringTokenizer(input)) + if err != nil || len(queries) != 2 || len(statements) != 2 || queries[0] != first || queries[1] != "SELECT 1" { + t.Fatalf("ParseMultiple(%q): %v, %v, %v", input, queries, statements, err) + } + if _, ok := statements[0].(*Explain); !ok || String(statements[1]) != "select 1" { + t.Fatalf("unexpected statements: %T, %v", statements[0], statements[1]) + } + for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} { + if stmt, err := ParseNext(tokenizer); err != nil { + t.Fatal(err) + } else if _, ok := stmt.(*Explain); !ok || tokenizer.Position != len(first)+1 { + t.Fatalf("EXPLAIN boundary: %T at %d", stmt, tokenizer.Position) + } + if stmt, err := ParseNext(tokenizer); err != nil || String(stmt) != "select 1" { + t.Fatalf("second statement: %v, %v", stmt, err) + } + if _, err := ParseNext(tokenizer); err != io.EOF { + t.Fatalf("after last statement: %v", err) + } + } +} + +func TestParseNextRecoverySemicolons(t *testing.T) { + for _, first := range []string{"SELECT ) 'a;b'", "SELECT X'g;h'", "SELECT [a'b]"} { + input := first + "; SELECT 1" + for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} { + if _, err := ParseNext(tokenizer); err == nil { + t.Fatalf("ParseNext(%q): want first-statement error", input) + } + if tokenizer.Position != len(first)+1 { + t.Errorf("ParseNext(%q): recovered at %d, want %d", input, tokenizer.Position, len(first)+1) + } + if stmt, err := ParseNext(tokenizer); err != nil || String(stmt) != "select 1" { + t.Errorf("recovered statement from %q: %v, %v", input, stmt, err) + } + if _, err := ParseNext(tokenizer); err != io.EOF { + t.Errorf("after last statement: %v", err) + } + } + } +} + +func TestParseNextRecoveryUnterminatedHex(t *testing.T) { + input := "select x'78 from t; select 1 from t" + for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} { + if _, err := ParseNext(tokenizer); err == nil { + t.Fatal("want an unterminated hex literal error") + } + if _, err := ParseNext(tokenizer); err != io.EOF { + t.Fatalf("unterminated hex literal: got %v, want EOF", err) + } + } +} + +func TestDDLTailReaderBoundary(t *testing.T) { + prefix := "create index i on t(a) " + for _, tail := range []string{"'a;b'", "/* a;b */"} { + // The construct opens in the first Reader buffer and ends in the next. + first := prefix + strings.Repeat(" ", defaultBufSize-2-len(prefix)) + tail + testDDLTail(t, first+"; select 1", []string{first, "select 1"}, false) + } +} + func testDDLTail(t *testing.T, input string, want []string, backslash bool) { t.Helper() if !backslash { @@ -1355,8 +1443,9 @@ var ( input: "select 0xH from t", output: "syntax error at position 10 near '0x'", }, { - input: "select x'78 from t", - output: "syntax error at position 12 near '78'", + input: "select x'78 from t", + output: "syntax error at position 12 near '78'", + excludeMulti: true, }, { input: "select x'777' from t", output: "syntax error at position 14 near '777'", diff --git a/token.go b/token.go index b2eb924..a69ffce 100644 --- a/token.go +++ b/token.go @@ -42,6 +42,7 @@ type Tokenizer struct { lastChar uint16 Position int lastToken []byte + errorDelimiter uint16 // Unclosed delimiter at a lexical error. LastError error posVarIndex int ParseTree Statement @@ -354,6 +355,12 @@ func (tkn *Tokenizer) Error(err string) { } tkn.LastError = errors.New(buf.String()) + // Finish a construct whose opener the failing token already consumed. + if tkn.errorDelimiter != 0 { + tkn.skipToDelimiter(tkn.errorDelimiter) + tkn.errorDelimiter = 0 + } + // Try and re-sync to the next statement if tkn.lastChar != ';' { tkn.skipStatement() @@ -363,6 +370,7 @@ func (tkn *Tokenizer) Error(err string) { // Scan scans the tokenizer for the next token and returns // the token type and an optional value. func (tkn *Tokenizer) Scan() (int, []byte) { + tkn.errorDelimiter = 0 scanAgain: if tkn.specialComment != nil { // Enter specialComment scan mode. @@ -502,6 +510,9 @@ scanAgain: case '`': return tkn.scanLiteralIdentifier() default: + if ch == '[' { + tkn.errorDelimiter = ']' + } return LEX_ERROR, []byte{byte(ch)} } } @@ -519,12 +530,7 @@ func (tkn *Tokenizer) skipStatement() { case '`': tkn.scanLiteralIdentifier() case '[': - for tkn.lastChar != ']' && tkn.lastChar != eofChar { - tkn.next() - } - if tkn.lastChar == ']' { - tkn.next() - } + tkn.skipToDelimiter(']') case '#': tkn.scanCommentType1("#") case '-': @@ -545,6 +551,16 @@ func (tkn *Tokenizer) skipStatement() { } } +// skipToDelimiter resumes after an already consumed opening delimiter. +func (tkn *Tokenizer) skipToDelimiter(delim uint16) { + for tkn.lastChar != delim && tkn.lastChar != eofChar { + tkn.next() + } + if tkn.lastChar == delim { + tkn.next() + } +} + func (tkn *Tokenizer) skipBlank() { ch := tkn.lastChar for ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t' { @@ -576,6 +592,7 @@ func (tkn *Tokenizer) scanHex() (int, []byte) { buffer := &bytes2.Buffer{} tkn.scanMantissa(16, buffer) if tkn.lastChar != '\'' { + tkn.errorDelimiter = '\'' return LEX_ERROR, buffer.Bytes() } tkn.next() From d282bc718db0069f9bac7921730da170d4e76154 Mon Sep 17 00:00:00 2001 From: Jin Xu Date: Sun, 8 Nov 2026 16:25:24 +0800 Subject: [PATCH 3/4] Cover single statements in skipped SQL tails Verify that EXPLAIN and CREATE INDEX containing quoted semicolons each retain one original statement. Consumers enforcing one statement per query pattern must not mistake these inputs for batches. --- parse_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/parse_test.go b/parse_test.go index 5125361..0316833 100644 --- a/parse_test.go +++ b/parse_test.go @@ -785,6 +785,34 @@ func TestFloatColumnPrecisionSemicolonDefault(t *testing.T) { } } +func TestSkippedTailSingleStatements(t *testing.T) { + for _, input := range []string{ + "EXPLAIN SELECT 'a;select 2 --'", + "create index i on t(a) where a = 'x;select 2 --'", + } { + t.Run(input, func(t *testing.T) { + if _, err := Parse(input); err != nil { + t.Fatal(err) + } + queries, statements, err := ParseMultiple(NewStringTokenizer(input)) + if err != nil || len(queries) != 1 || len(statements) != 1 || queries[0] != input { + t.Fatalf("want one unchanged statement: %v, %v, %v", queries, statements, err) + } + for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} { + if _, err := ParseNext(tokenizer); err != nil { + t.Fatal(err) + } + if tokenizer.Position != len(input)+1 { + t.Errorf("stopped at %d, want EOF at %d", tokenizer.Position, len(input)+1) + } + if _, err := ParseNext(tokenizer); err != io.EOF { + t.Errorf("after single statement: %v, want EOF", err) + } + } + }) + } +} + func TestDDLTailSemicolons(t *testing.T) { for _, prefix := range []string{"create index i on t(", "create table t (列 text check ("} { for _, expression := range []string{ From 3400501a671527e10066b9310a84368839d7806c Mon Sep 17 00:00:00 2001 From: Jin Xu Date: Sun, 8 Nov 2026 16:41:47 +0800 Subject: [PATCH 4/4] Drop malformed HEX recovery changes Malformed HEX cannot be a valid single SQLite statement, so special recovery does not help avoid false rejections. Remove its state and fixtures, and restore the existing ParseNext error continuation check. Keep consumed-bracket recovery for valid SQLite identifiers. --- parse_test.go | 21 +++------------------ token.go | 1 - 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/parse_test.go b/parse_test.go index 0316833..391f132 100644 --- a/parse_test.go +++ b/parse_test.go @@ -875,11 +875,9 @@ func TestDDLRecoveryOpeners(t *testing.T) { "CREATE TABLE t ([a;b] INTEGER)", "CREATE TABLE t ([a'b] INTEGER)", "CREATE TABLE t ([a--b] INTEGER)", - "CREATE TABLE t(a TEXT DEFAULT X'g;h')", // These errors are already past a closing delimiter, or inside a // fully consumed executable comment, and must not reopen it. "CREATE TABLE t '['", - "CREATE TABLE t(a TEXT DEFAULT X'a')", "CREATE TABLE t (`` INT)", "CREATE TABLE t (/*! [ */ a INT)", } { @@ -918,7 +916,7 @@ func TestExplainTailSemicolons(t *testing.T) { } func TestParseNextRecoverySemicolons(t *testing.T) { - for _, first := range []string{"SELECT ) 'a;b'", "SELECT X'g;h'", "SELECT [a'b]"} { + for _, first := range []string{"SELECT ) 'a;b'", "SELECT [a'b]"} { input := first + "; SELECT 1" for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} { if _, err := ParseNext(tokenizer); err == nil { @@ -937,18 +935,6 @@ func TestParseNextRecoverySemicolons(t *testing.T) { } } -func TestParseNextRecoveryUnterminatedHex(t *testing.T) { - input := "select x'78 from t; select 1 from t" - for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} { - if _, err := ParseNext(tokenizer); err == nil { - t.Fatal("want an unterminated hex literal error") - } - if _, err := ParseNext(tokenizer); err != io.EOF { - t.Fatalf("unterminated hex literal: got %v, want EOF", err) - } - } -} - func TestDDLTailReaderBoundary(t *testing.T) { prefix := "create index i on t(a) " for _, tail := range []string{"'a;b'", "/* a;b */"} { @@ -1471,9 +1457,8 @@ var ( input: "select 0xH from t", output: "syntax error at position 10 near '0x'", }, { - input: "select x'78 from t", - output: "syntax error at position 12 near '78'", - excludeMulti: true, + input: "select x'78 from t", + output: "syntax error at position 12 near '78'", }, { input: "select x'777' from t", output: "syntax error at position 14 near '777'", diff --git a/token.go b/token.go index a69ffce..11cc2b8 100644 --- a/token.go +++ b/token.go @@ -592,7 +592,6 @@ func (tkn *Tokenizer) scanHex() (int, []byte) { buffer := &bytes2.Buffer{} tkn.scanMantissa(16, buffer) if tkn.lastChar != '\'' { - tkn.errorDelimiter = '\'' return LEX_ERROR, buffer.Bytes() } tkn.next()