From f92066cdf20a3438b8fb68c77d9b78b6d126f4ce Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 22 Oct 2026 21:45:55 +0700 Subject: [PATCH 1/2] fix(safe-mode): read MySQL, MariaDB and TiDB executable comments as each server runs them --- .../TableProQuery/SQLWriteClassifier.swift | 14 +- .../TableProSQLGrammar/SQLCatalogEffect.swift | 12 +- .../SQLCodeProjection.swift | 4 +- .../SQLExecutableCommentReadings.swift | 550 ++++++++++++++++++ .../SQLLexicalReadings.swift | 25 + .../TableProSQLGrammar/SQLNonCodeSpan.swift | 75 ++- .../Sources/TableProSQLGrammar/SqlLexer.swift | 42 +- .../SQLStatementSplittingTests.swift | 17 + .../SQLWriteClassifierTests.swift | 18 + .../SQLExecutableCommentReadingsTests.swift | 180 ++++++ .../SQLNonCodeSpanTests.swift | 26 + .../MySQLSessionFootprint.swift | 70 ++- .../MySQLSessionFootprintTests.swift | 27 + .../TableProPluginKit/SQLFeatureLexer.swift | 16 +- .../DataWrite/RowChangeStatementFactory.swift | 9 +- .../Core/Utilities/SQL/QueryClassifier.swift | 87 ++- .../Core/Utilities/SQL/SQLFileParser.swift | 109 ++-- .../SQL/TableEditStatementParser.swift | 5 +- .../Helpers/SessionLocalStatements.swift | 4 +- .../ExternalStatementGateTests.swift | 52 ++ .../DashboardTilePolicyTests.swift | 9 + .../Execution/ExecutionGateTests.swift | 59 ++ .../SQL/CatalogChangeClassifierTests.swift | 6 + .../SQL/QueryClassifierHardeningTests.swift | 117 ++++ .../Utilities/SQL/SQLFileParserTests.swift | 38 ++ .../SQLLexicalFeatureMappingTests.swift | 5 + .../probes/check-mysql-executable-comments.sh | 187 ++++++ 27 files changed, 1629 insertions(+), 134 deletions(-) create mode 100644 apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLExecutableCommentReadings.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLExecutableCommentReadingsTests.swift create mode 100755 apple/scripts/probes/check-mysql-executable-comments.sh diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift b/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift index f545670898..cc2d9b5894 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift @@ -13,7 +13,8 @@ import TableProSQLGrammar /// /// The batch is split into statements the way its engine lexes it, by the grammar the Mac reads /// too, and under every reading the engine could be using: the run splits by one reading, and a -/// `DELETE` hidden behind a quote that only another reading closes still reaches the server. On an +/// `DELETE` hidden behind a quote that only another reading closes still reaches the server. Each +/// MySQL server version that skips a different set of `/*!` comments is one more reading. On an /// engine that needs no `;`, every statement that begins inside one is read as well, because SQL /// Server runs `SELECT 1` followed by `DROP TABLE t` as two statements. public enum SQLWriteClassifier { @@ -43,10 +44,13 @@ public enum SQLWriteClassifier { return verdict.tier != .read } let readings = SQLLexicalReadings.resolve(databaseTypeId: databaseType.rawValue, declared: nil, session: nil) - return readings.distinct(for: sql).contains { grammar in - SQLStatementScanner.executableStatements(in: sql, grammar: grammar).contains { statement in - SQLUnterminatedStatements.runnable(in: statement.sql, grammar: grammar).contains { runnable in - statementWrites(runnable, grammar: grammar) + guard let lexings = readings.lexings(of: sql) else { return true } + var seen: Set = [] + return lexings.contains { lexing in + SQLStatementScanner.executableStatements(in: lexing.text, grammar: lexing.grammar).contains { statement in + SQLUnterminatedStatements.runnable(in: statement.sql, grammar: lexing.grammar).contains { runnable in + seen.insert(SQLLexing(grammar: lexing.grammar, text: runnable)).inserted + && statementWrites(runnable, grammar: lexing.grammar) } } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift index bccdbb2a85..a899b93894 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift @@ -56,7 +56,15 @@ public enum SQLCatalogEffectClassifier { } } + // A `DROP` that only a server skipping a `/*!` comment runs still changes the catalog. public static func effect(of sql: String, grammar: SQLLexicalGrammar) -> SQLCatalogEffect { + guard let texts = SQLExecutableCommentReadings.texts(of: sql, grammar: grammar) else { return .opaque } + return texts.reduce(SQLCatalogEffect.none) { partial, text in + partial.union(textEffect(of: text, grammar: grammar)) + } + } + + private static func textEffect(of sql: String, grammar: SQLLexicalGrammar) -> SQLCatalogEffect { let trimmed = StatementBlank.trimming(SQLLeadingComments.stripping(sql)) guard !trimmed.isEmpty else { return .none } if SQLProceduralBlock.runs(trimmed, grammar: grammar) { @@ -201,9 +209,9 @@ public enum SQLCatalogEffectClassifier { ] } -// A MySQL executable comment (/*! and /*M!) is code, so it is never skipped. +// A MySQL executable comment (/*! and /*M!) is code, and so is TiDB's /*T!, so it is never skipped. public enum SQLLeadingComments { - public static let conditionalCommentOpeners: [String] = ["/*!", "/*M!"] + public static let conditionalCommentOpeners: [String] = ["/*!", "/*M!", "/*T!"] public static let lineCommentTerminators: Set = ["\n", "\r", "\r\n"] diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCodeProjection.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCodeProjection.swift index 6c8339d59a..bd102fa19b 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCodeProjection.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCodeProjection.swift @@ -6,8 +6,8 @@ import Foundation /// a string that ended where the engine ends it always does. Each blanked UTF-16 unit becomes a space and each line /// feed stays, so an offset into the projection is an offset into the original text. public enum SQLCodeProjection { - /// - Parameter revealingExecutableComments: whether a MySQL `/*! ... */` body is kept as code. A gate reveals it - /// whatever the grammar says, because the only cost of reading an ignored comment as code is a stricter tier. + /// - Parameter revealingExecutableComments: whether a MySQL `/*! ... */` body is kept as code, as a server that runs + /// it reads it. What a server that skips it runs is ``SQLExecutableCommentReadings``'s question. public static func code( of text: String, grammar: SQLLexicalGrammar, diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLExecutableCommentReadings.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLExecutableCommentReadings.swift new file mode 100644 index 0000000000..f68a5e2d11 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLExecutableCommentReadings.swift @@ -0,0 +1,550 @@ +import Foundation + +/// The texts a MySQL, MariaDB or TiDB server may run from one text, the text itself first. +/// +/// A `/*!` body runs as code when it has no version or one not above the server's. A higher version is skipped as bytes +/// to the first `*/` past one nested `/* */`, and MariaDB also skips `/*!50700` to `/*!99999`. MySQL reads MariaDB's +/// `/*M!` and TiDB's `/*T!` as plain comments, and every server reads `/*m!` as one. A text can mix run and skipped +/// comments, so there is one text per version threshold per server family. Measured on MySQL 8.0 and 8.4, MariaDB 10.11 +/// and 11.4 and TiDB 8.5 by `scripts/probes/check-mysql-executable-comments.sh`. +public enum SQLExecutableCommentReadings { + public enum Family: Sendable, Hashable, CaseIterable { + /// MySQL before 8.1: a version is five digits, and a sixth digit is the body's. + case mySQL5 + /// MySQL 8.1 and later: a sixth digit belongs to the version when whitespace follows it. + case mySQL81 + /// MariaDB: a sixth digit always belongs to the version. + case mariaDB + /// A MariaDB Galera node, which also runs `/*!99997` (`sql_lex.cc`, not measured). + case galera + /// TiDB: `/*!` runs whatever its version, `/*T!` runs, and `/*T![feature]` reads as plain when TiDB lacks it. + case tiDB + /// OceanBase rereads a statement that fails with every `/*!` as a plain comment (`ob_parser.cpp`, not measured). + case oceanBase + } + + public struct Server: Sendable, Hashable { + public let family: Family + public let version: Int + + public init(family: Family, version: Int) { + self.family = family + self.version = version + } + } + + /// More distinct versions than a gate reads one by one: a text that sets them is read as the worst statement. + public static let maximumVersions = 32 + + /// The texts `grammar` reads, or nil when `text` sets more than ``maximumVersions`` versions. Lazy, so a gate that + /// has its answer from the first texts never rewrites the rest. + public static func texts(of text: String, grammar: SQLLexicalGrammar) -> Texts? { + // By UTF-16 unit: a Prepend scalar such as U+0600 joins the `/` after it into one Character. + let units = text as NSString + let holdsOpener = ["/*!", "/*M!", "/*T!"].contains { + units.range(of: $0, options: .literal).location != NSNotFound + } + guard grammar.contains(.executableComments), holdsOpener else { + return Texts(text: text, source: nil, grammar: grammar) + } + let source = Source(text) + guard source.distinctVersions.count <= maximumVersions else { return nil } + return Texts(text: text, source: source, grammar: grammar) + } + + public static func text(of text: String, grammar: SQLLexicalGrammar, on server: Server) -> String { + let source = Source(text) + return source.rewritten(grammar: grammar) { $0.decision(on: server, in: source.units) } + } + + /// Whether `text` holds an executable comment where `grammar` reads code. + public static func holdsExecutableComment(_ text: String, grammar: SQLLexicalGrammar) -> Bool { + guard grammar.contains(.executableComments) else { return false } + let source = text as NSString + var index = 0 + while index < source.length { + guard let span = SQLNonCodeSpan.span(at: index, in: source, grammar: grammar) else { + index += 1 + continue + } + if span.kind == .executableComment { return true } + index = max(span.end, index + 1) + } + return false + } + + public struct Texts: Sequence { + fileprivate let text: String + fileprivate let source: Source? + fileprivate let grammar: SQLLexicalGrammar + + public func makeIterator() -> Iterator { + Iterator(texts: self) + } + } + + public struct Iterator: IteratorProtocol { + private let texts: Texts + private var returnedText = false + private var deciders: [Source.Decider]? + private var nextSet = 0 + private var seen: Set = [] + + fileprivate init(texts: Texts) { + self.texts = texts + } + + public mutating func next() -> String? { + guard returnedText else { + returnedText = true + return texts.text + } + guard let source = texts.source else { return nil } + if deciders == nil { + deciders = source.deciders(grammar: texts.grammar) + seen.insert(texts.text) + } + guard let deciders else { return nil } + while nextSet < deciders.count { + let decide = deciders[nextSet] + nextSet += 1 + let rewritten = source.rewritten(grammar: texts.grammar, deciding: decide) + if seen.insert(rewritten).inserted { return rewritten } + } + return nil + } + } +} + +extension SQLExecutableCommentReadings { + enum Decision: Hashable { + case runs(openerLength: Int) + case skipped + case plain + } + + fileprivate enum Marker: Hashable { + case bang + case mariaDB + case tiDB + } + + // What an opener's decision depends on: its marker, its length, and the digits a server can read as its version. + fileprivate struct OpenerKind: Hashable { + let marker: Marker + let markerLength: Int + let digits: Int + let followedByWhitespace: Bool + let leadingDigits: Int + + init(_ opener: Opener, in units: [UInt16]) { + marker = opener.marker + markerLength = opener.markerLength + digits = min(opener.digits, 7) + followedByWhitespace = opener.followedByWhitespace + let start = opener.offset + opener.markerLength + leadingDigits = units[start..<(start + min(opener.digits, 6))] + .reduce(0) { $0 * 10 + Int($1 - Source.digitZero) } + } + } + + fileprivate struct Opener { + let offset: Int + let marker: Marker + let markerLength: Int + let digits: Int + let followedByWhitespace: Bool + + var isMariaDBMarker: Bool { + marker == .mariaDB + } + + func version(for family: Family, in units: [UInt16]) -> (value: Int, length: Int)? { + guard digits >= 5, marker != .tiDB else { return nil } + let length: Int + switch family { + case .mySQL5, .tiDB: + length = 5 + case .mySQL81: + length = digits == 6 && followedByWhitespace ? 6 : 5 + case .mariaDB, .galera: + length = min(digits, 6) + case .oceanBase: + return nil + } + let start = offset + markerLength + let value = units[start..<(start + length)].reduce(0) { $0 * 10 + Int($1 - Source.digitZero) } + return (value, length) + } + + func decision(on server: Server, in units: [UInt16]) -> Decision { + switch (marker, server.family) { + case (_, .oceanBase), (.tiDB, .mySQL5), (.tiDB, .mySQL81), (.tiDB, .mariaDB), (.tiDB, .galera), + (.mariaDB, .mySQL5), (.mariaDB, .mySQL81), (.mariaDB, .tiDB): + return .plain + case (.tiDB, .tiDB): + return .runs(openerLength: markerLength) + default: + break + } + guard let version = version(for: server.family, in: units) else { + return .runs(openerLength: markerLength) + } + let runs: Bool + switch server.family { + case .mySQL5, .mySQL81: + runs = version.value <= server.version + case .tiDB: + runs = true + case .mariaDB, .galera: + let mySQLOnly = !isMariaDBMarker && (50_700...99_999).contains(version.value) + runs = (version.value <= server.version && !mySQLOnly) + || (server.family == .galera && !isMariaDBMarker && version.value == 99_997) + case .oceanBase: + runs = false + } + return runs ? .runs(openerLength: markerLength + version.length) : .skipped + } + } + + fileprivate struct Source { + static let digitZero = UInt16(UnicodeScalar("0").value) + static let digitNine = UInt16(UnicodeScalar("9").value) + static let exclamationMark = UInt16(UnicodeScalar("!").value) + static let capitalM = UInt16(UnicodeScalar("M").value) + static let capitalT = UInt16(UnicodeScalar("T").value) + static let openBracket = UInt16(UnicodeScalar("[").value) + static let closeBracket = UInt16(UnicodeScalar("]").value) + static let comma = UInt16(UnicodeScalar(",").value) + // MySQL 8.1 is the first version to read a sixth digit. + static let firstSixDigitMySQL = 80_100 + + let text: NSString + let units: [UInt16] + let openers: [Int: Opener] + let distinctVersions: Set + + init(_ string: String) { + text = string as NSString + units = Array(string.utf16) + openers = Self.openers(in: units) + var versions: Set = [] + for opener in openers.values { + for family in Family.allCases { + if let version = opener.version(for: family, in: units) { + versions.insert(version.value) + } + } + } + distinctVersions = versions + } + + static func openers(in units: [UInt16]) -> [Int: Opener] { + var found: [Int: Opener] = [:] + var index = 0 + while index + 2 < units.count { + defer { index += 1 } + guard units[index] == SqlLexer.slash, units[index + 1] == SqlLexer.star else { continue } + var cursor = index + 2 + let marker: Marker + switch units[cursor] { + case capitalM: marker = .mariaDB + case capitalT: marker = .tiDB + default: marker = .bang + } + if marker != .bang { cursor += 1 } + guard cursor < units.count, units[cursor] == exclamationMark else { continue } + cursor += 1 + // TiDB reads the features a `/*T![a,b]` names as part of the opener. + if marker == .tiDB, cursor < units.count, units[cursor] == openBracket, + let close = featureListEnd(in: units, from: cursor + 1) { + cursor = close + 1 + } + let markerLength = cursor - index + var digits = 0 + while cursor + digits < units.count, isDigit(units[cursor + digits]) { + digits += 1 + } + let after = cursor + digits + found[index] = Opener( + offset: index, + marker: marker, + markerLength: markerLength, + digits: digits, + followedByWhitespace: after < units.count && isWhitespace(units[after]) + ) + } + return found + } + + // A feature list holds names and commas, so the search stops at anything else and stays linear. + static func featureListEnd(in units: [UInt16], from start: Int) -> Int? { + var index = start + while index < units.count { + let unit = units[index] + if unit == closeBracket { return index } + let continues = SqlDollarQuote.isIdentifierPart(unit) || unit == comma || unit == SqlLexer.space + guard continues else { return nil } + index += 1 + } + return nil + } + + static func isDigit(_ unit: UInt16) -> Bool { + unit >= digitZero && unit <= digitNine + } + + // my_isspace: space, tab, line feed, vertical tab, form feed, carriage return. + static func isWhitespace(_ unit: UInt16) -> Bool { + unit == 0x20 || (0x09...0x0D).contains(unit) + } + + // Two servers that decide every kind of opener alike run the same text. Openers of one kind decide alike, so + // a text with thousands of them costs one decision per kind. + func distinctServers() -> [Server] { + guard !openers.isEmpty else { return [] } + var kinds: [OpenerKind: Opener] = [:] + for opener in openers.values { + kinds[OpenerKind(opener, in: units)] = opener + } + let representatives = Array(kinds.values) + var servers: [Server] = [] + var signatures: Set<[Decision]> = [] + for family in Family.allCases { + for version in thresholds(for: family, openers: representatives) { + let server = Server(family: family, version: version) + let signature = representatives.map { $0.decision(on: server, in: units) } + if signatures.insert(signature).inserted { + servers.append(server) + } + } + } + return servers + } + + private func thresholds(for family: Family, openers: [Opener]) -> [Int] { + let versions = Set(openers.compactMap { $0.version(for: family, in: units)?.value }) + switch family { + case .mySQL5: + return [0] + versions.filter { $0 < Self.firstSixDigitMySQL }.sorted() + case .mySQL81: + return [Self.firstSixDigitMySQL] + versions.filter { $0 > Self.firstSixDigitMySQL }.sorted() + case .mariaDB: + return [0] + versions.sorted() + case .galera: + guard versions.contains(99_997) else { return [] } + return [0] + versions.sorted() + case .tiDB: + return openers.contains { $0.marker == .tiDB } ? [0] : [] + case .oceanBase: + return [0] + } + } + + typealias Decider = (Opener) -> Decision + + func deciders(grammar: SQLLexicalGrammar) -> [Decider] { + let servers = distinctServers() + if let statementwise = statementwiseDeciders(servers: servers, grammar: grammar) { + return statementwise + } + return servers.map { server in { [units] opener in opener.decision(on: server, in: units) } } + } + + static let semicolon = UInt16(UnicodeScalar(";").value) + static let boundaryShapingFacts: SQLLexicalGrammar = [ + .unterminatedStatements, .batchSeparatorLines, .plsqlBlocks, .cqlBatches, .slashLineTerminators, + .sqlScriptBlocks, .terminatedMergeStatements, + ] + + // A dump holds a dozen versions, one text per threshold, and every text is scanned whole. When no comment can + // move a statement boundary, a statement only needs the readings of its own comments, so a few texts that each + // take one of them per statement hold every statement every server runs. That needs each comment to end at + // the same place however it is read and to hold no `;` or line break, and no statement to keep a `;` inside a + // BEGIN or a CASE. Otherwise nil, and every server's text is read. + func statementwiseDeciders(servers: [Server], grammar: SQLLexicalGrammar) -> [Decider]? { + guard let first = servers.first, grammar.isDisjoint(with: Self.boundaryShapingFacts), + text.range(of: "DELIMITER", options: .caseInsensitive).location == NSNotFound + else { return nil } + let plain = grammar.subtracting(.executableComments) + var sitesByStatement: [Int: [Int]] = [:] + var statement = 0 + var index = 0 + while index < units.count { + let unit = units[index] + if unit == SqlLexer.slash, index + 1 < units.count, units[index + 1] == SqlLexer.star { + if let opener = openers[index] { + guard let end = stableEnd(of: opener, servers: servers, plain: plain, grammar: grammar) else { + return nil + } + sitesByStatement[statement, default: []].append(index) + index = end + } else { + index = plainCommentEnd(at: index, grammar: grammar) + } + continue + } + guard let next = codeStep(at: index, plain: plain) else { return nil } + if unit == Self.semicolon { statement += 1 } + index = next + } + var readings: [Int: [[Decision]]] = [:] + for (statement, sites) in sitesByStatement { + var seen: Set<[Decision]> = [] + for server in servers { + let vector = sites.map { openers[$0]?.decision(on: server, in: units) ?? .skipped } + if seen.insert(vector).inserted { + readings[statement, default: []].append(vector) + } + } + } + let count = readings.values.map(\.count).max() ?? 0 + return (0.. Int? { + var end: Int? + for decision in Set(servers.map { opener.decision(on: $0, in: units) }) { + let candidate: Int? + switch decision { + case .runs(let openerLength): + candidate = runEnd(from: opener.offset + openerLength, plain: plain, grammar: grammar) + case .skipped: + candidate = consumeComment(from: opener.offset + 2, nestingAllowed: 1) + case .plain: + candidate = consumeComment(from: opener.offset + 2, nestingAllowed: 0) + } + guard let candidate, end == nil || end == candidate else { return nil } + end = candidate + } + return end + } + + // Where a run body's closing `*/` ends it, or nil when the body holds what could move a statement boundary. + private func runEnd(from start: Int, plain: SQLLexicalGrammar, grammar: SQLLexicalGrammar) -> Int? { + var index = start + while index < units.count { + let unit = units[index] + let next: UInt16 = index + 1 < units.count ? units[index + 1] : 0 + if unit == SqlLexer.star, next == SqlLexer.slash { return index + 2 } + if unit == Self.semicolon || unit == SqlLexer.newline { return nil } + if unit == SqlLexer.slash, next == SqlLexer.star { + guard openers[index] == nil else { return nil } + index = plainCommentEnd(at: index, grammar: grammar) + continue + } + guard let step = codeStep(at: index, plain: plain) else { return nil } + index = step + } + return nil + } + + // Past one literal, line comment, word or unit of code; nil at a word that can open a block a `;` stays in. + // Words are read as the statement scanner reads them, so `1BEGIN` holds the BEGIN it sees. + private func codeStep(at index: Int, plain: SQLLexicalGrammar) -> Int? { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: plain) { + return max(span.end, index + 1) + } + guard SqlBlockStructure.startsWord(text, at: index, length: units.count, grammar: plain) else { + return index + 1 + } + let word = SqlBlockStructure.readKeyword(text, at: index, length: units.count, grammar: plain) + return word.text == "BEGIN" || word.text == "CASE" ? nil : word.end + } + + private func plainCommentEnd(at index: Int, grammar: SQLLexicalGrammar) -> Int { + grammar.contains(.nestedBlockComments) + ? SqlLexer.skipNestedBlockComment(text, from: index, length: units.count).next + : SqlLexer.skipBlockComment(text, from: index, length: units.count).next + } + + func rewritten(grammar: SQLLexicalGrammar, deciding decide: (Opener) -> Decision) -> String { + guard !openers.isEmpty else { return text as String } + let plain = grammar.subtracting(.executableComments) + let length = units.count + var output = units + var index = 0 + var insideRunComment = false + while index < length { + let unit = units[index] + let next: UInt16 = index + 1 < length ? units[index + 1] : 0 + if insideRunComment, unit == SqlLexer.star, next == SqlLexer.slash { + blank(&output, from: index, to: index + 2) + insideRunComment = false + index += 2 + continue + } + if unit == SqlLexer.slash, next == SqlLexer.star { + switch openers[index].map(decide) { + case .runs(let openerLength): + blank(&output, from: index, to: index + openerLength) + insideRunComment = true + index += openerLength + case .skipped: + let end = consumeComment(from: index + 2, nestingAllowed: 1) + blank(&output, from: index, to: end) + index = end + case .plain: + let end = consumeComment(from: index + 2, nestingAllowed: 0) + blank(&output, from: index, to: end) + index = end + case nil: + index = plainCommentEnd(at: index, grammar: grammar) + } + continue + } + // A plain comment stays: every reader lexes it as the server does. + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: plain) { + index = max(span.end, index + 1) + continue + } + index += 1 + } + return String(utf16CodeUnits: output, count: length) + } + + // MySQL's consume_comment: the first `*/` ends the comment, after `nestingAllowed` levels of `/* */` inside it. + private func consumeComment(from start: Int, nestingAllowed: Int) -> Int { + var cursor = start + while cursor < units.count { + let unit = units[cursor] + cursor += 1 + let next: UInt16 = cursor < units.count ? units[cursor] : 0 + if nestingAllowed > 0, unit == SqlLexer.slash, next == SqlLexer.star { + cursor = consumeComment(from: cursor + 1, nestingAllowed: nestingAllowed - 1) + continue + } + if unit == SqlLexer.star, next == SqlLexer.slash { + return cursor + 1 + } + } + return units.count + } + + // A comment is a token boundary to the server, so `--/*!99999 x */` is two minus signs. Blanking it whole + // would turn the dashes into a `-- ` comment that hides the rest of the line, so its first unit stays. + private func blank(_ output: inout [UInt16], from start: Int, to end: Int) { + let followsDashes = start >= 2 && units[start - 1] == SqlLexer.dash && units[start - 2] == SqlLexer.dash + let first = followsDashes ? start + 1 : start + guard first < min(end, output.count) else { return } + for offset in first.. AnySequence? { + var readings: [(grammar: SQLLexicalGrammar, texts: SQLExecutableCommentReadings.Texts)] = [] + for grammar in distinct(for: text) { + guard let texts = SQLExecutableCommentReadings.texts(of: text, grammar: grammar) else { return nil } + readings.append((grammar, texts)) + } + return AnySequence(readings.lazy.flatMap { reading in + reading.texts.lazy.map { SQLLexing(grammar: reading.grammar, text: $0) } + }) + } + private static func baseReadings(databaseTypeId: String, declared: SQLLexicalGrammar?) -> SQLLexicalReadings { if let profile = SQLLexicalProfile.curated(forDatabaseTypeId: databaseTypeId) { return SQLLexicalReadings(execution: profile.grammar, all: profile.readings) @@ -89,6 +103,17 @@ public struct SQLLexicalReadings: Sendable, Hashable { } } +/// A text read by one grammar. +public struct SQLLexing: Sendable, Hashable { + public let grammar: SQLLexicalGrammar + public let text: String + + public init(grammar: SQLLexicalGrammar, text: String) { + self.grammar = grammar + self.text = text + } +} + /// Which lexical facts a text could possibly exercise, found in one pass over its UTF-16 units. enum SQLLexicalRelevance { private static let openBracket = UInt16(UnicodeScalar("[").value) diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift index 642f6d969c..7de400b5b6 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift @@ -139,19 +139,74 @@ public enum SQLNonCodeSpan { guard SqlLexer.startsBlockComment(text, at: index, length: length) else { return nil } if grammar.contains(.executableComments), let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: length) { - let span = SqlLexer.skipExecutableComment( - text, - from: index, - openerLength: opener, - length: length, - backslashEscapes: grammar.backslashEscapes(inQuote:) - ) - return closedSpan(.executableComment, at: index, span, closerLength: 2) + return executableCommentSpan(at: index, openerLength: opener, in: text, length: length, grammar: grammar) } - let span = grammar.contains(.nestedBlockComments) + let comment = plainBlockComment(at: index, in: text, length: length, grammar: grammar) + return closedSpan(.blockComment, at: index, comment, closerLength: 2) + } + + private static func plainBlockComment( + at index: Int, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> SqlLexer.Span { + grammar.contains(.nestedBlockComments) ? SqlLexer.skipNestedBlockComment(text, from: index, length: length) : SqlLexer.skipBlockComment(text, from: index, length: length) - return closedSpan(.blockComment, at: index, span, closerLength: 2) + } + + // The server lexes a run body as code, comments included, and a nested opener opens no second level: the first + // `*/` read as code closes the comment. Measured on MySQL 8.0 and 8.4 and MariaDB 10.11 and 11.4. + private static func executableCommentSpan( + at index: Int, + openerLength: Int, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> Span { + var cursor = index + openerLength + var newlines = 0 + while cursor < length { + let character = text.character(at: cursor) + if character == SqlLexer.star, cursor + 1 < length, text.character(at: cursor + 1) == SqlLexer.slash { + return Span( + kind: .executableComment, + start: index, + end: cursor + 2, + contentEnd: cursor, + newlines: newlines, + isTerminated: true + ) + } + if SqlLexer.startsBlockComment(text, at: cursor, length: length) { + if let nested = SqlLexer.executableCommentOpenerLength(text, at: cursor, length: length) { + cursor += nested + continue + } + let comment = plainBlockComment(at: cursor, in: text, length: length, grammar: grammar) + newlines += comment.newlines + cursor = comment.next + continue + } + if let inner = span(at: cursor, in: text, grammar: grammar) { + newlines += inner.newlines + cursor = max(inner.end, cursor + 1) + continue + } + if character == SqlLexer.newline { + newlines += 1 + } + cursor += 1 + } + return Span( + kind: .executableComment, + start: index, + end: length, + contentEnd: length, + newlines: newlines, + isTerminated: false + ) } private static func startsLineComment( diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift index 9d3ea485ca..4dfc3db915 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift @@ -254,11 +254,12 @@ public enum SqlLexer { return cursor } - /// The length of a MySQL `/*!NNNNN` or MariaDB `/*M!NNNNN` opener at `offset`, or nil when none starts there. + /// The length of a MySQL `/*!NNNNN` or MariaDB `/*M!NNNNN` opener at `offset`, or nil when none starts there. Both + /// servers read a lowercase `/*m!` as a plain comment. public static func executableCommentOpenerLength(_ text: NSString, at offset: Int, length: Int) -> Int? { guard startsBlockComment(text, at: offset, length: length) else { return nil } var cursor = offset + 2 - if cursor < length, text.character(at: cursor) == capitalM || text.character(at: cursor) == smallM { + if cursor < length, text.character(at: cursor) == capitalM { cursor += 1 } guard cursor < length, text.character(at: cursor) == exclamationMark else { return nil } @@ -365,43 +366,6 @@ public enum SqlLexer { return Span(next: length, newlines: 0, isClosed: false) } - /// Runs past a MySQL executable comment whose opener is `openerLength` long. The server lexes the body as SQL, so - /// a `*/` inside a quoted string does not close it, measured on 8.4 and 11.8 with `/*! , '*/' */`. - public static func skipExecutableComment( - _ text: NSString, - from offset: Int, - openerLength: Int, - length: Int, - backslashEscapes: (UInt16) -> Bool - ) -> Span { - var cursor = offset + openerLength - var newlines = 0 - while cursor < length { - let character = text.character(at: cursor) - if character == newline { - newlines += 1 - } - if character == star, cursor + 1 < length, text.character(at: cursor + 1) == slash { - return Span(next: cursor + 2, newlines: newlines) - } - if character == singleQuote || character == doubleQuote || character == backtick { - let quoted = skipQuotedString( - text, - from: cursor, - quote: character, - length: length, - backslashEscapes: backslashEscapes(character) - ) - newlines += quoted.newlines - cursor = quoted.next - continue - } - cursor += 1 - } - return Span(next: length, newlines: newlines, isClosed: false) - } - - private static let smallM = UInt16(UnicodeScalar("m").value) private static let capitalM = UInt16(UnicodeScalar("M").value) private static let digitZero = UInt16(UnicodeScalar("0").value) private static let digitNine = UInt16(UnicodeScalar("9").value) diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift index fb8276aefb..5b2a5bac77 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift @@ -77,6 +77,23 @@ struct SQLStatementSplittingTests { ) } + @Test("A run version-gated comment lexes its body as code, and a lowercase /*m! is a plain comment") + func executableCommentBodyIsCode() { + let mySQL: SQLLexicalFeatures = [.executableComments, .hashLineComments, .dashCommentsNeedWhitespace] + #expect( + SQLStatementSplitting.statements(in: "/*! /* ' */ SELECT 1 */; SELECT 2", lexicalFeatures: mySQL) + == ["/*! /* ' */ SELECT 1 */", "SELECT 2"] + ) + #expect( + SQLStatementSplitting.statements(in: "/*! SELECT 1 # '\n */; SELECT 2", lexicalFeatures: mySQL) + == ["/*! SELECT 1 # '\n */", "SELECT 2"] + ) + #expect( + SQLStatementSplitting.statements(in: "/*m! note */ CREATE TEMPORARY TABLE t (a INT)", lexicalFeatures: mySQL) + == ["CREATE TEMPORARY TABLE t (a INT)"] + ) + } + /// Keeping it must not make it read as a transaction statement here. The keyword is no longer /// the first word of the statement, and closing already requires the whole statement to be /// one. `MySQLSessionFootprint` reads the body itself, because MySQL does run it; every other diff --git a/apple/Packages/TableProCore/Tests/TableProQueryTests/SQLWriteClassifierTests.swift b/apple/Packages/TableProCore/Tests/TableProQueryTests/SQLWriteClassifierTests.swift index c285e9fbb0..37588f59a4 100644 --- a/apple/Packages/TableProCore/Tests/TableProQueryTests/SQLWriteClassifierTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQueryTests/SQLWriteClassifierTests.swift @@ -271,4 +271,22 @@ struct SQLWriteClassifierTests { func executableCommentIsAWrite() { #expect(isWrite("/*!40101 DELETE FROM t */", .mysql)) } + + @Test( + "a write a server runs past a comment it skips or reads as plain is a write", + arguments: [ + "WITH c AS (SELECT 1) /*m! ' */ DELETE FROM orders -- '", + "WITH c AS (SELECT 1) /*!99999 ' */ DELETE FROM orders -- '", + "SELECT * FROM orders /*m! ' */ INTO OUTFILE '/tmp/x' -- '", + "SELECT * FROM orders /*!99999 ' */ INTO OUTFILE '/tmp/x' -- '", + "SELECT /*!50000 \"*/\", */ 1; /*!99999 ' */ DROP TABLE orders -- '", + "SELECT 1--/*!99999 ' */1 INTO OUTFILE '/tmp/x' -- '", + "/*T! DELETE FROM orders */", + "SELECT 1 a\u{600}/*!99999 ' */ INTO OUTFILE '/tmp/x' -- '", + ], + [DatabaseType.mysql, .mariadb] + ) + func hiddenWriteIsAWrite(sql: String, databaseType: DatabaseType) { + #expect(isWrite(sql, databaseType)) + } } diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLExecutableCommentReadingsTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLExecutableCommentReadingsTests.swift new file mode 100644 index 0000000000..ab83ec6c44 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLExecutableCommentReadingsTests.swift @@ -0,0 +1,180 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct SQLExecutableCommentReadingsTests { + private static let mySQL = SQLLexicalReadings.resolve( + databaseTypeId: "MySQL", declared: nil, session: nil + ).execution + + private static let servers: [String: SQLExecutableCommentReadings.Server] = [ + "MySQL 8.0.46": .init(family: .mySQL5, version: 80_046), + "MySQL 8.4.11": .init(family: .mySQL81, version: 80_411), + "MariaDB 10.11.19": .init(family: .mariaDB, version: 101_119), + "MariaDB 11.4.13": .init(family: .mariaDB, version: 110_413), + ] + + private static func texts(_ sql: String, _ grammar: SQLLexicalGrammar = mySQL) -> [String] { + SQLExecutableCommentReadings.texts(of: sql, grammar: grammar).map { Array($0) } ?? [] + } + + private static func dropsTable(_ text: String) -> Bool { + SQLStatementScanner.executableStatements(in: text, grammar: mySQL).contains { statement in + let code = SQLCodeProjection.code(of: statement.sql, grammar: mySQL) + return code.range(of: #"DROP\s+TABLE\s+t\b"#, options: .regularExpression) != nil + } + } + + private static let every: Set = ["MySQL 8.0.46", "MySQL 8.4.11", "MariaDB 10.11.19", "MariaDB 11.4.13"] + + // Each text was sent as one query to the four servers; the second value names the ones that dropped `t`. + static let measured: [(String, Set)] = [ + ("/*!99999 ' */ DROP TABLE t -- '", every), + ("/*!50700 ' */ DROP TABLE t -- '", ["MariaDB 10.11.19", "MariaDB 11.4.13"]), + ("/*!40101 ' */ DROP TABLE t -- '", []), + ("/*!100000 ' */ DROP TABLE t -- '", ["MySQL 8.4.11"]), + ("/*!800000 ' */ DROP TABLE t -- '", ["MySQL 8.4.11", "MariaDB 10.11.19", "MariaDB 11.4.13"]), + ("/*!800000' */ DROP TABLE t -- '", ["MariaDB 10.11.19", "MariaDB 11.4.13"]), + ("/*!8000000 ' */ DROP TABLE t -- '", ["MariaDB 10.11.19", "MariaDB 11.4.13"]), + ("/*!80047 ' */ DROP TABLE t -- '", ["MySQL 8.0.46", "MariaDB 10.11.19", "MariaDB 11.4.13"]), + ("/*!DROP TABLE t */", every), + ("/*!99999 /* */ ' */ DROP TABLE t -- '", every), + ("/*!99999 /* /* */ */ ' */ DROP TABLE t -- '", []), + ("/*!99999 # ' */ DROP TABLE t -- '", every), + ("/*!99999 ; */ DROP TABLE t", every), + ("/*M! ' */ DROP TABLE t -- '", ["MySQL 8.0.46", "MySQL 8.4.11"]), + ("/*M!999999 ' */ DROP TABLE t -- '", every), + ("/*M!99999 ' */ DROP TABLE t -- '", ["MySQL 8.0.46", "MySQL 8.4.11"]), + ("/*M! /* */ ' */ DROP TABLE t -- '", []), + ("/*m! ' */ DROP TABLE t -- '", every), + ("/*! /* ' */ DROP TABLE t */", every), + ("/*! DROP TABLE t # */ '\n */", every), + ("/*! DROP TABLE /*! t */", every), + ("SELECT /*!50000 \"*/\", */ 1; /*!99999 ' */ DROP TABLE t -- '", every), + ("SELECT /*!80000 \"*/\", */ 1; /*!80100 ' */ DROP TABLE t -- '", ["MySQL 8.0.46"]), + ("SELECT 1--/*!99999 ' */1; DROP TABLE t -- '", every), + ] + + @Test("Each server's text drops the table exactly when that server did", arguments: measured) + func eachServerRunsWhatItRan(sql: String, droppedOn: Set) { + for (name, server) in Self.servers { + let text = SQLExecutableCommentReadings.text(of: sql, grammar: Self.mySQL, on: server) + #expect((text as NSString).length == (sql as NSString).length) + #expect(Self.dropsTable(text) == droppedOn.contains(name), "\(name): \(sql)") + } + } + + @Test("A text a server drops a table from has a reading that drops it", arguments: measured) + func everyDropHasAReading(sql: String, droppedOn: Set) { + let texts = Self.texts(sql) + #expect(texts.first == sql) + if !droppedOn.isEmpty { + #expect(texts.contains(where: Self.dropsTable), "\(sql)") + } + } + + private static func statements(_ text: String) -> Set { + Set(SQLStatementScanner.executableStatements(in: text, grammar: mySQL).map(\.sql)) + } + + @Test( + "Every statement any server runs is one the texts hold", + arguments: measured.map(\.0) + [ + "/*!50000 DELETE FROM t */ /*!99999 WHERE 1=0 */", + "/*!40101 SET NAMES utf8 */; /*!80016 DROP TABLE t */; SELECT /*!40001 SQL_NO_CACHE */ 1", + "/*M!100301 DELETE FROM t */; /*!50700 DROP TABLE t */ ; /*!800000 TRUNCATE t */", + "/*!40000 ALTER TABLE `t` DISABLE KEYS */;\nINSERT INTO t VALUES (1, 'case');\n/*!40000 ALTER TABLE `t` ENABLE KEYS */;", + "CREATE PROCEDURE p() 1BEGIN /*!50000 DELETE FROM t */; /*!99999 DROP TABLE t */; END", + "/*T! DROP TABLE t */; /*T![clustered_index] SELECT 1 */; SELECT 1--/*!99999 x */1", + "CREATE PROCEDURE p() \u{E9}BEGIN /*!50000 DELETE FROM t */; /*!99999 DROP TABLE t */; END", + ] + ) + func textsHoldEveryServersStatements(sql: String) { + let held = Self.texts(sql).reduce(into: Set()) { + $0.formUnion(Self.statements($1)) + } + let versions = [0, 40_000, 40_001, 40_101, 50_000, 50_700, 80_016, 80_046, 80_100, 80_411, 99_999, 100_301] + + [101_119, 110_413, 999_999] + for family in SQLExecutableCommentReadings.Family.allCases { + for version in versions { + let server = SQLExecutableCommentReadings.Server(family: family, version: version) + let ran = Self.statements(SQLExecutableCommentReadings.text(of: sql, grammar: Self.mySQL, on: server)) + #expect(ran.isSubset(of: held), "\(family) \(version): \(sql)") + } + } + } + + @Test("A mixed text is read under every threshold its versions set") + func mixedThresholdsAreEachRead() { + let sql = "SELECT /*!50000 \"*/\", */ 1; /*!99999 ' */ DROP TABLE t -- '" + let texts = Self.texts(sql) + #expect(texts.contains(where: Self.dropsTable)) + #expect(!Self.dropsTable(SQLExecutableCommentReadings.text( + of: sql, grammar: Self.mySQL, on: .init(family: .mySQL5, version: 0) + ))) + #expect(!Self.dropsTable(SQLExecutableCommentReadings.text( + of: sql, grammar: Self.mySQL, on: .init(family: .mySQL81, version: 99_999) + ))) + } + + // Measured on TiDB 8.5.1: each of these dropped `t`. + @Test("A DROP TiDB runs has a reading that shows it", arguments: [ + "/*T! DROP TABLE t */", + "/*T![clustered_index] DROP TABLE t */", + "/*T![no_such_feature] ' */ DROP TABLE t -- '", + "/*!99999 DROP TABLE t */", + "/*M! ' */ DROP TABLE t -- '", + ]) + func tiDBDropsHaveAReading(sql: String) { + #expect(Self.texts(sql).contains(where: Self.dropsTable)) + } + + @Test("TiDB runs /*T! and every /*!, whatever its version") + func tiDBRunsItsOwnComments() { + let tiDB = SQLExecutableCommentReadings.Server(family: .tiDB, version: 0) + #expect(Self.dropsTable(SQLExecutableCommentReadings.text(of: "/*T! DROP TABLE t */", grammar: Self.mySQL, on: tiDB))) + #expect(!Self.dropsTable(SQLExecutableCommentReadings.text( + of: "/*!99999 ' */ DROP TABLE t -- '", grammar: Self.mySQL, on: tiDB + ))) + } + + @Test("A Galera node runs /*!99997, which every other MariaDB skips") + func galeraRunsItsOwnVersion() { + let sql = "/*!99997 DROP TABLE t */" + let mariaDB = SQLExecutableCommentReadings.Server(family: .mariaDB, version: 110_413) + let galera = SQLExecutableCommentReadings.Server(family: .galera, version: 110_413) + #expect(!Self.dropsTable(SQLExecutableCommentReadings.text(of: sql, grammar: Self.mySQL, on: mariaDB))) + #expect(Self.dropsTable(SQLExecutableCommentReadings.text(of: sql, grammar: Self.mySQL, on: galera))) + } + + @Test( + "A text with no executable comment, or a grammar that reads none, is read once", + arguments: ["SELECT 1", "SELECT '/*!99999 x */'", "/*m! x */ SELECT 1", "SELECT 1 -- /*!99999"] + ) + func plainTextIsReadOnce(sql: String) { + #expect(Self.texts(sql) == [sql]) + #expect(Self.texts(sql, .ansi) == [sql]) + } + + @Test("A text that sets more versions than a gate reads has no texts, so the gate answers its worst case") + func tooManyVersionsAreNotRead() { + let many = (0..<40).map { "/*!\(50_000 + $0) x */" }.joined(separator: " ") + let quoted = (0..<40).map { "'/*!\(50_000 + $0) x */'" }.joined(separator: ", ") + #expect(SQLExecutableCommentReadings.texts(of: "SELECT 1 \(many)", grammar: Self.mySQL) == nil) + #expect(SQLExecutableCommentReadings.texts(of: "SELECT \(quoted)", grammar: Self.mySQL) == nil) + let few = (0.. Bool { + SQLExecutableCommentReadings.holdsExecutableComment(sql, grammar: grammar) + } + #expect(holds("SELECT /*!40001 SQL_NO_CACHE */ 1")) + #expect(holds("/*M!100301 SET @x = 1 */")) + #expect(!holds("/*m! SET @x = 1 */")) + #expect(!holds("SELECT '/*!40101 x */'")) + #expect(!holds("SELECT /*!40101 x */", .ansi)) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift index db703df0da..57b7cd1ce1 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift @@ -110,6 +110,32 @@ struct SQLNonCodeSpanTests { #expect(end(text, .ansi) == 14) } + @Test( + "A run executable comment lexes its body as code, comments included", + arguments: [ + ("/*! /* ' */ DROP TABLE t */ x", 27), + ("/*! DROP TABLE t # */ '\n */ x", 27), + ("/*! DROP TABLE t -- */ '\n */ x", 28), + ("/*! DROP TABLE /*! t */ x", 23), + ("/*!40101 SET @a = '*/' */ x", 25), + ] + ) + func executableCommentBodyIsCode(text: String, end: Int) { + let grammar: SQLLexicalGrammar = [.executableComments, .hashLineComments, .dashCommentsNeedWhitespace] + let span = SQLNonCodeSpan.span(at: 0, in: text as NSString, grammar: grammar) + #expect(span?.kind == .executableComment) + #expect(span?.end == end) + #expect(span?.isTerminated == true) + } + + @Test("A lowercase /*m! is a plain comment, as MySQL and MariaDB read it") + func lowercaseMarkerIsAPlainComment() { + let text = "/*m! ' */ DROP TABLE t -- '" as NSString + let span = SQLNonCodeSpan.span(at: 0, in: text, grammar: [.executableComments]) + #expect(span?.kind == .blockComment) + #expect(span?.end == 9) + } + @Test("A span the text ends inside is reported unterminated") func unterminatedSpans() { let grammar: SQLLexicalGrammar = [.taggedDollarQuotes, .bracketQuotedIdentifiers] diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift b/apple/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift index d8341ee733..8f55556a51 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift @@ -102,13 +102,50 @@ struct MySQLSessionFootprint: Equatable { /// `NO_BACKSLASH_ESCAPES` closes early is split where the server splits it. mutating func observe(_ sql: String, lexicalFeatures: SQLLexicalFeatures) { for statement in SQLStatementSplitting.statements(in: sql, lexicalFeatures: lexicalFeatures) { - let body = Self.executableBody(of: statement) + let readings = Self.executableBodies(of: statement).map { body in + var reading = self + reading.observeReading(body, lexicalFeatures: lexicalFeatures) + return reading + } + self = readings.dropFirst().reduce(readings[0]) { $0.holding($1) } + } + } + + // A reading can hold statements the split above kept inside the comment, as one whose `#` hid a `*/` does when + // the server skips it, so each is split again. + private mutating func observeReading(_ body: String, lexicalFeatures: SQLLexicalFeatures) { + let statements = SQLStatementSplitting.statements(in: body, lexicalFeatures: lexicalFeatures) + guard statements.count > 1 else { observeTransaction(body, lexicalFeatures: lexicalFeatures) observeStatement(body) observeTemporaryTableNames(body) + return + } + for statement in statements { + observe(statement, lexicalFeatures: lexicalFeatures) } } + // Holds what either holds, so a reading that clears a flag never clears it for a server that ran the other one. + private func holding(_ other: MySQLSessionFootprint) -> MySQLSessionFootprint { + var merged = self + merged.hasOpenTransaction = hasOpenTransaction || other.hasOpenTransaction + merged.hasTemporaryTables = hasTemporaryTables || other.hasTemporaryTables + merged.hasUserVariables = hasUserVariables || other.hasUserVariables + merged.hasPreparedStatements = hasPreparedStatements || other.hasPreparedStatements + merged.hasAdvisoryLocks = hasAdvisoryLocks || other.hasAdvisoryLocks + merged.hasLockedTables = hasLockedTables || other.hasLockedTables + merged.hasOpenHandlers = hasOpenHandlers || other.hasOpenHandlers + merged.hasSessionSettings = hasSessionSettings || other.hasSessionSettings + merged.hasChangedDatabase = hasChangedDatabase || other.hasChangedDatabase + merged.ranOpaqueRoutine = ranOpaqueRoutine || other.ranOpaqueRoutine + merged.heldSessionScopes = max(heldSessionScopes, other.heldSessionScopes) + for name in other.temporaryTableNames where !merged.temporaryTableNames.contains(name) { + merged.temporaryTableNames.append(name) + } + return merged + } + /// The server's own answer, taken from the status flags in its last reply, which is exact /// where reading the statements is a guess. Measured on MySQL 8.4.11: `SET autocommit = 0` /// followed by a plain `SELECT` reports a transaction that appears nowhere in the text, and so @@ -136,8 +173,10 @@ struct MySQLSessionFootprint: Equatable { mutating func observeFailure(of sql: String, lexicalFeatures: SQLLexicalFeatures) { let statements = SQLStatementSplitting.statements(in: sql, lexicalFeatures: lexicalFeatures) guard statements.count == 1, let failed = statements.first else { return } - let head = Self.collapsedHead(of: Self.executableBody(of: failed).uppercased()) - guard head.hasPrefix("LOCK TABLE") else { return } + let refusedLock = Self.executableBodies(of: failed).allSatisfy { body in + Self.collapsedHead(of: body.uppercased()).hasPrefix("LOCK TABLE") + } + guard refusedLock else { return } hasLockedTables = false } @@ -279,21 +318,24 @@ struct MySQLSessionFootprint: Equatable { /// because only the engine that executes the body can say what it is. /// /// Whatever follows the comment is kept, so a line that carries a note after it, or a second - /// version-gated block, is still classified by the first thing the server would run. The - /// version number itself is not checked against the server: counting a statement the server - /// is too old to run holds a connection that is in fact clean, which is the safe direction. - private static func executableBody(of statement: String) -> String { - guard statement.hasPrefix("/*!") || statement.hasPrefix("/*M!") else { return statement } - guard let close = statement.range(of: "*/") else { return statement } + /// version-gated block, is still classified by the first thing the server would run. A server + /// older than a version after 40101 skips the body, and MySQL reads `/*M!` as a plain comment, + /// so for those what follows is also read on its own. + private static func executableBodies(of statement: String) -> [String] { + let isMariaDBMarker = statement.hasPrefix("/*M!") + guard statement.hasPrefix("/*!") || isMariaDBMarker else { return [statement] } + guard let close = statement.range(of: "*/") else { return [statement] } let marked = statement[statement.index(statement.startIndex, offsetBy: 2)..= 5 ? Int(digits.prefix(5)) : nil + let body = afterMarker .drop(while: { $0.isNumber }) .trimmingCharacters(in: .whitespacesAndNewlines) let remainder = statement[close.upperBound...].trimmingCharacters(in: .whitespacesAndNewlines) - guard !remainder.isEmpty else { return body } - return body.isEmpty ? remainder : "\(body) \(remainder)" + let run = remainder.isEmpty ? body : body.isEmpty ? remainder : "\(body) \(remainder)" + let maySkip = isMariaDBMarker || (version ?? 0) > 40_101 + return maySkip ? [run] + executableBodies(of: remainder) : [run] } /// The statement's opening words with each run of whitespace collapsed, which is what the diff --git a/apple/Plugins/MySQLDriverTests/MySQLSessionFootprintTests.swift b/apple/Plugins/MySQLDriverTests/MySQLSessionFootprintTests.swift index 5c975a7a42..5aadc000e2 100644 --- a/apple/Plugins/MySQLDriverTests/MySQLSessionFootprintTests.swift +++ b/apple/Plugins/MySQLDriverTests/MySQLSessionFootprintTests.swift @@ -201,6 +201,33 @@ struct MySQLSessionFootprintTests { #expect(footprint(after: "/*! SET SESSION sql_mode = 'ANSI' */").hasSessionSettings) } + /// A server older than the version skips the comment, and both servers read `/*m!` as a plain + /// comment, so what follows it is the statement that runs. + @Test("A statement behind a comment the server skips is seen", arguments: [ + "/*!99999 note */ CREATE TEMPORARY TABLE t (a INT)", + "/*m! note */ CREATE TEMPORARY TABLE t (a INT)", + "/*M! note */ CREATE TEMPORARY TABLE t (a INT)", + ]) + func statementBehindSkippedCommentIsSeen(sql: String) { + #expect(footprint(after: sql).hasTemporaryTables) + #expect(footprint(after: sql.replacingOccurrences(of: "CREATE TEMPORARY TABLE t (a INT)", with: "LOCK TABLES t WRITE")) + .hasLockedTables) + } + + @Test("A statement a skipped comment's line comment hid from the split is still seen") + func statementsInsideASkippedCommentReadingAreSeen() { + let sql = "/*!99999 # */ BEGIN NOT ATOMIC DO 1; CREATE TEMPORARY TABLE t (id INT); END" + #expect(footprint(after: sql).hasTemporaryTables) + } + + @Test("A release one server runs and another skips does not release") + func ambiguousReleaseKeepsTheLock() { + #expect(footprint(after: "LOCK TABLES t WRITE", "/*!99999 note */ UNLOCK TABLES").hasLockedTables) + #expect(footprint(after: "LOCK TABLES t WRITE", "/*!99999 UNLOCK TABLES */").hasLockedTables) + #expect(!footprint(after: "LOCK TABLES t WRITE", "/*!40101 UNLOCK TABLES */").hasLockedTables) + #expect(!footprint(after: "LOCK TABLES t WRITE", "UNLOCK TABLES").hasLockedTables) + } + /// Only a statement that is entirely one of them. A version-gated comment inside a `CREATE /// TABLE`, which is how mysqldump writes a partition clause, is part of that statement. @Test("A version-gated comment inside another statement is left alone") diff --git a/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift b/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift index 461999a892..c4612aada4 100644 --- a/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift +++ b/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift @@ -143,7 +143,8 @@ struct SQLFeatureLexer { return cursor } - /// The server lexes an executable comment's body as SQL, so a `*/` inside a quoted string does not close it. + // The server lexes a run body as code, comments included, and a nested opener opens no second level: the first + // `*/` read as code closes the comment. Measured on MySQL 8.0 and 8.4 and MariaDB 10.11 and 11.4. private func executableCommentEnd(from index: Int) -> Int { var cursor = index + 2 while cursor < units.count { @@ -151,8 +152,14 @@ struct SQLFeatureLexer { if unit == Self.star, unitAt(cursor + 1) == Self.slash { return cursor + 2 } - if unit == Self.singleQuote || unit == Self.doubleQuote || unit == Self.backtick { - cursor = quotedEnd(from: cursor + 1, quote: unit, backslashEscapes: backslashEscapes(inQuote: unit)) + if unit == Self.slash, unitAt(cursor + 1) == Self.star { + cursor = isExecutableCommentOpener(at: cursor) + ? cursor + 2 + : blockCommentEnd(from: cursor, nests: features.contains(.nestedBlockComments)) + continue + } + if let inner = span(at: cursor) { + cursor = max(inner.end, cursor + 1) continue } cursor += 1 @@ -160,9 +167,10 @@ struct SQLFeatureLexer { return cursor } + // Both servers read a lowercase `/*m!` as a plain comment. private func isExecutableCommentOpener(at index: Int) -> Bool { var cursor = index + 2 - if unitAt(cursor) == 0x4D || unitAt(cursor) == 0x6D { + if unitAt(cursor) == 0x4D { cursor += 1 } return unitAt(cursor) == Self.exclamationMark diff --git a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift index 89e3f3a4c1..7847eece37 100644 --- a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift +++ b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift @@ -497,8 +497,13 @@ struct RowChangeStatementFactory { guard PluginManager.shared.editorLanguage(for: databaseType) == .sql else { return } let readings = databaseType.lexicalReadings for sql in statements { - for grammar in readings.distinct(for: sql) { - guard let write = SQLUnfilteredWrite.writes(in: sql, grammar: grammar).first else { continue } + // A driver writes no executable comments, so one that set too many to read is read as written. + let lexings = readings.lexings(of: sql).map { Array($0) } + ?? readings.distinct(for: sql).map { SQLLexing(grammar: $0, text: sql) } + for lexing in lexings { + guard let write = SQLUnfilteredWrite.writes(in: lexing.text, grammar: lexing.grammar).first else { + continue + } throw DataWriteError.changeRefused(table: tableName, kind: write.kind, reason: write.reason) } } diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index 06d1961cd2..14cc6a3c49 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -64,21 +64,42 @@ enum QueryClassifier { _ sql: String, databaseType: DatabaseType, readings: SQLLexicalReadings + ) -> QueryClassification { + classify(sql, databaseType: databaseType, readings: readings) { $0 == mostSevere } + } + + // A dump read under every server version is long, so a caller stops once nothing can change its answer. + private static func classify( + _ sql: String, + databaseType: DatabaseType, + readings: SQLLexicalReadings, + until isSettled: (QueryClassification) -> Bool ) -> QueryClassification { if let document = documentStoreClassification(sql, databaseType: databaseType) { return document } if let redis = redisClassification(sql, databaseType: databaseType) { return redis } let trimmed = StatementBlank.trimming(SQLLeadingComments.stripping(sql)) guard !trimmed.isEmpty else { return .safe } if let ledger = beancountClassification(trimmed, databaseType: databaseType) { return ledger } - return readings.distinct(for: sql).reduce(QueryClassification.safe) { worst, grammar in - runnableStatements(of: sql, grammar: grammar).reduce(worst) { partial, statement in - partial.escalated(with: statementClassification(statement, grammar: grammar, databaseType: databaseType)) + guard let lexings = readings.lexings(of: sql) else { return mostSevere } + var worst = QueryClassification.safe + var seen: Set = [] + for lexing in lexings { + for statement in runnableStatements(of: lexing.text, grammar: lexing.grammar) + where seen.insert(SQLLexing(grammar: lexing.grammar, text: statement)).inserted { + worst = worst.escalated( + with: statementClassification(statement, grammar: lexing.grammar, databaseType: databaseType) + ) + if isSettled(worst) { return worst } } } + return worst } + private static let mostSevere = QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: true) + static func isWriteQuery(_ sql: String, databaseType: DatabaseType) -> Bool { - classify(sql, databaseType: databaseType).tier != .safe + let readings = databaseType.lexicalReadings + return classify(sql, databaseType: databaseType, readings: readings) { $0.tier != .safe }.tier != .safe } static func isDangerousQuery(_ sql: String, databaseType: DatabaseType) -> Bool { @@ -86,7 +107,7 @@ enum QueryClassifier { } static func isDangerousQuery(_ sql: String, databaseType: DatabaseType, readings: SQLLexicalReadings) -> Bool { - let classification = classify(sql, databaseType: databaseType, readings: readings) + let classification = classify(sql, databaseType: databaseType, readings: readings) { $0.tier == .destructive } if classification.tier == .destructive { return true } // A MongoDB statement is JavaScript, where `delete d.x` removes a field from a local object. guard databaseType != .redis, databaseType != .mongodb else { return false } @@ -95,15 +116,19 @@ enum QueryClassifier { let code = SQLCodeProjection.code(of: trimmed, grammar: readings.execution).uppercased() return leadingKeyword(of: trimmed) == "DELETE" && !hasWhereClause(code: code) } - return readings.distinct(for: sql).contains { grammar in - runnableStatements(of: sql, grammar: grammar).contains { statement in - statementDeletesEverything(statement, grammar: grammar, databaseType: databaseType) + guard let lexings = readings.lexings(of: sql) else { return true } + var seen: Set = [] + return lexings.contains { lexing in + runnableStatements(of: lexing.text, grammar: lexing.grammar).contains { statement in + seen.insert(SQLLexing(grammar: lexing.grammar, text: statement)).inserted + && statementDeletesEverything(statement, grammar: lexing.grammar, databaseType: databaseType) } } } static func classifyTier(_ sql: String, databaseType: DatabaseType) -> QueryTier { - classify(sql, databaseType: databaseType).tier + let readings = databaseType.lexicalReadings + return classify(sql, databaseType: databaseType, readings: readings) { $0.tier == .destructive }.tier } static func reachesFilesystemOrExecutesCode(_ sql: String, databaseType: DatabaseType) -> Bool { @@ -118,8 +143,9 @@ enum QueryClassifier { /// A caller that sends one statement once cannot honour the count, and dropping it runs the statement once. static func isMultiStatement(_ sql: String, databaseType: DatabaseType, readings: SQLLexicalReadings) -> Bool { let model = QueryStatementModel.forDatabaseType(databaseType) - return readings.distinct(for: sql).contains { grammar in - let batches = QueryBatchPlanner.batches(in: sql, model: model, grammar: grammar) + guard let lexings = readings.lexings(of: sql) else { return true } + return lexings.contains { lexing in + let batches = QueryBatchPlanner.batches(in: lexing.text, model: model, grammar: lexing.grammar) return batches.flatMap(\.statements).count > 1 || batches.contains { $0.repeatCount > 1 } } } @@ -278,6 +304,26 @@ private extension QueryClassifier { "READFILE(", "WRITEFILE(", "FN_GET_AUDIT_FILE(", "XP_CMDSHELL", "SP_OACREATE" ] + // A comment, a line break or a run of spaces between two words, or a space before `(`, is nothing to the server: + // `INTO/**/OUTFILE` and `LOAD_FILE (` write and read files as their plain spellings do. + static func markerForm(of body: String) -> String { + var result = "" + result.reserveCapacity(body.utf8.count) + var pendingSpace = false + for character in body { + if character.isWhitespace { + pendingSpace = true + continue + } + if pendingSpace, character != "(", !result.isEmpty { + result.append(" ") + } + pendingSpace = false + result.append(character) + } + return result + } + static func hasWhereClause(code: String) -> Bool { let range = NSRange(code.startIndex..., in: code) return whereClauseRegex?.firstMatch(in: code, options: [], range: range) != nil @@ -290,7 +336,8 @@ private extension QueryClassifier { ) -> QueryClassification { let projection = StatementProjection(statement: statement, grammar: grammar) let body = projection.body - let touchesUnsafeSurface = filesystemMarkers.contains { body.contains($0) } + let markerBody = markerForm(of: body) + let touchesUnsafeSurface = filesystemMarkers.contains { markerBody.contains($0) } let keywordTier = keywordClassification(projection, grammar: grammar, databaseType: databaseType) let base = keywordTier.tier == .safe ? keywordTier.escalated(to: stateChangingCallTier(projection, databaseType: databaseType)) @@ -305,18 +352,19 @@ private extension QueryClassifier { return classification.escalated(with: conditional) } - /// A MySQL `/*! ... */` runs its body, so it is tiered by what the body says whatever the grammar thinks of it: - /// reading a comment another engine ignores as code only ever raises the tier. + /// A MySQL `/*! ... */` runs its body, so it is tiered by what the body says whatever the grammar thinks of it, and + /// a statement holding one is never safe. What a server that skips it runs is tiered as a text of its own. static func conditionalCommentClassification( _ statement: String, grammar: SQLLexicalGrammar ) -> QueryClassification? { - guard SQLLeadingComments.conditionalCommentOpeners.contains(where: { statement.contains($0) }) else { return nil } + let readsOpeners = grammar.union(.executableComments) + guard SQLExecutableCommentReadings.holdsExecutableComment(statement, grammar: readsOpeners) else { return nil } let revealed = SQLCodeProjection.code(of: statement, grammar: grammar, revealingExecutableComments: true) .uppercased() - guard SQLLeadingComments.conditionalCommentOpeners.contains(where: { revealed.contains($0) }) else { return nil } let dropsData = destructiveKeywords.contains { containsWord(revealed, $0) } - let reachesFilesystemOrExecutesCode = filesystemMarkers.contains { revealed.contains($0) } + let revealedMarkers = markerForm(of: revealed) + let reachesFilesystemOrExecutesCode = filesystemMarkers.contains { revealedMarkers.contains($0) } || filesystemOrCodeKeywords.contains { containsWord(revealed, $0) } return QueryClassification( tier: dropsData ? .destructive : .write, @@ -345,7 +393,10 @@ private extension QueryClassifier { } if keyword == "ALTER" { - let tier: QueryTier = body.range(of: " DROP ", options: .literal) != nil ? .destructive : .write + // `ALTER TABLE t TRUNCATE PARTITION ALL` empties the table, measured on MySQL 8.4. + let dropsData = body.range(of: " DROP ", options: .literal) != nil + || body.range(of: " TRUNCATE ", options: .literal) != nil + let tier: QueryTier = dropsData ? .destructive : .write return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: false) } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift index dc4103dfcb..c465dee946 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -55,7 +55,6 @@ final class SQLFileParser: Sendable { private static let kSlash: unichar = 0x2F private static let kStar: unichar = 0x2A private static let kHash: unichar = 0x23 - private static let kExclamation: unichar = 0x21 private static let kNewline: unichar = 0x0A private static let kSpace: unichar = 0x20 private static let kTab: unichar = 0x09 @@ -63,8 +62,6 @@ final class SQLFileParser: Sendable { private static let kDollar: unichar = 0x24 private static let kCapitalE: unichar = 0x45 private static let kSmallE: unichar = 0x65 - private static let kCapitalM: unichar = 0x4D - private static let kSmallM: unichar = 0x6D private static let kOpenBracket: unichar = 0x5B private static let kCloseBracket: unichar = 0x5D private static let kCapitalG: unichar = 0x47 @@ -139,6 +136,22 @@ final class SQLFileParser: Sendable { private static let delimiterPrefix = "DELIMITER " private static let delimiterPrefixLength = 10 + // mysqldump writes `DELIMITER ;;`, whose new delimiter is the rest of the line, so a `;` in it ends nothing. + private static func readsDelimiterDirective(_ ctx: ParserContext) -> Bool { + guard ctx.grammar.contains(.delimiterDirective), let statement = ctx.currentStatement else { return false } + let length = statement.length + var index = 0 + while index < length, isWhitespace(statement.character(at: index)) { + index += 1 + } + let prefix = delimiterPrefix as NSString + guard length - index >= prefix.length, + statement.substring(with: NSRange(location: index, length: prefix.length)).uppercased() == delimiterPrefix + else { return false } + let rest = NSRange(location: index, length: length - index) + return statement.range(of: "\n", options: [], range: rest).location == NSNotFound + } + private static func extractDelimiterChange(_ text: String) -> String? { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard trimmed.uppercased().hasPrefix(delimiterPrefix) else { return nil } @@ -154,7 +167,9 @@ final class SQLFileParser: Sendable { var hasStatementContent = false var currentLine = 1 var statementStartLine = 1 - var isConditionalComment = false + // A run `/*! ... */` body is code to the server, comments and quotes included, so it is read as code and sent + // whole, and a `;` inside it ends nothing. + var insideConditionalComment = false var commentDepth = 0 var currentDelimiter: NSString = ";" as NSString var isSingleCharDelimiter = true @@ -231,13 +246,7 @@ final class SQLFileParser: Sendable { /// Whether the comment being read goes into the text sent. var keepsComments: Bool { - readsBatches || (keepsStatementComments && hasStatementContent) - } - - /// Whether the block comment being read goes into the statement, which a conditional comment always does, - /// being SQL the server runs. - var keepsCommentText: Bool { - keepsComments || isConditionalComment + readsBatches || (keepsStatementComments && hasStatementContent) || insideConditionalComment } } @@ -249,6 +258,7 @@ final class SQLFileParser: Sendable { private static func resetStatement(_ ctx: inout ParserContext) { ctx.currentStatement?.setString("") ctx.hasStatementContent = false + ctx.insideConditionalComment = false ctx.boundaries?.reset() ctx.word.removeAll(keepingCapacity: true) } @@ -454,23 +464,8 @@ final class SQLFileParser: Sendable { return StepResult(advanced: false, deferred: false) } - if char == kSlash, let next = nextChar, next == kStar { - ctx.isConditionalComment = ctx.grammar.contains(.executableComments) - && SqlLexer.executableCommentOpenerLength(nsBuffer, at: i, length: bufLen) != nil - ctx.commentDepth = 1 - ctx.state = .inMultiLineComment - if ctx.isConditionalComment { - (ctx.hasStatementContent, ctx.statementStartLine) = markContent( - ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) - } else { - ctx.boundaries?.observeGap() - } - if ctx.keepsCommentText { - appendChar(char, to: ctx.currentStatement) - appendChar(next, to: ctx.currentStatement) - } - i += 2 - return StepResult(advanced: true, deferred: false) + if let step = processBlockCommentMarker(&ctx, char: char, nextChar: nextChar, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) { + return step } if ctx.grammar.contains(.escapeStringPrefix) @@ -545,8 +540,8 @@ final class SQLFileParser: Sendable { return StepResult(advanced: false, deferred: false) } - if !ctx.isSingleCharDelimiter - && matchesDelimiter(at: i, delimiter: ctx.currentDelimiter, in: nsBuffer, bufLen: bufLen) { + if !ctx.isSingleCharDelimiter, !ctx.insideConditionalComment, !readsDelimiterDirective(ctx), + matchesDelimiter(at: i, delimiter: ctx.currentDelimiter, in: nsBuffer, bufLen: bufLen) { yieldAndReset(&ctx) i += ctx.currentDelimiter.length return StepResult(advanced: true, deferred: false) @@ -565,10 +560,55 @@ final class SQLFileParser: Sendable { return StepResult(advanced: false, deferred: false) } + /// A `/*` opens a comment, or the code of a conditional comment the server runs, which a `*/` read as code closes. + private static func processBlockCommentMarker( + _ ctx: inout ParserContext, + char: unichar, + nextChar: unichar?, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult? { + if ctx.insideConditionalComment, char == kStar, nextChar == kSlash { + appendRange(&ctx, from: i, to: i + 2, in: nsBuffer) + ctx.insideConditionalComment = false + i += 2 + return StepResult(advanced: true, deferred: false) + } + guard char == kSlash, let next = nextChar, next == kStar else { return nil } + if ctx.grammar.contains(.executableComments) { + // `/*M!` takes four units to tell apart from a plain comment. + if i + 3 >= bufLen, !ctx.atEndOfInput { + return StepResult(advanced: false, deferred: true) + } + if SqlLexer.executableCommentOpenerLength(nsBuffer, at: i, length: bufLen) != nil { + (ctx.hasStatementContent, ctx.statementStartLine) = markContent( + ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) + ctx.insideConditionalComment = true + appendRange(&ctx, from: i, to: i + 2, in: nsBuffer) + i += 2 + return StepResult(advanced: true, deferred: false) + } + } + ctx.commentDepth = 1 + ctx.state = .inMultiLineComment + ctx.boundaries?.observeGap() + if ctx.keepsComments { + appendChar(char, to: ctx.currentStatement) + appendChar(next, to: ctx.currentStatement) + } + i += 2 + return StepResult(advanced: true, deferred: false) + } + /// A `;` ends the statement unless the grammar holds it inside a PL/SQL unit or a routine body, and a statement /// that owns the `;` that ends it keeps it. In a batch it ends nothing, unless the batch has grown past what the /// server takes in one request. private static func processSemicolon(_ ctx: inout ParserContext, at i: Int) { + if ctx.insideConditionalComment || readsDelimiterDirective(ctx) { + appendChar(kSemicolon, to: ctx.currentStatement) + return + } if ctx.readsBatches { appendChar(kSemicolon, to: ctx.currentStatement) let batchEnd = ctx.unitsBeforeBuffer + i + 1 @@ -836,10 +876,10 @@ final class SQLFileParser: Sendable { nextChar: unichar?, i: inout Int ) -> Bool { - if ctx.keepsCommentText { + if ctx.keepsComments { appendChar(char, to: ctx.currentStatement) } - if char == kSlash, nextChar == kStar, !ctx.isConditionalComment, ctx.grammar.contains(.nestedBlockComments) { + if char == kSlash, nextChar == kStar, ctx.grammar.contains(.nestedBlockComments) { if ctx.keepsComments { appendChar(kStar, to: ctx.currentStatement) } @@ -848,14 +888,13 @@ final class SQLFileParser: Sendable { return true } if char == kStar, let next = nextChar, next == kSlash { - if ctx.keepsCommentText { + if ctx.keepsComments { appendChar(next, to: ctx.currentStatement) } ctx.commentDepth -= 1 i += 2 - guard ctx.commentDepth <= 0 || ctx.isConditionalComment else { return true } + guard ctx.commentDepth <= 0 else { return true } ctx.state = .normal - ctx.isConditionalComment = false ctx.commentDepth = 0 return true } diff --git a/apple/TablePro/Core/Utilities/SQL/TableEditStatementParser.swift b/apple/TablePro/Core/Utilities/SQL/TableEditStatementParser.swift index 9a447522fe..35a38af6cb 100644 --- a/apple/TablePro/Core/Utilities/SQL/TableEditStatementParser.swift +++ b/apple/TablePro/Core/Utilities/SQL/TableEditStatementParser.swift @@ -82,8 +82,9 @@ enum TableEditStatementParser { /// may create still counts as a hazard, since counting one that never ran only skips more. static func parse(_ sql: String, dialect: TableEditDialect, grammar: SQLLexicalGrammar) -> TableEditStatement { let statement = read(sql, dialect: dialect, grammar: grammar) - guard statement.editsTable, grammar.contains(.executableComments), - sql.contains("/*!") || sql.contains("/*M!") else { return statement } + guard statement.editsTable, SQLExecutableCommentReadings.holdsExecutableComment(sql, grammar: grammar) else { + return statement + } return .other } diff --git a/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift b/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift index 7575ae077a..1e908b8c27 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift @@ -69,7 +69,9 @@ nonisolated enum SessionLocalStatements { // A MySQL executable comment can hide any statement, a temporary table among them, from the patterns. if usePattern?.firstMatch(in: code, range: range) != nil || contextPattern?.firstMatch(in: code, range: range) != nil - || SQLLeadingComments.conditionalCommentOpeners.contains(where: { code.contains($0) }) + || SQLLeadingComments.conditionalCommentOpeners.contains(where: { + (code as NSString).range(of: $0, options: .literal).location != NSNotFound + }) || SQLCatalogEffectClassifier.effect(of: statement, grammar: grammar).runsUnseenStatements { effect.leavesCatalog = true } diff --git a/apple/TableProTests/Core/Execution/ExternalStatementGateTests.swift b/apple/TableProTests/Core/Execution/ExternalStatementGateTests.swift index 15f1cb212e..262e470279 100644 --- a/apple/TableProTests/Core/Execution/ExternalStatementGateTests.swift +++ b/apple/TableProTests/Core/Execution/ExternalStatementGateTests.swift @@ -293,3 +293,55 @@ struct ExternalStatementGateTests { )) } } + +struct ExternalStatementGateExecutableCommentTests { + private func refusal(_ sql: String, access: ExternalAccessLevel, allowsDestructive: Bool) -> ExternalStatementGateError? { + let statement = ExternalStatementGate.Statement( + sql: sql, + connectionId: UUID(), + databaseType: .mysql, + externalAccess: access, + loadsExtensions: false, + allowsDestructive: allowsDestructive, + allowsMultiStatement: false, + destructiveAlternative: nil + ) + do { + _ = try ExternalStatementGate.classify(statement) + return nil + } catch let error as ExternalStatementGateError { + return error + } catch { + return nil + } + } + + @Test("A DROP behind an executable comment is refused when the caller may not drop", arguments: [ + "/*!99999 ' */ DROP TABLE orders -- '", + "/*M!999999 ' */ DROP TABLE orders -- '", + "/*m! ' */ DROP TABLE orders -- '", + "/*! /* ' */ DROP TABLE orders */", + ]) + func hiddenDropIsRefused(sql: String) { + #expect(refusal(sql, access: .readWrite, allowsDestructive: false) + == .denied(String(localized: "This statement drops or truncates data."))) + } + + @Test("A write behind a lowercase /*m! is refused on a read-only connection") + func hiddenWriteIsRefusedOnReadOnly() { + #expect(refusal("WITH c AS (SELECT 1) /*m! ' */ DELETE FROM orders -- '", access: .readOnly, allowsDestructive: true) + == .denied(String(localized: "This connection is read only for external clients."))) + } + + @Test("A file written behind an executable comment is refused", arguments: [ + "SELECT * FROM orders /*m! ' */ INTO OUTFILE '/tmp/x' -- '", + "SELECT * FROM orders /*!99999 ' */ INTO OUTFILE '/tmp/x' -- '", + ]) + func hiddenOutfileIsRefused(sql: String) { + guard case .denied(let reason) = refusal(sql, access: .readWrite, allowsDestructive: true) else { + Issue.record("not refused: \(sql)") + return + } + #expect(reason.contains("files")) + } +} diff --git a/apple/TableProTests/Core/ServerDashboard/DashboardTilePolicyTests.swift b/apple/TableProTests/Core/ServerDashboard/DashboardTilePolicyTests.swift index fc5baeb20e..f6c772b199 100644 --- a/apple/TableProTests/Core/ServerDashboard/DashboardTilePolicyTests.swift +++ b/apple/TableProTests/Core/ServerDashboard/DashboardTilePolicyTests.swift @@ -30,6 +30,15 @@ struct DashboardTilePolicyTests { ) } + @Test("A write behind an executable comment is refused", arguments: [ + "WITH c AS (SELECT 1) /*m! ' */ DELETE FROM orders -- '", + "WITH c AS (SELECT 1) /*!99999 ' */ DELETE FROM orders -- '", + "SELECT * FROM orders /*m! ' */ INTO OUTFILE '/tmp/x' -- '", + ]) + func hiddenWriteIsRefused(sql: String) { + #expect(DashboardTilePolicy.refusal(for: sql, databaseType: .mysql, forcesWrite: false) != nil) + } + @Test("Two statements are refused, even when both read") func multipleStatementsAreRefused() { #expect( diff --git a/apple/TableProTests/Core/Services/Execution/ExecutionGateTests.swift b/apple/TableProTests/Core/Services/Execution/ExecutionGateTests.swift index f8c2f8cbd2..0e304c6e92 100644 --- a/apple/TableProTests/Core/Services/Execution/ExecutionGateTests.swift +++ b/apple/TableProTests/Core/Services/Execution/ExecutionGateTests.swift @@ -860,3 +860,62 @@ struct ExecutionGateTests { } } } + +// The editor's request for a text the gate must tier by what the server runs, behind each executable comment form. +@MainActor +struct ExecutionGateExecutableCommentTests { + nonisolated static let hiddenDrops: [(String, DatabaseType)] = QueryClassifierExecutableCommentTests.hiddenDrops.flatMap { sql in + QueryClassifierExecutableCommentTests.engines.map { (sql, $0) } + } + + private func authorize(_ sql: String, on databaseType: DatabaseType, at level: SafeModeLevel) async + -> (decision: OperationDecision, confirm: StubConfirming) + { + let confirm = StubConfirming(answer: true) + let gate = DefaultExecutionGate( + confirming: confirm, + authenticating: StubAuthenticating(answer: true), + safeModeLevelResolver: { _ in level }, + forcesWriteResolver: { _ in false } + ) + let request = OperationRequest( + connectionId: UUID(), + databaseType: databaseType, + sql: sql, + kind: OperationKind.worst(of: [sql], databaseType: databaseType), + caller: .userInterface, + capabilities: .interactiveUser, + operationDescription: "Execute Query" + ) + return (await gate.authorize(request), confirm) + } + + @Test( + "A DROP behind an executable comment is refused in Read-Only and asked about as destructive elsewhere", + arguments: hiddenDrops, SafeModeLevel.allCases + ) + func hiddenDropAtEveryLevel(text: (String, DatabaseType), level: SafeModeLevel) async { + let (decision, confirm) = await authorize(text.0, on: text.1, at: level) + if level == .readOnly { + #expect(!decision.isAuthorized) + #expect(confirm.callCount == 0) + } else { + #expect(decision.isAuthorized) + #expect(confirm.callCount == 1) + #expect(confirm.lastDestructive) + } + } + + @Test( + "A write behind a comment a server reads as plain is refused in Read-Only", + arguments: [ + "WITH c AS (SELECT 1) /*m! ' */ DELETE FROM orders -- '", + "SELECT * FROM orders /*m! ' */ INTO OUTFILE '/tmp/x' -- '", + "SELECT * FROM orders /*!99999 ' */ INTO OUTFILE '/tmp/x' -- '", + ] + ) + func hiddenWriteIsRefusedInReadOnly(sql: String) async { + let (decision, _) = await authorize(sql, on: .mysql, at: .readOnly) + #expect(!decision.isAuthorized) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift b/apple/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift index 3159854311..0bced6ceb1 100644 --- a/apple/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/CatalogChangeClassifierTests.swift @@ -69,6 +69,12 @@ struct CatalogChangeClassifierTests { #expect(kinds("/*!40101 SET NAMES utf8mb4 */", .mysql).isEmpty) } + @Test("a DROP a server runs past a comment it skips still refreshes the tables") + func skippedExecutableCommentDoesNotHideADrop() { + #expect(kinds("/*!99999 ' */ DROP TABLE orders -- '", .mysql).contains(.tables)) + #expect(kinds("/*m! ' */ DROP TABLE orders -- '", .mysql).contains(.tables)) + } + @Test("a SQL Server batch is read past its first command") func sqlServerBatchIsReadPastItsFirstCommand() { let cases: [(sql: String, expected: SQLCatalogObjectKinds)] = [ diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 01fc9c7bcb..aeb52f8d45 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -807,3 +807,120 @@ struct QueryClassifierNonSqlTests { ) } } + +struct QueryClassifierExecutableCommentTests { + // One per form a MySQL or MariaDB server runs a DROP from while every other reading hides it. + static let hiddenDrops = [ + "/*!99999 ' */ DROP TABLE orders -- '", + "/*!99999 /* */ ' */ DROP TABLE orders -- '", + "/*!50700 ' */ DROP TABLE orders -- '", + "/*!800000 ' */ DROP TABLE orders -- '", + "/*!8000000 ' */ DROP TABLE orders -- '", + "/*M!999999 ' */ DROP TABLE orders -- '", + "/*M! ' */ DROP TABLE orders -- '", + "/*m! ' */ DROP TABLE orders -- '", + "/*! /* ' */ DROP TABLE orders */", + "/*! DROP TABLE orders # */ '\n */", + "SELECT /*!50000 \"*/\", */ 1; /*!99999 ' */ DROP TABLE orders -- '", + "SELECT /*!80000 \"*/\", */ 1; /*!80100 ' */ DROP TABLE orders -- '", + "SELECT 1--/*!99999 ' */1; DROP TABLE orders -- '", + "ALTER TABLE orders ALTER COLUMN note SET DEFAULT (1 --/*!99999 ' */ 1), DROP COLUMN total -- '", + "/*T! DROP TABLE orders */", + "/*! SELECT 1 /*!99999 ' */ */; DROP TABLE orders -- '", + "ALTER TABLE orders ADD COLUMN c\u{600}/*!99999 ' */ INT, DROP COLUMN total -- '", + ] + + static let engines: [DatabaseType] = [.mysql, .mariadb] + + @Test("A DROP TiDB runs from its own comment is destructive", arguments: [ + "/*T! DROP TABLE orders */", + "/*T![clustered_index] DROP TABLE orders */", + "/*T![no_such_feature] ' */ DROP TABLE orders -- '", + ]) + func tiDBCommentDropIsDestructive(sql: String) { + #expect(QueryClassifier.classifyTier(sql, databaseType: .tidb) == .destructive) + } + + @Test("A DROP a server runs from behind an executable comment is destructive", arguments: hiddenDrops, engines) + func hiddenDropIsDestructive(sql: String, databaseType: DatabaseType) { + #expect(QueryClassifier.classifyTier(sql, databaseType: databaseType) == .destructive) + #expect(QueryClassifier.isDangerousQuery(sql, databaseType: databaseType)) + } + + @Test("A TRUNCATE a server runs past a skipped comment is destructive", arguments: engines) + func hiddenTruncateIsDestructive(databaseType: DatabaseType) { + let sql = "/*!99999 ' */ TRUNCATE orders -- '" + #expect(QueryClassifier.classifyTier(sql, databaseType: databaseType) == .destructive) + #expect(QueryClassifier.classifyTier("ALTER TABLE p TRUNCATE PARTITION ALL", databaseType: databaseType) + == .destructive) + } + + @Test( + "A DELETE with no WHERE that a server runs from an executable comment deletes every row", + arguments: [ + "/*!99999 ' */ DELETE FROM orders -- '", + "/*! DELETE FROM orders */", + "/*M! DELETE FROM orders */", + "/*! -- '\n */ DELETE FROM orders -- '", + "WITH c AS (SELECT 1) /*m! ' */ DELETE FROM orders -- '", + "DELETE FROM orders a\u{600}/*!99999 '*/#' */ WHERE id = 1", + ], + engines + ) + func hiddenDeleteDeletesEverything(sql: String, databaseType: DatabaseType) { + #expect(QueryClassifier.isDangerousQuery(sql, databaseType: databaseType)) + } + + @Test( + "A write past a comment one server reads as plain is never a read", + arguments: [ + "WITH c AS (SELECT 1) /*m! ' */ DELETE FROM orders -- '", + "WITH c AS (SELECT 1) /*!99999 ' */ DELETE FROM orders -- '", + "SELECT * FROM orders /*m! ' */ INTO OUTFILE '/tmp/x' -- '", + "SELECT * FROM orders /*!99999 ' */ INTO OUTFILE '/tmp/x' -- '", + ], + engines + ) + func hiddenWriteIsNotARead(sql: String, databaseType: DatabaseType) { + #expect(QueryClassifier.isWriteQuery(sql, databaseType: databaseType)) + } + + @Test( + "A file written past an executable comment reaches the file system", + arguments: [ + "SELECT * FROM orders /*m! ' */ INTO OUTFILE '/tmp/x' -- '", + "SELECT * FROM orders /*!99999 ' */ INTO OUTFILE '/tmp/x' -- '", + "SELECT 1--/*!99999 ' */1 INTO OUTFILE '/tmp/x' -- '", + "SELECT 1 a\u{600}/*!99999 ' */ INTO OUTFILE '/tmp/x' -- '", + "SELECT 1 INTO/**/OUTFILE '/tmp/x'", + "SELECT LOAD_FILE ('/etc/hosts')", + "SELECT LOAD_FILE/**/('/etc/hosts')", + ] + ) + func hiddenOutfileReachesTheFileSystem(sql: String) { + #expect(QueryClassifier.reachesFilesystemOrExecutesCode(sql, databaseType: .mysql)) + } + + @Test( + "A second statement past a skipped comment is counted", + arguments: ["/*!99999 ' */ DROP TABLE a; DROP TABLE b -- '", "SELECT 1--/*!99999 ' */1; DROP TABLE b -- '"], + engines + ) + func hiddenSecondStatementIsCounted(sql: String, databaseType: DatabaseType) { + #expect(QueryClassifier.isMultiStatement(sql, databaseType: databaseType)) + } + + @Test("A text that sets more versions than a gate reads is treated as the worst statement") + func tooManyVersionsFailClosed() { + let sql = "SELECT " + (0..<40).map { "'/*!\(50_000 + $0) x */'" }.joined(separator: ", ") + #expect(QueryClassifier.classify(sql, databaseType: .mysql).tier == .destructive) + #expect(QueryClassifier.reachesFilesystemOrExecutesCode(sql, databaseType: .mysql)) + #expect(QueryClassifier.isMultiStatement(sql, databaseType: .mysql)) + } + + @Test("A read whose comment every server runs or skips alike stays a read once the floor is kept") + func plainReadsKeepTheirTier() { + #expect(QueryClassifier.classifyTier("SELECT 1 /*m! note */", databaseType: .mysql) == .safe) + #expect(QueryClassifier.classifyTier("SELECT /*!40001 SQL_NO_CACHE */ 1", databaseType: .mysql) == .write) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index ff9b28423b..fba6f05c3e 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -267,6 +267,44 @@ struct SQLFileParserTests { #expect(stmts == [Self.merge, "SELECT 1"]) } + @Test("MySQL: a run executable comment's body is read as code, quotes and comments included", arguments: [ + ("/*!40101 SET @a = '*/' */;\nSELECT 1;", ["/*!40101 SET @a = '*/' */", "SELECT 1"]), + ("/*! SELECT 1 /* ' */ */;\nSELECT 2;", ["/*! SELECT 1 /* ' */ */", "SELECT 2"]), + ("/*! SELECT 1 # */ '\n */;\nSELECT 2;", ["/*! SELECT 1 # */ '\n */", "SELECT 2"]), + ("/*m! ' */ SELECT 1;\nSELECT 2;", ["SELECT 1", "SELECT 2"]), + ]) + func mysqlExecutableCommentBodyIsCode(sql: String, expected: [String]) async throws { + #expect(try await Self.parse(sql, grammar: TestGrammar.mysql) == expected) + } + + @Test("MySQL: a DELIMITER line ends at its line break, so mysqldump's DELIMITER ;; is read") + func mysqlDelimiterOfSemicolons() async throws { + let sql = "DELIMITER ;;\nSELECT 1;;\nDELIMITER ;\nSELECT 2;\n" + #expect(try await Self.parse(sql, grammar: TestGrammar.mysql) == ["SELECT 1", "SELECT 2"]) + } + + @Test("MySQL: a dumped trigger inside version comments arrives whole") + func mysqlDumpedTriggerArrivesWhole() async throws { + let trigger = """ + /*!50003 CREATE*/ /*!50017 DEFINER=`root`@`%`*/ /*!50003 TRIGGER `trg` BEFORE INSERT ON `t` FOR EACH ROW BEGIN + SET NEW.a = 'x;y */'; + END */ + """ + let sql = "DELIMITER ;;\n\(trigger);;\nDELIMITER ;\nSELECT 1;\n" + #expect(try await Self.parse(sql, grammar: TestGrammar.mysql) == [trigger, "SELECT 1"]) + } + + @Test("MySQL: an executable opener a read chunk splits is still read as one") + func mysqlExecutableOpenerAcrossChunkBoundary() async throws { + let script = "/*M!100301 SET NAMES utf8mb4 */;\nSELECT 1;" + let chunkSize = 65_536 + for boundary in 0..<8 { + let padding = "-- " + String(repeating: "x", count: chunkSize - boundary - 4) + "\n" + let statements = try await Self.parse(padding + script, grammar: TestGrammar.mysql) + #expect(statements == ["/*M!100301 SET NAMES utf8mb4 */", "SELECT 1"], "boundary at \(boundary)") + } + } + @Test("Multi-byte UTF-8 char straddling 64KB chunk boundary parses correctly") func multibyte_utf8_at_chunk_boundary() async throws { let chunkSize = 65_536 diff --git a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift index 6919cda96d..329ccca060 100644 --- a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift +++ b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift @@ -81,6 +81,11 @@ struct SQLLexicalFeatureMappingTests { "SELECT $a('); DROP TABLE t; --'", "SELECT '''it's'''; DROP TABLE t; SELECT 'x'", "SELECT 1 /*! , '*/' */; SELECT 2", + "/*! /* ' */ SELECT 1 */; SELECT 2", + "/*! SELECT 1 # '\n */; SELECT 2", + "/*! SELECT 1 -- '\n */; SELECT 2", + "/*! SELECT /*! 1 */; SELECT 2", + "/*m! ' */ SELECT 1; SELECT 2 -- '", ] static let engines: [DatabaseType] = [ diff --git a/apple/scripts/probes/check-mysql-executable-comments.sh b/apple/scripts/probes/check-mysql-executable-comments.sh new file mode 100755 index 0000000000..a5e77e7d03 --- /dev/null +++ b/apple/scripts/probes/check-mysql-executable-comments.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# +# Check SQLExecutableCommentReadings against live MySQL, MariaDB and TiDB servers. +# +# Packages/TableProCore/Sources/TableProSQLGrammar/SQLExecutableCommentReadings.swift says which `/*! */` comments a +# server runs, skips or reads as plain, by product and version. Safe Mode tiers a statement by those texts, so a rule +# that is wrong there lets a server run a DROP the gate never saw. This sends texts that hide a `DROP TABLE` behind one +# comment form each, as one query per text, and checks that the server dropped the table exactly when the model's text +# for that server's product and version shows the DROP. +# +# Usage: +# scripts/probes/check-mysql-executable-comments.sh +# +# Each server is skipped unless its connection is set in the environment, as mysql client arguments: +# MYSQL_ARGS "-h 127.0.0.1 -P 13306 -uroot -pprobe" +# MARIADB_ARGS "-h 127.0.0.1 -P 13307 -uroot -pprobe" +# TIDB_ARGS "-h 127.0.0.1 -P 14000 -uroot" +# +# Throwaway servers that match those defaults: +# docker run -d --name probe-mysql -p 13306:3306 -e MYSQL_ROOT_PASSWORD=probe mysql:8.4 +# docker run -d --name probe-mariadb -p 13307:3306 -e MARIADB_ROOT_PASSWORD=probe mariadb:11.4 +# docker run -d --name probe-tidb -p 14000:4000 pingcap/tidb:v8.5.1 +# +# Each text goes through --init-command, which libmysqlclient sends as one query and the client's own statement +# splitter never reads. The probe works in a scratch database of its own and drops only that one. +# Exits non-zero on a disagreement, 3 when nothing could be checked. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +DATABASE="tp_executable_comment_probe" + +if [ -z "${DEVELOPER_DIR:-}" ]; then + for candidate in /Applications/Xcode-beta.app /Applications/Xcode.app; do + if [ -d "$candidate/Contents/Developer" ]; then + export DEVELOPER_DIR="$candidate/Contents/Developer" + break + fi + done +fi +command -v swift > /dev/null || { + echo "swift not found" >&2 + exit 3 +} + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/Sources/Probe" + +cat > "$WORK/Package.swift" < "$WORK/Sources/Probe/main.swift" <<'EOF' +import Foundation +import TableProSQLGrammar + +let grammar = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil).execution +let families: [String: SQLExecutableCommentReadings.Family] = [ + "mySQL5": .mySQL5, "mySQL81": .mySQL81, "mariaDB": .mariaDB, "tiDB": .tiDB, +] + +while let line = readLine() { + let fields = line.split(separator: "\t").map(String.init) + guard fields.count == 3, let family = families[fields[0]], let version = Int(fields[1]), + let data = Data(base64Encoded: fields[2]), let sql = String(data: data, encoding: .utf8) + else { + print("?") + continue + } + let server = SQLExecutableCommentReadings.Server(family: family, version: version) + let text = SQLExecutableCommentReadings.text(of: sql, grammar: grammar, on: server) + let drops = SQLStatementScanner.executableStatements(in: text, grammar: grammar).contains { statement in + let code = SQLCodeProjection.code(of: statement.sql, grammar: grammar) + return code.range(of: #"DROP\s+TABLE\s+t\b"#, options: .regularExpression) != nil + } + print(drops ? 1 : 0) +} +EOF + +echo "Building the probe against Packages/TableProCore (the first build takes a minute)" +(cd "$WORK" && swift build -c debug > "$WORK/build.log" 2>&1) || { + grep -E "error:" "$WORK/build.log" | head -20 >&2 + exit 3 +} +PROBE="$WORK/.build/debug/Probe" + +CASES=( + "/*!99999 ' */ DROP TABLE t -- '" + "/*!50700 ' */ DROP TABLE t -- '" + "/*!40101 ' */ DROP TABLE t -- '" + "/*!100000 ' */ DROP TABLE t -- '" + "/*!800000 ' */ DROP TABLE t -- '" + "/*!800000' */ DROP TABLE t -- '" + "/*!8000000 ' */ DROP TABLE t -- '" + "/*!80047 ' */ DROP TABLE t -- '" + "/*!DROP TABLE t */" + "/*!99999 /* */ ' */ DROP TABLE t -- '" + "/*!99999 /* /* */ */ ' */ DROP TABLE t -- '" + "/*!99999 # ' */ DROP TABLE t -- '" + "/*M! ' */ DROP TABLE t -- '" + "/*M!999999 ' */ DROP TABLE t -- '" + "/*M!99999 ' */ DROP TABLE t -- '" + "/*M! /* */ ' */ DROP TABLE t -- '" + "/*m! ' */ DROP TABLE t -- '" + "/*! /* ' */ DROP TABLE t */" + $'/*! DROP TABLE t # */ \'\n */' + "/*! DROP TABLE /*! t */" + "SELECT /*!50000 \"*/\", */ 1; /*!99999 ' */ DROP TABLE t -- '" + "SELECT /*!80000 \"*/\", */ 1; /*!80100 ' */ DROP TABLE t -- '" + "SELECT 1--/*!99999 ' */1; DROP TABLE t -- '" + "/*T! DROP TABLE t */" + "/*T![clustered_index] DROP TABLE t */" +) + +failures=0 +checked=0 + +check_server() { + local name="$1" client="$2" args="$3" + read -r -a argv <<< "$args" + if ! command -v "$client" > /dev/null 2>&1; then + echo "$name: no $client client, skipped" + return + fi + local version + version=$("$client" "${argv[@]}" -N -e "SELECT VERSION()" 2> /dev/null) || { + echo "$name: cannot connect, skipped" + return + } + local family numbers major minor patch id + numbers="${version%%-*}" + IFS=. read -r major minor patch <<< "$numbers" + id=$((major * 10000 + minor * 100 + ${patch:-0})) + if [[ "$version" == *TiDB* ]]; then + family="tiDB" + elif [[ "$version" == *MariaDB* ]]; then + family="mariaDB" + elif [ "$id" -ge 80100 ]; then + family="mySQL81" + else + family="mySQL5" + fi + echo "$name: $version ($family $id)" + checked=$((checked + 1)) + + for sql in "${CASES[@]}"; do + "$client" "${argv[@]}" -e "DROP DATABASE IF EXISTS $DATABASE; CREATE DATABASE $DATABASE; \ + CREATE TABLE $DATABASE.t (id INT)" > /dev/null 2>&1 + "$client" "${argv[@]}" -D "$DATABASE" --init-command="$sql" -e "SELECT 1" > /dev/null 2>&1 + local remaining dropped expected + remaining=$("$client" "${argv[@]}" -N -e "SELECT COUNT(*) FROM information_schema.tables \ + WHERE table_schema = '$DATABASE' AND table_name = 't'" 2> /dev/null) + dropped=$([ "$(echo "$remaining" | tr -d '[:space:]')" = "0" ] && echo 1 || echo 0) + expected=$(printf '%s\t%s\t%s\n' "$family" "$id" "$(printf '%s' "$sql" | base64 | tr -d '\n')" | "$PROBE") + if [ "$dropped" = "$expected" ]; then + echo " ok $(printf '%q' "$sql")" + else + echo " FAIL $(printf '%q' "$sql"): server dropped=$dropped, model=$expected" + failures=$((failures + 1)) + fi + done + "$client" "${argv[@]}" -e "DROP DATABASE IF EXISTS $DATABASE" > /dev/null 2>&1 +} + +[ -n "${MYSQL_ARGS:-}" ] && check_server "MySQL" mysql "$MYSQL_ARGS" +[ -n "${MARIADB_ARGS:-}" ] && check_server "MariaDB" mariadb "$MARIADB_ARGS" +[ -n "${TIDB_ARGS:-}" ] && check_server "TiDB" mysql "$TIDB_ARGS" + +if [ "$checked" -eq 0 ]; then + echo "No server checked: set MYSQL_ARGS or MARIADB_ARGS" >&2 + exit 3 +fi +[ "$failures" -eq 0 ] || exit 1 From edca6c7f1614209324067a78816139ef9932399e Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 22 Oct 2026 21:46:22 +0700 Subject: [PATCH 2/2] docs(changelog): add fragments for #3791 --- apple/changelog.d/3791.fixed.1.md | 1 + apple/changelog.d/3791.fixed.md | 1 + apple/changelog.d/3791.security.md | 1 + 3 files changed, 3 insertions(+) create mode 100644 apple/changelog.d/3791.fixed.1.md create mode 100644 apple/changelog.d/3791.fixed.md create mode 100644 apple/changelog.d/3791.security.md diff --git a/apple/changelog.d/3791.fixed.1.md b/apple/changelog.d/3791.fixed.1.md new file mode 100644 index 0000000000..ef88578a6d --- /dev/null +++ b/apple/changelog.d/3791.fixed.1.md @@ -0,0 +1 @@ +Importing a MySQL dump failed on a version comment holding a quoted `*/` or on mysqldump's `DELIMITER ;;` line. diff --git a/apple/changelog.d/3791.fixed.md b/apple/changelog.d/3791.fixed.md new file mode 100644 index 0000000000..5697fc147d --- /dev/null +++ b/apple/changelog.d/3791.fixed.md @@ -0,0 +1 @@ +An idle MySQL connection could be released, dropping a temporary table or lock created behind a version comment. diff --git a/apple/changelog.d/3791.security.md b/apple/changelog.d/3791.security.md new file mode 100644 index 0000000000..84c5a1157b --- /dev/null +++ b/apple/changelog.d/3791.security.md @@ -0,0 +1 @@ +Safe Mode, MCP and AI chat could run a MySQL, MariaDB or TiDB statement hidden in a version comment without asking.