diff --git a/.claude/rules/plugin-system.md b/.claude/rules/plugin-system.md index 770f38343f..1c254d78b3 100644 --- a/.claude/rules/plugin-system.md +++ b/.claude/rules/plugin-system.md @@ -55,6 +55,12 @@ TableProPluginKit is built with Library Evolution, so an already-built plugin ke - The `TablePro` scheme builds only the bundled plugins, and a registry plugin's compile error still ends in `BUILD SUCCEEDED`. Run `verify.sh plugins` (the `AllPlugins` aggregate) after any change under `apple/Plugins/`. - `apple/scripts/ci/verify-plugin-loads.sh ...` dlopens every built `.tableplugin` and names any that will not load. It does not instantiate the principal class. +## Column type corpus + +- PostgreSQL, MySQL, MariaDB, ClickHouse, DuckDB, SQL Server, Oracle and SQLite each have a corpus in `apple/Plugins/ColumnTypeCorpus/Corpus/`: a column per type the driver lists, its values and the cell each reads as. A type added to `columnTypesByCategory` needs a column there; Repo Hygiene checks it. +- After changing how one of those drivers reads, writes or exports a value, run `apple/scripts/probes/check-column-type-corpus.sh `. SQLite's corpus runs in `SQLiteDriverTests`. +- A wrong cell is fixed in the driver, never in the corpus. A defect left for later is a `known` entry naming it, which fails the run once it passes. + ## Structure - PluginKit-free logic a plugin shares with the iOS app lives in a TableProCore product, such as `TableProMySQLCore`, `TableProPostgreSQLCore` or `TableProRedisCore`. diff --git a/.github/workflows/repo-hygiene.yml b/.github/workflows/repo-hygiene.yml index 0d02be3259..2ecf1f6edf 100644 --- a/.github/workflows/repo-hygiene.yml +++ b/.github/workflows/repo-hygiene.yml @@ -117,6 +117,11 @@ jobs: - name: Check every test target is in its test plan run: python3 apple/scripts/ci/check-test-plans.py + # The corpora run by hand against servers (scripts/probes/check-column-type-corpus.sh), so a type a driver + # starts listing is caught here. + - name: Check every listed column type has a corpus column + run: python3 apple/scripts/ci/check-column-type-corpus.py + - name: Set up Go if: inputs.hana uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDatetimeFormatter.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDatetimeFormatter.swift index be2f32e977..136172be6b 100644 --- a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDatetimeFormatter.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDatetimeFormatter.swift @@ -1,11 +1,22 @@ import Foundation public enum MSSQLDatetimeFormatter { + // db-lib converts a `date` and a `time` as a datetime, so a date reads with midnight and a time with 1900-01-01: + // parts the column does not hold. Measured on SQL Server 2022. public static func reformat(_ raw: String, type: MSSQLColumnType) -> String? { - guard type.isDateOrTime else { return nil } - return parse(raw) + guard type.isDateOrTime, let text = parse(raw) else { return nil } + switch type { + case .date: + return String(text.prefix(10)) + case .time where text.hasPrefix(timeBaseDate): + return String(text.dropFirst(timeBaseDate.count)) + default: + return text + } } + private static let timeBaseDate = "1900-01-01 " + /// db-lib's text for a `datetimeoffset` is the local wall clock with no offset, which SQL Server reads back as /// +00:00. The ISO 8601 form carries the offset, so the value reads back as the same instant. public static func dateTimeOffset(_ raw: String, offsetMinutes: Int) -> String? { diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDatetimeFormatterTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDatetimeFormatterTests.swift index e67478d617..05da437bc3 100644 --- a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDatetimeFormatterTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDatetimeFormatterTests.swift @@ -49,6 +49,21 @@ struct MSSQLDatetimeFormatterTests { #expect(MSSQLDatetimeFormatter.reformat("Dec 31 2025 11:59:59:999PM", type: .dateTime2) == "2025-12-31 23:59:59.999") } + // Measured on SQL Server 2022: dbconvert writes a time(7) 23:59:59.9999999 as 1900-01-01 23:59:59.9999999. + @Test("A time reads as the time of day, without the date db-lib converts it on") + func timeDropsTheBaseDate() { + #expect(MSSQLDatetimeFormatter.reformat("1900-01-01 23:59:59.9999999", type: .time) == "23:59:59.9999999") + #expect(MSSQLDatetimeFormatter.reformat("Jan 1 1900 12:00:00:0000000AM", type: .time) == "00:00:00.0000000") + #expect(MSSQLDatetimeFormatter.reformat("1900-01-01 08:00:00", type: .dateTime) == "1900-01-01 08:00:00") + } + + // Measured on SQL Server 2022: dbconvert writes the date 0001-01-01 as Jan 1 0001 12:00:00:0000000AM. + @Test("A date reads as the date alone, without the midnight db-lib adds") + func dateDropsTheTime() { + #expect(MSSQLDatetimeFormatter.reformat("Jan 1 0001 12:00:00:0000000AM", type: .date) == "0001-01-01") + #expect(MSSQLDatetimeFormatter.reformat("9999-12-31 00:00:00", type: .date) == "9999-12-31") + } + @Test func testDateTimeOffsetKeepsAPositiveOffset() { #expect(MSSQLDatetimeFormatter.dateTimeOffset("Jan 2 2024 3:04:05:1234567AM", offsetMinutes: 330) == "2024-01-02T03:04:05.1234567+05:30") } diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseResponseClassifierTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseResponseClassifierTests.swift index 1e4154c28d..f5513f16f2 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseResponseClassifierTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseResponseClassifierTests.swift @@ -98,6 +98,19 @@ struct ClickHouseResponseClassifierTests { #expect(outcome.rows == [[.text("a\tb\nc\\d")]]) } + // Measured on ClickHouse 25.8: the String column is TSV-escaped, the compound ones are not escaped again. + @Test("A quote or backslash inside an Array, Map or Tuple keeps the escape ClickHouse wrote") + func compoundValuesKeepTheirEscapes() { + let outcome = classify( + headers: matchingFormatHeaders, + bodyText: "a\tm\tt\ts\nArray(Nullable(String))\tMap(String, String)\tTuple(UInt8, String)\tString\n" + + #"['a',NULL,'it\'s']"# + "\t" + #"{'k\'':'v\\'}"# + "\t" + #"(1,'x\'y')"# + "\t" + #"it\'s"# + "\n" + ) + #expect(outcome.rows == [[ + .text(#"['a',NULL,'it\'s']"#), .text(#"{'k\'':'v\\'}"#), .text(#"(1,'x\'y')"#), .text("it's") + ]]) + } + // MARK: - The Full Escape Table ClickHouse Writes @Test("Every escape the server emits is decoded, not left as two characters") diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseTabSeparatedRowDecoderTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseTabSeparatedRowDecoderTests.swift index 8d70d23607..103dfec5a8 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseTabSeparatedRowDecoderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseTabSeparatedRowDecoderTests.swift @@ -127,6 +127,20 @@ struct ClickHouseTabSeparatedRowDecoderTests { #expect(outcome.rows == [[.text("['a','b']"), .text("{'k':'v'}"), .text("(7,'x')")]]) } + // Measured on ClickHouse 25.8: the String column is TSV-escaped, the compound ones are not escaped again. + @Test("A quote or backslash inside an Array, Map or Tuple keeps the escape ClickHouse wrote") + func keepsEscapesInsideCompoundValues() { + var body = Data( + "tags\tattrs\tpair\tname\nArray(Nullable(String))\tMap(String, String)\tTuple(UInt8, String)\tString\n".utf8 + ) + let row = #"['a',NULL,'it\'s']"# + "\t" + #"{'k\'':'v\\'}"# + "\t" + #"(1,'x\'y')"# + "\t" + #"it\'s"# + "\n" + body.append(contentsOf: Data(row.utf8)) + let outcome = decodeAll(body) + #expect(outcome.rows == [[ + .text(#"['a',NULL,'it\'s']"#), .text(#"{'k\'':'v\\'}"#), .text(#"(1,'x\'y')"#), .text("it's") + ]]) + } + // MARK: - Chunk boundaries @Test("A row split across two chunks decodes once it is whole") diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseParameterBinding.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseParameterBinding.swift index 489a94ed98..0198bb8ae8 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseParameterBinding.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseParameterBinding.swift @@ -56,12 +56,30 @@ nonisolated internal enum ClickHouseParameterBinding { namedCount += 1 let name = "p\(namedCount)" converted.append("{\(name):String}") - paramMap[name] = parameter.asText + paramMap.updateValue(parameter.asText.map(escaped), forKey: name) } return Bound(query: converted, params: paramMap) } + // ClickHouse reads an HTTP parameter in TabSeparated's escaped form: a backslash starts an escape and a raw tab or + // newline is refused, so `C:\new` arrived as `C:`, a newline and `ew`. Measured on ClickHouse 25.8. + static func escaped(_ value: String) -> String { + var result = "" + result.unicodeScalars.reserveCapacity(value.unicodeScalars.count) + for scalar in value.unicodeScalars { + switch scalar { + case "\\": result += "\\\\" + case "\t": result += "\\t" + case "\n": result += "\\n" + case "\r": result += "\\r" + case "\0": result += "\\0" + default: result.unicodeScalars.append(scalar) + } + } + return result + } + static func hexLiteral(_ data: Data) -> String { let hex = data.map { String(format: "%02X", $0) }.joined() return "unhex('\(hex)')" diff --git a/apple/Plugins/ClickHouseDriverTests/ClickHouseParameterBindingTests.swift b/apple/Plugins/ClickHouseDriverTests/ClickHouseParameterBindingTests.swift index 4e4f2173d6..23085e5605 100644 --- a/apple/Plugins/ClickHouseDriverTests/ClickHouseParameterBindingTests.swift +++ b/apple/Plugins/ClickHouseDriverTests/ClickHouseParameterBindingTests.swift @@ -19,6 +19,17 @@ struct ClickHouseParameterBindingTests { #expect(bound.params["p2"] == "two") } + // Measured on ClickHouse 25.8: `C:\new` sent as it is arrived as `C:`, a newline and `ew`, and a raw tab was refused. + @Test("A text parameter is sent in the escaped form ClickHouse reads it in") + func textParametersAreEscaped() { + let bound = ClickHouseParameterBinding.bind( + query: "SELECT ?, ?", + parameters: [.text(#"C:\new\table"#), .text("a\tb\nc\rd\u{0}e")] + ) + #expect(bound.params["p1"] == #"C:\\new\\table"#) + #expect(bound.params["p2"] == #"a\tb\nc\rd\0e"#) + } + @Test("A null parameter keeps its slot with no value") func nullParameterKeepsItsSlot() { let bound = ClickHouseParameterBinding.bind( diff --git a/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpus.swift b/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpus.swift new file mode 100644 index 0000000000..c77d73f9d6 --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpus.swift @@ -0,0 +1,135 @@ +// +// ColumnTypeCorpus.swift +// ColumnTypeCorpus +// + +import Foundation + +struct ColumnTypeCorpus: Decodable, Sendable { + enum Step: String, CaseIterable, Sendable { + case create, insert, columns, browse, stream, edit, csv, json, sql + } + + struct Value: Decodable, Sendable { + let sql: String + // nil is SQL NULL. The key is required, so a forgotten expectation never reads as NULL. + let cell: String? + // For a value the server makes up, such as a rowversion: the cell has to match this instead. + let matches: String? + + private enum CodingKeys: String, CodingKey { case sql, cell, `repeat`, matches } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + sql = try container.decode(String.self, forKey: .sql) + matches = try container.decodeIfPresent(String.self, forKey: .matches) + if matches != nil { + cell = nil + return + } + guard container.contains(.cell) else { + throw DecodingError.keyNotFound( + CodingKeys.cell, + DecodingError.Context(codingPath: container.codingPath, debugDescription: "\(sql) has no cell") + ) + } + let written = try container.decodeIfPresent(String.self, forKey: .cell) + // A value too long to write out: `cell` repeated, as REPEAT('x', 70000) reads. + let count = try container.decodeIfPresent(Int.self, forKey: .repeat) ?? 1 + cell = written.map { String(repeating: $0, count: count) } + } + } + + struct Edit: Decodable, Sendable { + let row: Int? + let text: String? + let hex: String? + let cell: String? + } + + // A defect left for later, and what its failure looks like: any other failure at that step still fails the run. + struct KnownIssue: Decodable, Sendable { + let defect: String + let matches: String + } + + struct Column: Decodable, Sendable { + let type: String + let covers: [String] + let values: [Value] + let edit: Edit? + let skip: [Step: String] + let known: [Step: KnownIssue] + + private enum CodingKeys: String, CodingKey { case type, covers, values, edit, skip, known } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + type = try container.decode(String.self, forKey: .type) + covers = try container.decodeIfPresent([String].self, forKey: .covers) ?? [Self.baseName(of: type)] + values = try container.decode([Value].self, forKey: .values) + edit = try container.decodeIfPresent(Edit.self, forKey: .edit) + skip = try Self.steps(container.decodeIfPresent([String: String].self, forKey: .skip), in: container, key: .skip) + known = try Self.steps( + container.decodeIfPresent([String: KnownIssue].self, forKey: .known), in: container, key: .known + ) + if edit == nil, skip[.edit] == nil { + throw DecodingError.dataCorruptedError( + forKey: .edit, in: container, debugDescription: "\(type) needs an edit or a skip reason for it" + ) + } + } + + // The listed name a declaration stands for: `NUMERIC(10,2)` covers `NUMERIC`. Kept in step with + // apple/scripts/ci/check-column-type-corpus.py. + static func baseName(of type: String) -> String { + let name = type.split(separator: "(", maxSplits: 1).first.map(String.init) ?? type + return name.trimmingCharacters(in: .whitespaces) + } + + private static func steps( + _ raw: [String: Reason]?, + in container: KeyedDecodingContainer, + key: CodingKeys + ) throws -> [Step: Reason] { + var steps: [Step: Reason] = [:] + for (name, reason) in raw ?? [:] { + guard let step = Step(rawValue: name) else { + throw DecodingError.dataCorruptedError( + forKey: key, in: container, debugDescription: "\(name) is not a step" + ) + } + steps[step] = reason + } + return steps + } + } + + let engine: String + let databaseTypeId: String + let typeList: String + let createTable: String + let dropTable: String? + let clearTable: String? + // Listed types no column can have, such as SQL Server's CURSOR, with why. + let notColumnTypes: [String: String]? + // Polled after a save until it reads 0, for an engine that applies a save after answering it. + let settleAfterSave: String? + let columns: [Column] + + static func load(_ url: URL) throws -> ColumnTypeCorpus { + try JSONDecoder().decode(ColumnTypeCorpus.self, from: Data(contentsOf: url)) + } + + func uncoveredTypes(listed: [String: [String]]) -> [String] { + let covered = Set(columns.flatMap(\.covers)).union((notColumnTypes ?? [:]).keys) + return listed.values.flatMap { $0 }.filter { !covered.contains($0) }.sorted() + } + + // By what the column covers, else by the first word of its type: `TINYINT UNSIGNED` is an Integer. + func category(of column: Column, listed: [String: [String]]) -> String? { + let firstWord = Column.baseName(of: column.type).split(separator: " ").first.map(String.init) ?? column.type + let names = column.covers + [firstWord] + return listed.first { _, types in names.contains(where: types.contains) }?.key + } +} diff --git a/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusExportSource.swift b/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusExportSource.swift new file mode 100644 index 0000000000..1d6cebe13f --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusExportSource.swift @@ -0,0 +1,73 @@ +// +// ColumnTypeCorpusExportSource.swift +// ColumnTypeCorpus +// + +import Foundation +import TableProPluginKit + +// What the host's ExportDataSourceAdapter hands an export plugin for one table of the connected database. +final class ColumnTypeCorpusExportSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId: String + let lexicalFeatures: SQLLexicalFeatures + let supportsCascadeDrop: Bool + private let driver: any PluginDatabaseDriver + + init( + driver: any PluginDatabaseDriver, + databaseTypeId: String, + lexicalFeatures: SQLLexicalFeatures, + supportsCascadeDrop: Bool + ) { + self.driver = driver + self.databaseTypeId = databaseTypeId + self.lexicalFeatures = lexicalFeatures + self.supportsCascadeDrop = supportsCascadeDrop + } + + static func readQuery(table: String, driver: any PluginDatabaseDriver) -> (sql: String, context: PluginStatementContext) { + var context = PluginStatementContext() + context.readOnly = true + if let custom = driver.defaultExportQuery(table: table, schema: nil) { + return (custom, context) + } + return ("SELECT * FROM \(driver.quoteIdentifier(table))", context) + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + let read = Self.readQuery(table: table, driver: driver) + return driver.streamRows(query: read.sql, context: read.context) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + try await driver.fetchTableDDL(table: table, schema: nil) + } + + func execute(query: String) async throws -> PluginQueryResult { + try await driver.execute(query: query) + } + + func quoteIdentifier(_ identifier: String) -> String { + driver.quoteIdentifier(identifier) + } + + func escapeStringLiteral(_ value: String) -> String { + driver.escapeStringLiteral(value) + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + try await driver.fetchApproximateRowCount(table: table, schema: nil) + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + try await driver.fetchColumns(table: table, schema: nil) + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + try await driver.openSessionScope(for: purpose) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + try await driver.closeSessionScope(scope) + } +} diff --git a/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusReaders.swift b/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusReaders.swift new file mode 100644 index 0000000000..a0498151e9 --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusReaders.swift @@ -0,0 +1,272 @@ +// +// ColumnTypeCorpusReaders.swift +// ColumnTypeCorpus +// + +import Foundation + +// Read back independently of the writers, so a check never agrees with an export by sharing its code. +enum CorpusCSVReader { + static func records(_ text: String) -> [[String]] { + var records: [[String]] = [] + var record: [String] = [] + var field = "" + var quoted = false + var characters = Array(text)[...] + while let character = characters.popFirst() { + if quoted { + if character == "\"" { + if characters.first == "\"" { + field.append("\"") + characters = characters.dropFirst() + } else { + quoted = false + } + } else { + field.append(character) + } + continue + } + switch character { + case "\"" where field.isEmpty: + quoted = true + case ",": + record.append(field) + field = "" + case "\n", "\r\n": + record.append(field) + records.append(record) + record = [] + field = "" + default: + field.append(character) + } + } + if !field.isEmpty || !record.isEmpty { + record.append(field) + records.append(record) + } + return records + } +} + +indirect enum CorpusJSONValue: Equatable, Sendable { + case object([(key: String, value: CorpusJSONValue)]) + case array([CorpusJSONValue]) + case string(String) + // The literal as written, so a 78-digit integer is compared digit for digit. + case number(String) + case bool(Bool) + case null + + static func == (lhs: CorpusJSONValue, rhs: CorpusJSONValue) -> Bool { + switch (lhs, rhs) { + case (.object(let left), .object(let right)): + left.map(\.key) == right.map(\.key) && left.map(\.value) == right.map(\.value) + case (.array(let left), .array(let right)): left == right + case (.string(let left), .string(let right)): left == right + case (.number(let left), .number(let right)): left == right + case (.bool(let left), .bool(let right)): left == right + case (.null, .null): true + default: false + } + } + + subscript(key: String) -> CorpusJSONValue? { + guard case .object(let members) = self else { return nil } + return members.first { $0.key == key }?.value + } +} + +struct CorpusJSONReader { + struct Malformed: Error, CustomStringConvertible { + let offset: Int + var description: String { "malformed JSON at byte \(offset)" } + } + + private let bytes: [UInt8] + private var index = 0 + + static func parse(_ data: Data) throws -> CorpusJSONValue { + var reader = CorpusJSONReader(bytes: Array(data)) + let value = try reader.value() + reader.skipWhitespace() + guard reader.index == reader.bytes.count else { throw Malformed(offset: reader.index) } + return value + } + + private init(bytes: [UInt8]) { + self.bytes = bytes + } + + private mutating func skipWhitespace() { + while index < bytes.count, [0x20, 0x09, 0x0A, 0x0D].contains(bytes[index]) { + index += 1 + } + } + + private mutating func value() throws -> CorpusJSONValue { + skipWhitespace() + guard index < bytes.count else { throw Malformed(offset: index) } + switch bytes[index] { + case UInt8(ascii: "{"): return try object() + case UInt8(ascii: "["): return try array() + case UInt8(ascii: "\""): return .string(try string()) + case UInt8(ascii: "t"): return try literal("true", .bool(true)) + case UInt8(ascii: "f"): return try literal("false", .bool(false)) + case UInt8(ascii: "n"): return try literal("null", .null) + default: return try number() + } + } + + private mutating func literal(_ word: String, _ value: CorpusJSONValue) throws -> CorpusJSONValue { + let end = index + word.utf8.count + guard end <= bytes.count, Array(bytes[index.. CorpusJSONValue { + let start = index + func digits() -> Int { + let first = index + while index < bytes.count, (UInt8(ascii: "0")...UInt8(ascii: "9")).contains(bytes[index]) { + index += 1 + } + return index - first + } + func take(_ character: Character) -> Bool { + guard index < bytes.count, bytes[index] == character.asciiValue else { return false } + index += 1 + return true + } + _ = take("-") + let leadingZero = index < bytes.count && bytes[index] == UInt8(ascii: "0") + let whole = digits() + guard whole > 0, !(leadingZero && whole > 1) else { throw Malformed(offset: start) } + if take("."), digits() == 0 { throw Malformed(offset: start) } + if take("e") || take("E") { + if !take("+") { _ = take("-") } + guard digits() > 0 else { throw Malformed(offset: start) } + } + guard let text = String(bytes: bytes[start.. CorpusJSONValue { + index += 1 + var members: [(key: String, value: CorpusJSONValue)] = [] + skipWhitespace() + if index < bytes.count, bytes[index] == UInt8(ascii: "}") { + index += 1 + return .object(members) + } + while true { + skipWhitespace() + let key = try string() + skipWhitespace() + guard index < bytes.count, bytes[index] == UInt8(ascii: ":") else { throw Malformed(offset: index) } + index += 1 + members.append((key, try value())) + skipWhitespace() + guard index < bytes.count else { throw Malformed(offset: index) } + if bytes[index] == UInt8(ascii: ",") { + index += 1 + } else if bytes[index] == UInt8(ascii: "}") { + index += 1 + return .object(members) + } else { + throw Malformed(offset: index) + } + } + } + + private mutating func array() throws -> CorpusJSONValue { + index += 1 + var elements: [CorpusJSONValue] = [] + skipWhitespace() + if index < bytes.count, bytes[index] == UInt8(ascii: "]") { + index += 1 + return .array(elements) + } + while true { + elements.append(try value()) + skipWhitespace() + guard index < bytes.count else { throw Malformed(offset: index) } + if bytes[index] == UInt8(ascii: ",") { + index += 1 + } else if bytes[index] == UInt8(ascii: "]") { + index += 1 + return .array(elements) + } else { + throw Malformed(offset: index) + } + } + } + + private mutating func string() throws -> String { + guard index < bytes.count, bytes[index] == UInt8(ascii: "\"") else { throw Malformed(offset: index) } + index += 1 + var scalars = String.UnicodeScalarView() + var pendingHighSurrogate: UInt32? + var run: [UInt8] = [] + + func flushRun() throws { + guard !run.isEmpty else { return } + guard let text = String(bytes: run, encoding: .utf8) else { throw Malformed(offset: index) } + scalars.append(contentsOf: text.unicodeScalars) + run = [] + } + + while index < bytes.count { + let byte = bytes[index] + index += 1 + if byte == UInt8(ascii: "\"") { + try flushRun() + return String(scalars) + } + guard byte == UInt8(ascii: "\\") else { + // RFC 8259 has a control character in a string escaped, so a raw newline is a broken export. + guard byte >= 0x20 else { throw Malformed(offset: index - 1) } + run.append(byte) + continue + } + try flushRun() + guard index < bytes.count else { throw Malformed(offset: index) } + let escape = bytes[index] + index += 1 + switch escape { + case UInt8(ascii: "\""): scalars.append("\"") + case UInt8(ascii: "\\"): scalars.append("\\") + case UInt8(ascii: "/"): scalars.append("/") + case UInt8(ascii: "b"): scalars.append("\u{08}") + case UInt8(ascii: "f"): scalars.append("\u{0C}") + case UInt8(ascii: "n"): scalars.append("\n") + case UInt8(ascii: "r"): scalars.append("\r") + case UInt8(ascii: "t"): scalars.append("\t") + case UInt8(ascii: "u"): + guard index + 4 <= bytes.count, + let hex = String(bytes: bytes[index.. Void + + init(log: @escaping @Sendable (String) -> Void) { + self.log = log + } + + var failures: [String] { + entries.compactMap { entry in + guard case .failed = entry.outcome else { return nil } + return entry.line + } + } + + mutating func record(_ column: String, _ step: String, _ outcome: Outcome) { + let entry = Entry(column: column, step: step, outcome: outcome) + entries.append(entry) + log(entry.line) + } + + // A step the corpus marks known passes only while it still fails, so a fix forces the marker out. + mutating func record( + _ column: ColumnTypeCorpus.Column, + _ step: ColumnTypeCorpus.Step, + problems: [String] + ) { + let known = column.known[step] + switch (problems.isEmpty, known) { + case (true, nil): + record(column.type, step.rawValue, .passed) + case (true, let issue?): + record(column.type, step.rawValue, .failed("passes now, remove the known issue \"\(issue.defect)\"")) + case (false, nil): + record(column.type, step.rawValue, .failed(problems.joined(separator: "; "))) + case (false, let issue?): + let others = problems.filter { $0.range(of: issue.matches, options: .regularExpression) == nil } + if others.isEmpty { + record(column.type, step.rawValue, .knownIssue("\(issue.defect) (\(problems.joined(separator: "; ")))")) + } else { + record( + column.type, + step.rawValue, + .failed("not the known issue \"\(issue.defect)\": \(others.joined(separator: "; "))") + ) + } + } + } +} diff --git a/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusRunner.swift b/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusRunner.swift new file mode 100644 index 0000000000..8ea7b2b032 --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusRunner.swift @@ -0,0 +1,632 @@ +// +// ColumnTypeCorpusRunner.swift +// ColumnTypeCorpus +// + +import Foundation +import TableProPluginKit +import TableProSQLGrammar + +struct ColumnTypeCorpusExport: Sendable { + let formatId: String + let optionValues: [Bool] + let write: @Sendable (_ tables: [PluginExportTable], _ source: any PluginExportDataSource, _ destination: URL) + async throws -> Void +} + +// Takes each corpus column through what a user does with a table: open it, edit a cell, save, read it back, and +// export it as CSV, JSON and SQL. Each step goes through the driver call the host makes for it. +struct ColumnTypeCorpusRunner: Sendable { + private typealias Rows = [Int: [PluginCellValue]] + + private struct Read { + let columns: [String] + let columnTypeNames: [String] + let rows: Rows + } + + private struct StepFailure: Error { + let problems: [String] + } + + static let idColumn = "id" + static let valueColumn = "v" + private static let pageSize = 200 + + let corpus: ColumnTypeCorpus + let driver: any PluginDatabaseDriver + let listedTypes: [String: [String]] + let dialect: SQLDialectDescriptor? + let supportsCascadeDrop: Bool + let exports: [ColumnTypeCorpusExport] + let workDirectory: URL + var tablePrefix = "tpc_" + var keepTables = false + var includes: @Sendable (ColumnTypeCorpus.Column) -> Bool = { _ in true } + + private var grammar: SQLLexicalGrammar { + let declared = dialect?.lexicalFeatures.map { SQLLexicalGrammar(rawValue: $0.rawValue) } + return SQLLexicalReadings.resolve(databaseTypeId: corpus.databaseTypeId, declared: declared, session: nil) + .execution + } + + // The kit's features are the grammar facts below bit 25, bit for bit (SQLLexicalResolver.pluginFeaturePairs). + private var lexicalFeatures: SQLLexicalFeatures { + SQLLexicalFeatures(rawValue: grammar.rawValue & ((1 << 25) - 1)) + } + + func run(log: @escaping @Sendable (String) -> Void) async -> ColumnTypeCorpusReport { + var report = ColumnTypeCorpusReport(log: log) + let uncovered = corpus.uncoveredTypes(listed: listedTypes) + report.record( + corpus.engine, + "coverage", + uncovered.isEmpty ? .passed : .failed("listed with no corpus column: \(uncovered.joined(separator: ", "))") + ) + let selected = corpus.columns.enumerated().filter { includes($0.element) } + guard !selected.isEmpty else { + report.record(corpus.engine, "columns", .failed("no corpus column is selected")) + return report + } + for (index, column) in selected { + await run(column, table: "\(tablePrefix)\(index)", report: &report) + } + return report + } + + // MARK: - One column + + private func run(_ column: ColumnTypeCorpus.Column, table: String, report: inout ColumnTypeCorpusReport) async { + await dropTable(table) + await runSteps(column, table: table, report: &report) + if !keepTables { + await dropTable(table) + } + } + + private func runSteps( + _ column: ColumnTypeCorpus.Column, + table: String, + report: inout ColumnTypeCorpusReport + ) async { + do { + _ = try await driver.execute(query: createStatement(table, column: column)) + report.record(column, .create, problems: []) + } catch { + report.record(column, .create, problems: [describe(error)]) + return + } + + var insertProblems: [String] = [] + for (offset, value) in column.values.enumerated() { + do { + _ = try await driver.execute(query: insertStatement(table, id: offset + 1, valueSQL: value.sql)) + } catch { + insertProblems.append("\(value.sql): \(describe(error))") + } + } + report.record(column, .insert, problems: insertProblems) + guard insertProblems.isEmpty else { return } + + var columnInfo: [PluginColumnInfo] = [] + do { + columnInfo = try await driver.fetchColumns(table: table, schema: nil) + let names = columnInfo.map(\.name) + report.record( + column, + .columns, + problems: names.contains(Self.valueColumn) ? [] : ["the column list is \(names)"] + ) + } catch { + report.record(column, .columns, problems: [describe(error)]) + } + + var page: Read + do { + page = try await browse(table, columnNames: columnInfo.map(\.name)) + report.record(column, .browse, problems: expectedCellProblems(column, in: page)) + } catch { + report.record(column, .browse, problems: [describe(error)]) + return + } + + if let reason = column.skip[.stream] { + report.record(column.type, ColumnTypeCorpus.Step.stream.rawValue, .skipped(reason)) + } else { + do { + let streamed = try await stream(table) + report.record(column, .stream, problems: differences(streamed, from: page, what: "the export read")) + } catch { + report.record(column, .stream, problems: [describe(error)]) + } + } + + if let reason = column.skip[.edit] { + report.record(column.type, ColumnTypeCorpus.Step.edit.rawValue, .skipped(reason)) + } else if let edit = column.edit { + do { + page = try await save(edit, table: table, page: page, columnInfo: columnInfo) + report.record(column, .edit, problems: []) + } catch { + report.record(column, .edit, problems: [describe(error)]) + // The exports compare against what the table holds now, whatever the save did. + if let current = try? await browse(table, columnNames: page.columns) { + page = current + } + } + } + + let source = ColumnTypeCorpusExportSource( + driver: driver, + databaseTypeId: corpus.databaseTypeId, + lexicalFeatures: lexicalFeatures, + supportsCascadeDrop: supportsCascadeDrop + ) + let category = corpus.category(of: column, listed: listedTypes) + for export in exports { + guard let step = ColumnTypeCorpus.Step(rawValue: export.formatId) else { continue } + if let reason = column.skip[step] { + report.record(column.type, step.rawValue, .skipped(reason)) + continue + } + do { + let destination = workDirectory.appendingPathComponent("\(table).\(export.formatId)") + defer { try? FileManager.default.removeItem(at: destination) } + let exported = PluginExportTable( + name: table, + databaseName: "", + tableType: "table", + optionValues: export.optionValues, + schema: nil, + kind: .table + ) + try await export.write([exported], source, destination) + let problems: [String] + switch step { + case .csv: problems = try csvProblems(destination, page: page) + case .json: problems = try jsonProblems(destination, table: table, page: page, category: category) + case .sql: problems = try await sqlProblems(destination, table: table, page: page) + default: problems = [] + } + report.record(column, step, problems: problems) + } catch { + report.record(column, step, problems: [describe(error)]) + } + } + } + + // MARK: - Statements + + private func fill(_ template: String, table: String, column: ColumnTypeCorpus.Column? = nil) -> String { + var sql = template + .replacingOccurrences(of: "{table}", with: driver.quoteIdentifier(table)) + .replacingOccurrences(of: "{id}", with: driver.quoteIdentifier(Self.idColumn)) + .replacingOccurrences(of: "{column}", with: driver.quoteIdentifier(Self.valueColumn)) + .replacingOccurrences(of: "{tableName}", with: table) + if let column { + sql = sql.replacingOccurrences(of: "{type}", with: column.type) + } + return sql + } + + private func createStatement(_ table: String, column: ColumnTypeCorpus.Column) -> String { + fill(corpus.createTable, table: table, column: column) + } + + private func dropStatement(_ table: String) -> String { + fill(corpus.dropTable ?? "DROP TABLE IF EXISTS {table}", table: table) + } + + private func dropTable(_ table: String) async { + _ = try? await driver.execute(query: dropStatement(table)) + } + + private func insertStatement(_ table: String, id: Int, valueSQL: String) -> String { + let quoted = driver.quoteIdentifier + return "INSERT INTO \(quoted(table)) (\(quoted(Self.idColumn)), \(quoted(Self.valueColumn))) " + + "VALUES (\(id), \(valueSQL))" + } + + // The table tab sorted by its id column: the driver's own browse statement when it writes one, else the host's. + private func browseStatement(_ table: String, columnNames: [String]) -> String { + let columns = columnNames.isEmpty ? [Self.idColumn, Self.valueColumn] : columnNames + let idIndex = columns.firstIndex(of: Self.idColumn) ?? 0 + if let sql = driver.buildBrowseQuery( + table: table, schema: nil, sortColumns: [(columnIndex: idIndex, ascending: true)], + columns: columns, limit: Self.pageSize, offset: 0 + ) { + return sql + } + let quoted = driver.quoteIdentifier + let pagination = dialect?.paginationStyle == .offsetFetch + ? "OFFSET 0 ROWS FETCH NEXT \(Self.pageSize) ROWS ONLY" + : "LIMIT \(Self.pageSize) OFFSET 0" + return "SELECT * FROM \(quoted(table)) ORDER BY \(quoted(Self.idColumn)) ASC \(pagination)" + } + + private func placeholder(_ position: Int) -> String { + driver.parameterStyle == .dollar ? "$\(position)" : "?" + } + + // SQLStatementGenerator's UPDATE for a grid edit, for a driver that leaves its statements to the host. + private func hostUpdate( + table: String, + read: Read, + original: [PluginCellValue], + newValue: PluginCellValue, + primaryKey: [String] + ) -> (sql: String, parameters: [PluginCellValue]) { + let quoted = driver.quoteIdentifier + var parameters: [PluginCellValue] = [newValue] + var conditions: [String] = [] + let matched = primaryKey.isEmpty ? read.columns : primaryKey + for name in matched { + guard let index = read.columns.firstIndex(of: name), index < original.count else { continue } + if original[index].isNull { + conditions.append("\(quoted(name)) IS NULL") + } else { + parameters.append(original[index]) + conditions.append("\(quoted(name)) = \(placeholder(parameters.count))") + } + } + let sql = "UPDATE \(quoted(table)) SET \(quoted(Self.valueColumn)) = \(placeholder(1)) WHERE " + + conditions.joined(separator: " AND ") + return (sql, parameters) + } + + // MARK: - Reads + + private func rows(columns: [String], values: [[PluginCellValue]]) throws -> Rows { + guard let idIndex = columns.firstIndex(of: Self.idColumn) else { + throw StepFailure(problems: ["no \(Self.idColumn) column in \(columns)"]) + } + var rows: Rows = [:] + for row in values { + guard idIndex < row.count, let text = Self.text(of: row[idIndex]), let id = Int(text) else { + throw StepFailure(problems: ["a row without a readable id: \(row.map(Self.text(of:)))"]) + } + guard rows.updateValue(row, forKey: id) == nil else { + throw StepFailure(problems: ["row \(id) is read twice"]) + } + } + return rows + } + + private func browse(_ table: String, columnNames: [String]) async throws -> Read { + let result = try await driver.executeUserQuery( + query: browseStatement(table, columnNames: columnNames), rowCap: nil, parameters: nil + ) + return Read( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + rows: try rows(columns: result.columns, values: result.rows) + ) + } + + private func stream(_ table: String) async throws -> Read { + let read = ColumnTypeCorpusExportSource.readQuery(table: table, driver: driver) + var header: PluginStreamHeader? + var values: [[PluginCellValue]] = [] + for try await element in driver.streamRows(query: read.sql, context: read.context) { + switch element { + case .header(let streamed): header = streamed + case .rows(let rows): values.append(contentsOf: rows) + } + } + guard let header else { throw StepFailure(problems: ["no header"]) } + return Read( + columns: header.columns, + columnTypeNames: header.columnTypeNames, + rows: try rows(columns: header.columns, values: values) + ) + } + + // MARK: - Save + + private func save( + _ edit: ColumnTypeCorpus.Edit, + table: String, + page: Read, + columnInfo: [PluginColumnInfo] + ) async throws -> Read { + let rowID = edit.row ?? 1 + guard let original = page.rows[rowID], let valueIndex = page.columns.firstIndex(of: Self.valueColumn) else { + throw StepFailure(problems: ["no row \(rowID) to edit"]) + } + let newValue: PluginCellValue + if let hex = edit.hex { + guard let data = Self.data(hex: hex) else { throw StepFailure(problems: ["\(hex) is not hex"]) } + newValue = .bytes(data) + } else if let text = edit.text { + newValue = .text(text) + } else { + newValue = .null + } + + var context = PluginRowWriteContext() + context.serverOwnedColumns = Set(columnInfo.filter { $0.isGenerated || $0.isIdentity }.map(\.name)) + context.columnTypeNames = Dictionary( + zip(page.columns, page.columnTypeNames), uniquingKeysWith: { first, _ in first } + ) + let primaryKey = columnInfo.filter(\.isPrimaryKey).map(\.name) + let change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [(columnIndex: valueIndex, columnName: Self.valueColumn, oldValue: original[valueIndex], + newValue: newValue)], + originalRow: original + ) + let statements: [(sql: String, parameters: [PluginCellValue])] + if let writes = try driver.generateRowWrites( + table: table, schema: nil, columns: page.columns, primaryKeyColumns: primaryKey, changes: [change], + insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [], context: context + ) { + statements = writes.map { ($0.statement, $0.parameters) } + } else { + statements = [hostUpdate( + table: table, read: page, original: original, newValue: newValue, primaryKey: primaryKey + )] + } + + let transactional = driver.supportsTransactions + do { + if transactional { try await driver.beginTransaction() } + for statement in statements { + if statement.parameters.isEmpty { + _ = try await driver.execute(query: statement.sql) + } else { + _ = try await driver.executeParameterized(query: statement.sql, parameters: statement.parameters) + } + } + if transactional { try await driver.commitTransaction() } + } catch { + if transactional { try? await driver.rollbackTransaction() } + throw StepFailure(problems: ["\(statements.map(\.sql).joined(separator: "; ")): \(describe(error))"]) + } + + if let settle = corpus.settleAfterSave { + try await waitForZero(fill(settle, table: table)) + } + let readBack = try await browse(table, columnNames: page.columns) + var problems: [String] = [] + let edited = readBack.rows[rowID].flatMap { valueIndex < $0.count ? Self.text(of: $0[valueIndex]) : nil } + if edited != edit.cell { + problems.append("row \(rowID) reads back \(Self.show(edited)), expected \(Self.show(edit.cell))") + } + var untouched = page.rows + untouched[rowID] = readBack.rows[rowID] + problems += differences(readBack, from: Read(columns: page.columns, columnTypeNames: [], rows: untouched), + what: "after the save") + guard problems.isEmpty else { throw StepFailure(problems: problems) } + return readBack + } + + private func waitForZero(_ query: String) async throws { + let deadline = ContinuousClock.now + .seconds(60) + while ContinuousClock.now < deadline { + let result = try await driver.execute(query: query) + if result.rows.first?.first.flatMap(Self.text(of:)) == "0" { return } + try await Task.sleep(for: .milliseconds(200)) + } + throw StepFailure(problems: ["\(query) did not reach 0 in 60 seconds"]) + } + + // MARK: - Checks + + private func expectedCellProblems(_ column: ColumnTypeCorpus.Column, in read: Read) -> [String] { + guard let valueIndex = read.columns.firstIndex(of: Self.valueColumn) else { + return ["no \(Self.valueColumn) column in \(read.columns)"] + } + var problems: [String] = [] + if read.rows.count != column.values.count { + problems.append("\(read.rows.count) rows, expected \(column.values.count)") + } + for (offset, value) in column.values.enumerated() { + let cell = read.rows[offset + 1].flatMap { valueIndex < $0.count ? Self.text(of: $0[valueIndex]) : nil } + if let pattern = value.matches { + if cell?.range(of: pattern, options: .regularExpression) == nil { + problems.append("\(value.sql) reads \(Self.show(cell)), expected a match for \(pattern)") + } + } else if cell != value.cell { + problems.append("\(value.sql) reads \(Self.show(cell)), expected \(Self.show(value.cell))") + } + } + return problems + } + + private func differences(_ read: Read, from page: Read, what: String) -> [String] { + guard let valueIndex = read.columns.firstIndex(of: Self.valueColumn), + let pageIndex = page.columns.firstIndex(of: Self.valueColumn) else { + return ["\(what) has columns \(read.columns)"] + } + var problems: [String] = [] + if read.rows.count != page.rows.count { + problems.append("\(what) has \(read.rows.count) rows, the table tab \(page.rows.count)") + } + for (id, row) in page.rows.sorted(by: { $0.key < $1.key }) { + let shown = pageIndex < row.count ? Self.text(of: row[pageIndex]) : nil + let other = read.rows[id].flatMap { valueIndex < $0.count ? Self.text(of: $0[valueIndex]) : nil } + if shown != other { + problems.append("row \(id): \(what) reads \(Self.show(other)), the table tab \(Self.show(shown))") + } + } + return problems + } + + private func csvProblems(_ file: URL, page: Read) throws -> [String] { + let records = CorpusCSVReader.records(try String(contentsOf: file, encoding: .utf8)) + guard let header = records.first, + let idIndex = header.firstIndex(of: Self.idColumn), + let valueIndex = header.firstIndex(of: Self.valueColumn), + let pageIndex = page.columns.firstIndex(of: Self.valueColumn) else { + return ["header \(records.first ?? [])"] + } + var problems: [String] = [] + var seen = Set() + for record in records.dropFirst() { + guard record.count > max(idIndex, valueIndex), let id = Int(record[idIndex]), let row = page.rows[id], + seen.insert(id).inserted else { + problems.append("an unexpected or repeated record \(record)") + continue + } + let shown = Self.text(of: row[pageIndex]) ?? "" + let field = record[valueIndex] + let sanitized = shown.first.map { "=+-@\t\r".contains($0) } == true && field == "'" + shown + if field != shown, !sanitized { + problems.append("row \(id): \(Self.show(field)), the table tab \(Self.show(shown))") + } + } + problems += Self.missingRows(Set(page.rows.keys).subtracting(seen)) + return problems + } + + private func jsonProblems(_ file: URL, table: String, page: Read, category: String?) throws -> [String] { + let root = try CorpusJSONReader.parse(Data(contentsOf: file)) + guard case .array(let objects)? = root[table], let pageIndex = page.columns.firstIndex(of: Self.valueColumn) + else { + return ["no array under \(table)"] + } + var problems: [String] = [] + var seen = Set() + for object in objects { + guard let id = Self.jsonText(object[Self.idColumn]).flatMap(Int.init), let row = page.rows[id], + seen.insert(id).inserted else { + problems.append("an unexpected or repeated row \(object)") + continue + } + if let problem = Self.jsonProblem(object[Self.valueColumn], cell: row[pageIndex], category: category) { + problems.append("row \(id): \(problem)") + } + } + problems += Self.missingRows(Set(page.rows.keys).subtracting(seen)) + return problems + } + + // The dump, run into the emptied table, has to put back what the table tab showed. + private func sqlProblems(_ file: URL, table: String, page: Read) async throws -> [String] { + let dump = try String(contentsOf: file, encoding: .utf8) + let statements = SQLStatementScanner.locatedStatements(in: dump, grammar: grammar) + .filter(\.hasContent) + .map { SQLStatementSplitting.stripLeadingComments($0.sql, lexicalFeatures: lexicalFeatures) } + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .map { $0.hasSuffix(";") ? String($0.dropLast()) : $0 } + .filter { !$0.isEmpty } + guard !statements.isEmpty else { return ["the dump has no statement"] } + _ = try await driver.execute(query: fill(corpus.clearTable ?? "DELETE FROM {table}", table: table)) + for statement in statements { + do { + _ = try await driver.execute(query: statement) + } catch { + return ["replaying \(statement.prefix(200)): \(describe(error))"] + } + } + let replayed = try await browse(table, columnNames: page.columns) + return differences(replayed, from: page, what: "the replayed dump") + } + + private static func missingRows(_ ids: Set) -> [String] { + ids.isEmpty ? [] : ["rows \(ids.sorted()) are missing"] + } + + // MARK: - Values + + static func text(of cell: PluginCellValue) -> String? { + switch cell { + case .null: nil + case .text(let text): text + case .bytes(let data): "0x" + data.map { String(format: "%02X", $0) }.joined() + } + } + + private static func show(_ text: String?) -> String { + guard let text else { return "NULL" } + guard text.count > 80 else { return String(reflecting: text) } + return String(reflecting: String(text.prefix(60))) + "… (\(text.count) characters)" + } + + static func data(hex: String) -> Data? { + let digits = Array(hex.hasPrefix("0x") ? String(hex.dropFirst(2)) : hex) + guard digits.count.isMultiple(of: 2) else { return nil } + var data = Data() + for index in stride(from: 0, to: digits.count, by: 2) { + guard let byte = UInt8(String(digits[index...index + 1]), radix: 16) else { return nil } + data.append(byte) + } + return data + } + + private static func jsonText(_ value: CorpusJSONValue?) -> String? { + switch value { + case .string(let text)?: text + case .number(let literal)?: literal + default: nil + } + } + + private static func jsonProblem(_ value: CorpusJSONValue?, cell: PluginCellValue, category: String?) -> String? { + switch (cell, value) { + case (.null, nil), (.null, .null?): + return nil + case (.bytes(let data), .string(let text)?) where Data(base64Encoded: text) == data: + return nil + case (.text(let text), .string(let written)?) where written == text: + return nil + case (.text(let text), .number(let literal)?) where ["Integer", "Float", "Boolean"].contains(category ?? ""): + return sameNumber(literal, text) ? nil : "the number \(literal), the table tab \(show(text))" + case (.text(let text), .bool(let flag)?) where category == "Boolean": + return booleanText(text) == flag ? nil : "\(flag), the table tab \(show(text))" + default: + return "\(value.map { "\($0)" } ?? "missing"), the table tab \(show(text(of: cell)))" + } + } + + private static func booleanText(_ text: String) -> Bool? { + switch text.lowercased() { + case "true", "t", "1", "yes", "on": true + case "false", "f", "0", "no", "off": false + default: nil + } + } + + // Equal when both spell the same value: 1.50 and 1.5, 1.0e-06 and -1e-6 written either way, 1e+308 and 1e308. + private static func sameNumber(_ literal: String, _ text: String) -> Bool { + guard let left = decimalForm(literal), let right = decimalForm(text) else { return literal == text } + return left == right + } + + // Sign, significant digits and the exponent of the last one, or nil for text that is not a decimal number. + private static func decimalForm(_ number: String) -> String? { + var rest = Substring(number) + var negative = false + if let sign = rest.first, sign == "-" || sign == "+" { + negative = sign == "-" + rest = rest.dropFirst() + } + var exponent = 0 + if let marker = rest.firstIndex(where: { $0 == "e" || $0 == "E" }) { + guard let value = Int(rest[rest.index(after: marker)...]) else { return nil } + exponent = value + rest = rest[.. 1 ? String(parts[1]) : "" + guard !(whole + fraction).isEmpty, (whole + fraction).allSatisfy(\.isASCII), (whole + fraction).allSatisfy(\.isNumber) + else { return nil } + var digits = Substring(whole + fraction) + exponent -= fraction.count + while digits.count > 1, digits.first == "0" { digits = digits.dropFirst() } + while digits.count > 1, digits.last == "0" { + digits = digits.dropLast() + exponent += 1 + } + if digits == "0" { return "0" } + return "\(negative ? "-" : "")\(digits)e\(exponent)" + } + + private func describe(_ error: Error) -> String { + if let failure = error as? StepFailure { return failure.problems.joined(separator: "; ") } + let localized = (error as? LocalizedError)?.errorDescription + return localized ?? String(describing: error) + } +} diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/clickhouse.json b/apple/Plugins/ColumnTypeCorpus/Corpus/clickhouse.json new file mode 100644 index 0000000000..2d6051b794 --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/clickhouse.json @@ -0,0 +1,252 @@ +{ + "engine": "clickhouse", + "databaseTypeId": "ClickHouse", + "typeList": "Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift", + "createTable": "CREATE TABLE {table} ({id} Int32, {column} {type}) ENGINE = MergeTree ORDER BY {id}", + "clearTable": "TRUNCATE TABLE {table}", + "settleAfterSave": "SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = '{tableName}' AND NOT is_done", + "columns": [ + { + "type": "UInt8", + "values": [{"sql": "0", "cell": "0"}, {"sql": "255", "cell": "255"}], + "edit": {"text": "12", "cell": "12"} + }, + { + "type": "UInt16", + "values": [{"sql": "65535", "cell": "65535"}], + "edit": {"text": "1", "cell": "1"} + }, + { + "type": "UInt32", + "values": [{"sql": "4294967295", "cell": "4294967295"}], + "edit": {"text": "2", "cell": "2"} + }, + { + "type": "UInt64", + "values": [{"sql": "18446744073709551615", "cell": "18446744073709551615"}], + "edit": {"text": "18446744073709551614", "cell": "18446744073709551614"} + }, + { + "type": "UInt128", + "values": [ + {"sql": "340282366920938463463374607431768211455", "cell": "340282366920938463463374607431768211455"} + ], + "edit": {"text": "3", "cell": "3"} + }, + { + "type": "UInt256", + "values": [ + {"sql": "115792089237316195423570985008687907853269984665640564039457584007913129639935", "cell": "115792089237316195423570985008687907853269984665640564039457584007913129639935"} + ], + "edit": {"text": "4", "cell": "4"} + }, + { + "type": "Int8", + "values": [{"sql": "-128", "cell": "-128"}, {"sql": "127", "cell": "127"}], + "edit": {"text": "-5", "cell": "-5"} + }, + { + "type": "Int16", + "values": [{"sql": "-32768", "cell": "-32768"}], + "edit": {"text": "6", "cell": "6"} + }, + { + "type": "Int32", + "values": [{"sql": "-2147483648", "cell": "-2147483648"}], + "edit": {"text": "7", "cell": "7"} + }, + { + "type": "Int64", + "values": [{"sql": "-9223372036854775808", "cell": "-9223372036854775808"}], + "edit": {"text": "9223372036854775807", "cell": "9223372036854775807"} + }, + { + "type": "Int128", + "values": [ + {"sql": "-170141183460469231731687303715884105728", "cell": "-170141183460469231731687303715884105728"} + ], + "edit": {"text": "-8", "cell": "-8"} + }, + { + "type": "Int256", + "values": [ + {"sql": "-57896044618658097711785492504343953926634992332820282019728792003956564819968", "cell": "-57896044618658097711785492504343953926634992332820282019728792003956564819968"} + ], + "edit": {"text": "9", "cell": "9"} + }, + { + "type": "Nullable(Int32)", + "covers": [], + "values": [{"sql": "NULL", "cell": null}, {"sql": "1", "cell": "1"}], + "edit": {"row": 2, "cell": null} + }, + { + "type": "Float32", + "values": [{"sql": "1.5", "cell": "1.5"}, {"sql": "-inf", "cell": "-inf"}, {"sql": "nan", "cell": "nan"}], + "edit": {"text": "2.25", "cell": "2.25"} + }, + { + "type": "Float64", + "values": [{"sql": "0.1", "cell": "0.1"}, {"sql": "1.7976931348623157e308", "cell": "1.7976931348623157e308"}], + "edit": {"text": "-0.5", "cell": "-0.5"} + }, + { + "type": "Decimal(38, 10)", + "values": [{"sql": "12345.6789", "cell": "12345.6789"}, {"sql": "-0.0000000001", "cell": "-0.0000000001"}], + "edit": {"text": "1.10", "cell": "1.1"} + }, + { + "type": "Decimal32(2)", + "values": [{"sql": "1.5", "cell": "1.5"}], + "edit": {"text": "-99.99", "cell": "-99.99"} + }, + { + "type": "Decimal64(4)", + "values": [{"sql": "123456789.1234", "cell": "123456789.1234"}], + "edit": {"text": "0", "cell": "0"} + }, + { + "type": "Decimal128(20)", + "values": [{"sql": "toDecimal128('1.00000000000000000001', 20)", "cell": "1.00000000000000000001"}], + "edit": {"text": "2.5", "cell": "2.5"} + }, + { + "type": "Decimal256(40)", + "values": [{"sql": "toDecimal256('0.1', 40)", "cell": "0.1"}], + "edit": {"text": "3", "cell": "3"} + }, + { + "type": "String", + "values": [ + {"sql": "''", "cell": ""}, + {"sql": "'Ünïcödé ✓ 日本語 🦄'", "cell": "Ünïcödé ✓ 日本語 🦄"}, + {"sql": "'line one\\nline two\\ttab'", "cell": "line one\nline two\ttab"}, + {"sql": "'back\\\\slash ''quote'' \"double\"'", "cell": "back\\slash 'quote' \"double\""}, + {"sql": "'true'", "cell": "true"}, + {"sql": "'=1+1'", "cell": "=1+1"} + ], + "edit": {"row": 3, "text": "C:\\new\\table", "cell": "C:\\new\\table"} + }, + { + "type": "Nullable(String)", + "covers": [], + "values": [{"sql": "NULL", "cell": null}, {"sql": "'x'", "cell": "x"}], + "edit": {"text": "y", "cell": "y"} + }, + { + "type": "LowCardinality(String)", + "covers": [], + "values": [{"sql": "'low'", "cell": "low"}], + "edit": {"text": "card", "cell": "card"} + }, + { + "type": "FixedString(4)", + "values": [{"sql": "'abcd'", "cell": "abcd"}], + "edit": {"text": "wxyz", "cell": "wxyz"} + }, + { + "type": "Enum8('small' = 1, 'large' = 2, 'it''s' = -1)", + "covers": ["Enum8"], + "values": [{"sql": "'small'", "cell": "small"}, {"sql": "'it''s'", "cell": "it's"}], + "edit": {"text": "large", "cell": "large"} + }, + { + "type": "Enum16('a' = 1000, 'b' = -1000)", + "covers": ["Enum16"], + "values": [{"sql": "'a'", "cell": "a"}], + "edit": {"text": "b", "cell": "b"} + }, + { + "type": "Date", + "values": [{"sql": "'1970-01-01'", "cell": "1970-01-01"}, {"sql": "'2149-06-06'", "cell": "2149-06-06"}], + "edit": {"text": "2000-02-29", "cell": "2000-02-29"} + }, + { + "type": "Date32", + "values": [{"sql": "'1900-01-01'", "cell": "1900-01-01"}, {"sql": "'2299-12-31'", "cell": "2299-12-31"}], + "edit": {"text": "2026-10-20", "cell": "2026-10-20"} + }, + { + "type": "DateTime", + "values": [{"sql": "'2026-10-20 15:22:20'", "cell": "2026-10-20 15:22:20"}], + "edit": {"text": "1999-12-31 23:59:59", "cell": "1999-12-31 23:59:59"} + }, + { + "type": "DateTime64(3)", + "values": [{"sql": "'2026-10-20 15:22:20.123'", "cell": "2026-10-20 15:22:20.123"}], + "edit": {"text": "2000-01-01 00:00:00.5", "cell": "2000-01-01 00:00:00.500"} + }, + { + "type": "Bool", + "values": [{"sql": "true", "cell": "true"}, {"sql": "false", "cell": "false"}], + "edit": {"text": "false", "cell": "false"} + }, + { + "type": "JSON", + "values": [{"sql": "'{\"a\": 1, \"b\": {\"c\": \"x\"}}'", "cell": "{\"a\":1,\"b\":{\"c\":\"x\"}}"}], + "edit": {"text": "{\"k\": \"v\"}", "cell": "{\"k\":\"v\"}"}, + "known": { + "edit": {"defect": "the save matches the row on every column, and a JSON value never equals the text it was read as, so the save changes nothing", "matches": "row 1 reads back"} + } + }, + { + "type": "UUID", + "values": [{"sql": "'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'", "cell": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}], + "edit": {"text": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12", "cell": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12"} + }, + { + "type": "Array(Int32)", + "values": [{"sql": "[1, 2, 3]", "cell": "[1,2,3]"}, {"sql": "[]", "cell": "[]"}], + "edit": {"text": "[4,5]", "cell": "[4,5]"} + }, + { + "type": "Array(Nullable(String))", + "covers": [], + "values": [{"sql": "['a', NULL, 'it''s']", "cell": "['a',NULL,'it\\'s']"}], + "edit": {"text": "['x']", "cell": "['x']"} + }, + { + "type": "Map(String, UInt64)", + "values": [{"sql": "map('a', 1, 'b', 2)", "cell": "{'a':1,'b':2}"}], + "edit": {"text": "{'c':3}", "cell": "{'c':3}"}, + "known": { + "edit": {"defect": "the grid binds the new value as a String parameter, which ClickHouse cannot cast to a Map (53)", "matches": "Code: 53\\."} + } + }, + { + "type": "Tuple(Int32, String)", + "values": [{"sql": "(1, 'x')", "cell": "(1,'x')"}], + "edit": {"text": "(2,'y')", "cell": "(2,'y')"} + }, + { + "type": "IPv4", + "values": [{"sql": "'127.0.0.1'", "cell": "127.0.0.1"}], + "edit": {"text": "10.0.0.1", "cell": "10.0.0.1"} + }, + { + "type": "IPv6", + "values": [{"sql": "'::1'", "cell": "::1"}], + "edit": {"text": "2001:db8::1", "cell": "2001:db8::1"} + }, + { + "type": "Point", + "values": [{"sql": "(1.5, 2)", "cell": "(1.5,2)"}], + "edit": {"text": "(0,-1)", "cell": "(0,-1)"} + }, + { + "type": "Ring", + "values": [{"sql": "[(0, 0), (1, 1)]", "cell": "[(0,0),(1,1)]"}], + "edit": {"text": "[(2,2)]", "cell": "[(2,2)]"} + }, + { + "type": "Polygon", + "values": [{"sql": "[[(0, 0), (1, 0), (1, 1)]]", "cell": "[[(0,0),(1,0),(1,1)]]"}], + "edit": {"text": "[[(0,0),(2,0),(2,2)]]", "cell": "[[(0,0),(2,0),(2,2)]]"} + }, + { + "type": "MultiPolygon", + "values": [{"sql": "[[[(0, 0), (1, 0), (1, 1)]]]", "cell": "[[[(0,0),(1,0),(1,1)]]]"}], + "edit": {"text": "[[[(0,0),(3,0),(3,3)]]]", "cell": "[[[(0,0),(3,0),(3,3)]]]"} + } + ] +} diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/duckdb.json b/apple/Plugins/ColumnTypeCorpus/Corpus/duckdb.json new file mode 100644 index 0000000000..3c0d6463d7 --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/duckdb.json @@ -0,0 +1,239 @@ +{ + "engine": "duckdb", + "databaseTypeId": "DuckDB", + "typeList": "Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift", + "createTable": "CREATE TABLE {table} ({id} INTEGER PRIMARY KEY, {column} {type})", + "columns": [ + { + "type": "TINYINT", + "values": [{"sql": "NULL", "cell": null}, {"sql": "-128", "cell": "-128"}, {"sql": "127", "cell": "127"}], + "edit": {"text": "12", "cell": "12"} + }, + { + "type": "SMALLINT", + "values": [{"sql": "-32768", "cell": "-32768"}], + "edit": {"text": "1", "cell": "1"} + }, + { + "type": "INTEGER", + "values": [{"sql": "2147483647", "cell": "2147483647"}], + "edit": {"text": "-2", "cell": "-2"} + }, + { + "type": "BIGINT", + "values": [{"sql": "-9223372036854775808", "cell": "-9223372036854775808"}], + "edit": {"text": "9223372036854775807", "cell": "9223372036854775807"} + }, + { + "type": "HUGEINT", + "values": [ + {"sql": "170141183460469231731687303715884105727", "cell": "170141183460469231731687303715884105727"} + ], + "edit": {"text": "-3", "cell": "-3"} + }, + { + "type": "UTINYINT", + "values": [{"sql": "255", "cell": "255"}], + "edit": {"text": "4", "cell": "4"} + }, + { + "type": "USMALLINT", + "values": [{"sql": "65535", "cell": "65535"}], + "edit": {"text": "5", "cell": "5"} + }, + { + "type": "UINTEGER", + "values": [{"sql": "4294967295", "cell": "4294967295"}], + "edit": {"text": "6", "cell": "6"} + }, + { + "type": "UBIGINT", + "values": [{"sql": "18446744073709551615", "cell": "18446744073709551615"}], + "edit": {"text": "7", "cell": "7"} + }, + { + "type": "UHUGEINT", + "covers": [], + "values": [ + {"sql": "340282366920938463463374607431768211455", "cell": "340282366920938463463374607431768211455"} + ], + "edit": {"text": "8", "cell": "8"} + }, + { + "type": "FLOAT", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "1.5", "cell": "1.5"}, + {"sql": "'-inf'", "cell": "-inf"}, + {"sql": "'nan'", "cell": "nan"} + ], + "edit": {"text": "2.25", "cell": "2.25"} + }, + { + "type": "DOUBLE", + "values": [ + {"sql": "0.1", "cell": "0.1"}, + {"sql": "1.7976931348623157e308", "cell": "1.7976931348623157e+308"} + ], + "edit": {"text": "-0.5", "cell": "-0.5"} + }, + { + "type": "DECIMAL(38,10)", + "covers": ["DECIMAL"], + "values": [ + {"sql": "1234567890123456789012345678.0123456789", "cell": "1234567890123456789012345678.0123456789"}, + {"sql": "-0.5", "cell": "-0.5000000000"} + ], + "edit": {"text": "1.1", "cell": "1.1000000000"} + }, + { + "type": "NUMERIC(10,2)", + "covers": ["NUMERIC"], + "values": [{"sql": "123.4", "cell": "123.40"}], + "edit": {"text": "99.99", "cell": "99.99"} + }, + { + "type": "VARCHAR", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "''", "cell": ""}, + {"sql": "'Ünïcödé ✓ 日本語 🦄'", "cell": "Ünïcödé ✓ 日本語 🦄"}, + {"sql": "'line one' || chr(10) || 'line two'", "cell": "line one\nline two"}, + {"sql": "'back\\slash ''quote'' \"double\"'", "cell": "back\\slash 'quote' \"double\""}, + {"sql": "'true'", "cell": "true"}, + {"sql": "'=1+1'", "cell": "=1+1"} + ], + "edit": {"text": "edited, \"quoted\"", "cell": "edited, \"quoted\""} + }, + { + "type": "TEXT", + "values": [{"sql": "'text'", "cell": "text"}], + "edit": {"text": "", "cell": ""} + }, + { + "type": "CHAR", + "values": [{"sql": "'c'", "cell": "c"}], + "edit": {"text": "d", "cell": "d"} + }, + { + "type": "BPCHAR", + "values": [{"sql": "'bp'", "cell": "bp"}], + "edit": {"text": "q", "cell": "q"} + }, + { + "type": "DATE", + "values": [ + {"sql": "'2026-10-20'", "cell": "2026-10-20"}, + {"sql": "'0001-01-01'", "cell": "0001-01-01"}, + {"sql": "'infinity'", "cell": "infinity"} + ], + "edit": {"text": "2000-02-29", "cell": "2000-02-29"} + }, + { + "type": "TIME", + "values": [{"sql": "'23:59:59.999999'", "cell": "23:59:59.999999"}], + "edit": {"text": "12:00:00", "cell": "12:00:00"} + }, + { + "type": "TIMESTAMP", + "values": [{"sql": "'2026-10-20 15:22:20.123456'", "cell": "2026-10-20 15:22:20.123456"}], + "edit": {"text": "1999-12-31 23:59:59", "cell": "1999-12-31 23:59:59"} + }, + { + "type": "TIMESTAMPTZ", + "values": [{"sql": "'2026-10-20 15:22:20+00'", "cell": "2026-10-20 15:22:20+00"}], + "edit": {"text": "2000-01-01 00:00:00+00", "cell": "2000-01-01 00:00:00+00"} + }, + { + "type": "TIMESTAMP_S", + "values": [{"sql": "'2026-10-20 15:22:20'", "cell": "2026-10-20 15:22:20"}], + "edit": {"text": "2000-01-01 00:00:01", "cell": "2000-01-01 00:00:01"} + }, + { + "type": "TIMESTAMP_MS", + "values": [{"sql": "'2026-10-20 15:22:20.123'", "cell": "2026-10-20 15:22:20.123"}], + "edit": {"text": "2000-01-01 00:00:00.5", "cell": "2000-01-01 00:00:00.5"} + }, + { + "type": "TIMESTAMP_NS", + "values": [{"sql": "'2026-10-20 15:22:20.123456789'", "cell": "2026-10-20 15:22:20.123456789"}], + "edit": {"text": "2000-01-01 00:00:00.000000001", "cell": "2000-01-01 00:00:00.000000001"} + }, + { + "type": "INTERVAL", + "values": [{"sql": "INTERVAL '1 year 2 months 3 days 04:05:06'", "cell": "1 year 2 months 3 days 04:05:06"}], + "edit": {"text": "2 hours", "cell": "02:00:00"} + }, + { + "type": "BLOB", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'\\xAA\\xBB'::BLOB", "cell": "0xAABB"}, + {"sql": "''::BLOB", "cell": "0x"} + ], + "edit": {"hex": "DEADBEEF", "cell": "0xDEADBEEF"} + }, + { + "type": "BYTEA", + "values": [{"sql": "'\\x00\\xFF'::BLOB", "cell": "0x00FF"}], + "edit": {"hex": "01", "cell": "0x01"} + }, + { + "type": "BIT", + "values": [{"sql": "'101010'::BIT", "cell": "101010"}], + "edit": {"text": "1111", "cell": "1111"} + }, + { + "type": "BITSTRING", + "values": [{"sql": "'0001'::BITSTRING", "cell": "0001"}], + "edit": {"text": "10", "cell": "10"} + }, + { + "type": "BOOLEAN", + "values": [{"sql": "NULL", "cell": null}, {"sql": "true", "cell": "true"}, {"sql": "false", "cell": "false"}], + "edit": {"text": "false", "cell": "false"} + }, + { + "type": "JSON", + "values": [{"sql": "'{\"b\": 1, \"a\": [2, 3]}'", "cell": "{\"b\": 1, \"a\": [2, 3]}"}], + "edit": {"text": "[1,2]", "cell": "[1,2]"} + }, + { + "type": "UUID", + "values": [{"sql": "'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'", "cell": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}], + "edit": {"text": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12", "cell": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12"} + }, + { + "type": "INTEGER[]", + "covers": ["LIST"], + "values": [{"sql": "[1, 2, NULL]", "cell": "[1, 2, NULL]"}, {"sql": "[]", "cell": "[]"}], + "edit": {"text": "[4, 5]", "cell": "[4, 5]"} + }, + { + "type": "VARCHAR[][]", + "covers": [], + "values": [{"sql": "[['a', 'b'], ['it''s']]", "cell": "[[a, b], ['it\\'s']]"}], + "edit": {"text": "[[x]]", "cell": "[[x]]"} + }, + { + "type": "STRUCT(a INTEGER, b VARCHAR)", + "values": [{"sql": "{'a': 1, 'b': 'x'}", "cell": "{'a': 1, 'b': x}"}], + "edit": {"text": "{'a': 2, 'b': 'y'}", "cell": "{'a': 2, 'b': y}"} + }, + { + "type": "MAP(VARCHAR, INTEGER)", + "values": [{"sql": "MAP {'a': 1, 'b': 2}", "cell": "{a=1, b=2}"}], + "edit": {"text": "{c=3}", "cell": "{c=3}"} + }, + { + "type": "UNION(num INTEGER, str VARCHAR)", + "values": [{"sql": "union_value(num := 2)", "cell": "2"}, {"sql": "union_value(str := 'x')", "cell": "x"}], + "edit": {"text": "9", "cell": "9"} + }, + { + "type": "ENUM('sad', 'ok', 'happy')", + "values": [{"sql": "'sad'", "cell": "sad"}, {"sql": "NULL", "cell": null}], + "edit": {"text": "happy", "cell": "happy"} + } + ] +} diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/mariadb.json b/apple/Plugins/ColumnTypeCorpus/Corpus/mariadb.json new file mode 100644 index 0000000000..5913fc3011 --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/mariadb.json @@ -0,0 +1,305 @@ +{ + "engine": "mariadb", + "databaseTypeId": "MariaDB", + "typeList": "Plugins/MySQLDriverPlugin/MySQLPlugin.swift", + "createTable": "CREATE TABLE {table} ({id} INT PRIMARY KEY, {column} {type})", + "columns": [ + { + "type": "TINYINT", + "values": [{"sql": "NULL", "cell": null}, {"sql": "-128", "cell": "-128"}, {"sql": "127", "cell": "127"}], + "edit": {"text": "12", "cell": "12"} + }, + { + "type": "TINYINT UNSIGNED", + "covers": [], + "values": [{"sql": "255", "cell": "255"}], + "edit": {"text": "0", "cell": "0"} + }, + { + "type": "SMALLINT", + "values": [{"sql": "-32768", "cell": "-32768"}, {"sql": "32767", "cell": "32767"}], + "edit": {"text": "1", "cell": "1"} + }, + { + "type": "MEDIUMINT", + "values": [{"sql": "-8388608", "cell": "-8388608"}, {"sql": "16777215 - 8388608", "cell": "8388607"}], + "edit": {"text": "2", "cell": "2"} + }, + { + "type": "INT", + "values": [{"sql": "-2147483648", "cell": "-2147483648"}, {"sql": "2147483647", "cell": "2147483647"}], + "edit": {"text": "3", "cell": "3"} + }, + { + "type": "INTEGER", + "values": [{"sql": "NULL", "cell": null}, {"sql": "0", "cell": "0"}], + "edit": {"text": "-4", "cell": "-4"} + }, + { + "type": "BIGINT", + "values": [ + {"sql": "-9223372036854775808", "cell": "-9223372036854775808"}, + {"sql": "9223372036854775807", "cell": "9223372036854775807"} + ], + "edit": {"text": "42", "cell": "42"} + }, + { + "type": "BIGINT UNSIGNED", + "covers": [], + "values": [{"sql": "18446744073709551615", "cell": "18446744073709551615"}], + "edit": {"text": "18446744073709551614", "cell": "18446744073709551614"} + }, + { + "type": "FLOAT", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "1.5", "cell": "1.5"}, + {"sql": "-3.4e38", "cell": "-3.4e38"} + ], + "edit": {"text": "2.25", "cell": "2.25"} + }, + { + "type": "DOUBLE", + "values": [{"sql": "0.1", "cell": "0.1"}, {"sql": "1.7976931348623157e308", "cell": "1.7976931348623157e308"}], + "edit": {"text": "-0.5", "cell": "-0.5"} + }, + { + "type": "REAL", + "values": [{"sql": "2.5", "cell": "2.5"}], + "edit": {"text": "1e-10", "cell": "0.0000000001"} + }, + { + "type": "DECIMAL(65,30)", + "covers": ["DECIMAL"], + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "12345678901234567890.123456789", "cell": "12345678901234567890.123456789000000000000000000000"}, + {"sql": "-0.5", "cell": "-0.500000000000000000000000000000"} + ], + "edit": {"text": "1.1", "cell": "1.100000000000000000000000000000"} + }, + { + "type": "NUMERIC(10,2)", + "covers": ["NUMERIC"], + "values": [{"sql": "123.4", "cell": "123.40"}], + "edit": {"text": "99.99", "cell": "99.99"} + }, + { + "type": "CHAR(5)", + "covers": ["CHAR"], + "values": [{"sql": "''", "cell": ""}, {"sql": "'ab'", "cell": "ab"}], + "edit": {"text": "xyz", "cell": "xyz"} + }, + { + "type": "VARCHAR(50)", + "covers": ["VARCHAR"], + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "''", "cell": ""}, + {"sql": "'Ünïcödé ✓ 日本語 🦄'", "cell": "Ünïcödé ✓ 日本語 🦄"}, + {"sql": "'back\\\\slash ''quote'' \"double\"'", "cell": "back\\slash 'quote' \"double\""}, + {"sql": "'line one\\nline two'", "cell": "line one\nline two"}, + {"sql": "'true'", "cell": "true"}, + {"sql": "'=1+1'", "cell": "=1+1"} + ], + "edit": {"text": "edited, \"quoted\" \\ back", "cell": "edited, \"quoted\" \\ back"} + }, + { + "type": "TINYTEXT", + "values": [{"sql": "'tiny'", "cell": "tiny"}], + "edit": {"text": "t", "cell": "t"} + }, + { + "type": "TEXT", + "values": [{"sql": "'text'", "cell": "text"}], + "edit": {"text": "", "cell": ""} + }, + { + "type": "MEDIUMTEXT", + "values": [{"sql": "'medium'", "cell": "medium"}], + "edit": {"text": "m", "cell": "m"} + }, + { + "type": "LONGTEXT", + "values": [{"sql": "REPEAT('x', 70000)", "cell": "x", "repeat": 70000}], + "edit": {"text": "long", "cell": "long"} + }, + { + "type": "ENUM('small','large','it''s')", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'small'", "cell": "small"}, + {"sql": "'it''s'", "cell": "it's"} + ], + "edit": {"text": "large", "cell": "large"} + }, + { + "type": "SET('red','green','blue')", + "values": [{"sql": "''", "cell": ""}, {"sql": "'blue,red'", "cell": "red,blue"}], + "edit": {"text": "green,red", "cell": "red,green"} + }, + { + "type": "DATE", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'1000-01-01'", "cell": "1000-01-01"}, + {"sql": "'9999-12-31'", "cell": "9999-12-31"} + ], + "edit": {"text": "2000-02-29", "cell": "2000-02-29"} + }, + { + "type": "TIME(6)", + "covers": ["TIME"], + "values": [ + {"sql": "'-838:59:59'", "cell": "-838:59:59.000000"}, + {"sql": "'838:59:59'", "cell": "838:59:59.000000"}, + {"sql": "'12:34:56.789012'", "cell": "12:34:56.789012"} + ], + "edit": {"text": "01:02:03", "cell": "01:02:03.000000"} + }, + { + "type": "DATETIME(6)", + "covers": ["DATETIME"], + "values": [ + {"sql": "'1000-01-01 00:00:00'", "cell": "1000-01-01 00:00:00.000000"}, + {"sql": "'9999-12-31 23:59:59.999999'", "cell": "9999-12-31 23:59:59.999999"} + ], + "edit": {"text": "2026-10-20 15:22:20", "cell": "2026-10-20 15:22:20.000000"} + }, + { + "type": "TIMESTAMP", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'1970-01-01 00:00:01'", "cell": "1970-01-01 00:00:01"}, + {"sql": "'2038-01-19 03:14:07'", "cell": "2038-01-19 03:14:07"} + ], + "edit": {"text": "2026-10-20 15:22:20", "cell": "2026-10-20 15:22:20"} + }, + { + "type": "YEAR", + "values": [{"sql": "1901", "cell": "1901"}, {"sql": "2155", "cell": "2155"}], + "edit": {"text": "2026", "cell": "2026"} + }, + { + "type": "BINARY(4)", + "covers": ["BINARY"], + "values": [{"sql": "NULL", "cell": null}, {"sql": "X'00FF'", "cell": "0x00FF0000"}], + "edit": {"hex": "DEADBEEF", "cell": "0xDEADBEEF"} + }, + { + "type": "VARBINARY(16)", + "covers": ["VARBINARY"], + "values": [{"sql": "X''", "cell": "0x"}, {"sql": "X'00FF7F80'", "cell": "0x00FF7F80"}], + "edit": {"hex": "0102", "cell": "0x0102"} + }, + { + "type": "TINYBLOB", + "values": [{"sql": "X'01'", "cell": "0x01"}], + "edit": {"hex": "02", "cell": "0x02"} + }, + { + "type": "BLOB", + "values": [{"sql": "X'48656C6C6F'", "cell": "0x48656C6C6F"}], + "edit": {"hex": "FF", "cell": "0xFF"} + }, + { + "type": "MEDIUMBLOB", + "values": [{"sql": "X'00'", "cell": "0x00"}], + "edit": {"hex": "0A0D", "cell": "0x0A0D"} + }, + { + "type": "LONGBLOB", + "values": [{"sql": "X'CAFE'", "cell": "0xCAFE"}], + "edit": {"hex": "BABE", "cell": "0xBABE"} + }, + { + "type": "BIT(8)", + "covers": ["BIT"], + "values": [{"sql": "NULL", "cell": null}, {"sql": "b'10101010'", "cell": "170"}, {"sql": "b'1'", "cell": "1"}], + "edit": {"row": 2, "text": "5", "cell": "5"}, + "known": { + "edit": {"defect": "the grid saves a BIT cell as text, and the server stores the text's bytes: 5 saves as 53", "matches": "reads back \"53\""}, + "sql": {"defect": "the dump writes a BIT value as quoted text, which restores as the text's bytes", "matches": "the replayed dump reads"} + } + }, + { + "type": "BOOLEAN", + "values": [{"sql": "TRUE", "cell": "1"}, {"sql": "FALSE", "cell": "0"}], + "edit": {"text": "0", "cell": "0"} + }, + { + "type": "BOOL", + "values": [{"sql": "NULL", "cell": null}, {"sql": "1", "cell": "1"}], + "edit": {"text": "1", "cell": "1"} + }, + { + "type": "JSON", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'{\"b\": 1, \"a\": [2, 3]}'", "cell": "{\"b\": 1, \"a\": [2, 3]}"}, + {"sql": "'\"text\"'", "cell": "\"text\""}, + {"sql": "'null'", "cell": "null"} + ], + "edit": {"text": "[1,2]", "cell": "[1,2]"} + }, + { + "type": "UUID", + "covers": [], + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'", "cell": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"} + ], + "edit": {"text": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12", "cell": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12"} + }, + { + "type": "INET4", + "covers": [], + "values": [{"sql": "'192.168.0.1'", "cell": "192.168.0.1"}], + "edit": {"text": "10.0.0.1", "cell": "10.0.0.1"} + }, + { + "type": "INET6", + "covers": [], + "values": [{"sql": "'::1'", "cell": "::1"}, {"sql": "'::ffff:192.168.0.1'", "cell": "::ffff:192.168.0.1"}], + "edit": {"text": "2001:db8::1", "cell": "2001:db8::1"} + }, + { + "type": "GEOMETRY", + "values": [{"sql": "NULL", "cell": null}, {"sql": "ST_GeomFromText('POINT(1 2)')", "cell": "POINT(1.0 2.0)"}], + "edit": {"text": "POINT(3 4)", "cell": "POINT(3.0 4.0)"}, + "known": { + "edit": {"defect": "the grid saves a geometry as WKT bound as text, which the server refuses (1416)", "matches": "\\[1416\\]"}, + "sql": {"defect": "the dump writes a geometry as a WKT string, which the server refuses (1416)", "matches": "\\[1416\\]"} + } + }, + { + "type": "POINT", + "values": [{"sql": "ST_GeomFromText('POINT(1.5 -2)')", "cell": "POINT(1.5 -2.0)"}], + "edit": {"text": "POINT(0 0)", "cell": "POINT(0.0 0.0)"}, + "known": { + "edit": {"defect": "the grid saves a geometry as WKT bound as text, which the server refuses (1416)", "matches": "\\[1416\\]"}, + "sql": {"defect": "the dump writes a geometry as a WKT string, which the server refuses (1416)", "matches": "\\[1416\\]"} + } + }, + { + "type": "LINESTRING", + "values": [{"sql": "ST_GeomFromText('LINESTRING(0 0,1 1)')", "cell": "LINESTRING(0.0 0.0, 1.0 1.0)"}], + "edit": {"text": "LINESTRING(1 1,2 2)", "cell": "LINESTRING(1.0 1.0, 2.0 2.0)"}, + "known": { + "edit": {"defect": "the grid saves a geometry as WKT bound as text, which the server refuses (1416)", "matches": "\\[1416\\]"}, + "sql": {"defect": "the dump writes a geometry as a WKT string, which the server refuses (1416)", "matches": "\\[1416\\]"} + } + }, + { + "type": "POLYGON", + "values": [ + {"sql": "ST_GeomFromText('POLYGON((0 0,1 0,1 1,0 0))')", "cell": "POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 0.0))"} + ], + "edit": {"text": "POLYGON((0 0,2 0,2 2,0 0))", "cell": "POLYGON((0.0 0.0, 2.0 0.0, 2.0 2.0, 0.0 0.0))"}, + "known": { + "edit": {"defect": "the grid saves a geometry as WKT bound as text, which the server refuses (1416)", "matches": "\\[1416\\]"}, + "sql": {"defect": "the dump writes a geometry as a WKT string, which the server refuses (1416)", "matches": "\\[1416\\]"} + } + } + ] +} diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/mssql.json b/apple/Plugins/ColumnTypeCorpus/Corpus/mssql.json new file mode 100644 index 0000000000..7fa6baff58 --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/mssql.json @@ -0,0 +1,240 @@ +{ + "engine": "mssql", + "databaseTypeId": "SQL Server", + "typeList": "Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift", + "createTable": "CREATE TABLE {table} ({id} INT PRIMARY KEY, {column} {type} NULL)", + "notColumnTypes": { + "CURSOR": "a cursor is a variable or parameter type; CREATE TABLE refuses it", + "TABLE": "a table type is for variables and parameters; CREATE TABLE refuses it" + }, + "columns": [ + { + "type": "TINYINT", + "values": [{"sql": "NULL", "cell": null}, {"sql": "0", "cell": "0"}, {"sql": "255", "cell": "255"}], + "edit": {"text": "12", "cell": "12"} + }, + { + "type": "SMALLINT", + "values": [{"sql": "-32768", "cell": "-32768"}], + "edit": {"text": "1", "cell": "1"} + }, + { + "type": "INT", + "values": [{"sql": "-2147483648", "cell": "-2147483648"}], + "edit": {"text": "2147483647", "cell": "2147483647"} + }, + { + "type": "BIGINT", + "values": [{"sql": "-9223372036854775808", "cell": "-9223372036854775808"}], + "edit": {"text": "9223372036854775807", "cell": "9223372036854775807"} + }, + { + "type": "FLOAT", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "0.1", "cell": "0.1"}, + {"sql": "1.7976931348623157E+308", "cell": "1.7976931348623157e+308"} + ], + "edit": {"text": "-0.5", "cell": "-0.5"}, + "known": { + "browse": {"defect": "a float reads with 17 significant digits, so 0.1 shows as 0.10000000000000001", "matches": "reads \"0\\.10000000000000001\""} + } + }, + { + "type": "REAL", + "values": [{"sql": "1.5", "cell": "1.5"}], + "edit": {"text": "2.25", "cell": "2.25"} + }, + { + "type": "DECIMAL(38,10)", + "covers": ["DECIMAL"], + "values": [ + {"sql": "1234567890123456789012345678.0123456789", "cell": "1234567890123456789012345678.0123456789"}, + {"sql": "-0.5", "cell": "-0.5000000000"} + ], + "edit": {"text": "1.1", "cell": "1.1000000000"} + }, + { + "type": "NUMERIC(10,2)", + "covers": ["NUMERIC"], + "values": [{"sql": "123.4", "cell": "123.40"}], + "edit": {"text": "99.99", "cell": "99.99"} + }, + { + "type": "MONEY", + "values": [{"sql": "922337203685477.5807", "cell": "922337203685477.5807"}, {"sql": "-1", "cell": "-1.0000"}], + "edit": {"text": "5.25", "cell": "5.2500"} + }, + { + "type": "SMALLMONEY", + "values": [{"sql": "214748.3647", "cell": "214748.3647"}], + "edit": {"text": "1", "cell": "1.0000"} + }, + { + "type": "CHAR(5)", + "covers": ["CHAR"], + "values": [{"sql": "'ab'", "cell": "ab "}], + "edit": {"text": "xyz", "cell": "xyz "} + }, + { + "type": "VARCHAR(MAX)", + "covers": ["VARCHAR"], + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "''", "cell": ""}, + {"sql": "'line one' + CHAR(10) + 'line two'", "cell": "line one\nline two"}, + {"sql": "'back\\slash ''quote'' \"double\"'", "cell": "back\\slash 'quote' \"double\""}, + {"sql": "'true'", "cell": "true"}, + {"sql": "'=1+1'", "cell": "=1+1"}, + {"sql": "REPLICATE(CAST('x' AS VARCHAR(MAX)), 70000)", "cell": "x", "repeat": 70000} + ], + "edit": {"text": "edited, \"quoted\"", "cell": "edited, \"quoted\""} + }, + { + "type": "TEXT", + "values": [{"sql": "'text'", "cell": "text"}], + "edit": {"text": "t", "cell": "t"} + }, + { + "type": "NCHAR(3)", + "covers": ["NCHAR"], + "values": [{"sql": "N'é'", "cell": "é "}], + "edit": {"text": "ü", "cell": "ü "} + }, + { + "type": "NVARCHAR(MAX)", + "covers": ["NVARCHAR"], + "values": [{"sql": "N'Ünïcödé ✓ 日本語 🦄'", "cell": "Ünïcödé ✓ 日本語 🦄"}], + "edit": {"text": "ñandú 日本", "cell": "ñandú 日本"} + }, + { + "type": "NTEXT", + "values": [{"sql": "N'ntext ✓'", "cell": "ntext ✓"}], + "edit": {"text": "n", "cell": "n"} + }, + { + "type": "DATE", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'0001-01-01'", "cell": "0001-01-01"}, + {"sql": "'9999-12-31'", "cell": "9999-12-31"} + ], + "edit": {"text": "2000-02-29", "cell": "2000-02-29"} + }, + { + "type": "TIME(7)", + "covers": ["TIME"], + "values": [{"sql": "'23:59:59.9999999'", "cell": "23:59:59.9999999"}], + "edit": {"text": "12:00:00", "cell": "12:00:00.0000000"} + }, + { + "type": "DATETIME", + "values": [{"sql": "'2026-10-20 15:22:20.123'", "cell": "2026-10-20 15:22:20.123"}], + "edit": {"text": "1999-12-31 23:59:59", "cell": "1999-12-31 23:59:59.000"} + }, + { + "type": "DATETIME2(7)", + "covers": ["DATETIME2"], + "values": [ + {"sql": "'2026-10-20 15:22:20.1234567'", "cell": "2026-10-20 15:22:20.1234567"}, + {"sql": "'0001-01-01 00:00:00'", "cell": "0001-01-01 00:00:00.0000000"} + ], + "edit": {"text": "2000-01-01 00:00:00.5", "cell": "2000-01-01 00:00:00.5000000"} + }, + { + "type": "SMALLDATETIME", + "values": [{"sql": "'2026-10-20 15:22:00'", "cell": "2026-10-20 15:22:00.000"}], + "edit": {"text": "2000-01-01 12:30:00", "cell": "2000-01-01 12:30:00.000"} + }, + { + "type": "DATETIMEOFFSET(7)", + "covers": ["DATETIMEOFFSET"], + "values": [{"sql": "'2026-10-20 15:22:20.1234567 +05:30'", "cell": "2026-10-20T15:22:20.1234567+05:30"}], + "edit": {"text": "2000-01-01 00:00:00 -08:00", "cell": "2000-01-01T00:00:00.0000000-08:00"} + }, + { + "type": "BINARY(4)", + "covers": ["BINARY"], + "values": [{"sql": "NULL", "cell": null}, {"sql": "0x00FF", "cell": "0x00FF0000"}], + "edit": {"hex": "DEADBEEF", "cell": "0xDEADBEEF"} + }, + { + "type": "VARBINARY(MAX)", + "covers": ["VARBINARY"], + "values": [{"sql": "0x", "cell": "0x"}, {"sql": "0x00FF7F80", "cell": "0x00FF7F80"}], + "edit": {"hex": "0102", "cell": "0x0102"} + }, + { + "type": "IMAGE", + "values": [{"sql": "0xCAFE", "cell": "0xCAFE"}], + "edit": {"hex": "BABE", "cell": "0xBABE"} + }, + { + "type": "BIT", + "values": [{"sql": "NULL", "cell": null}, {"sql": "1", "cell": "1"}, {"sql": "0", "cell": "0"}], + "edit": {"text": "0", "cell": "0"} + }, + { + "type": "XML", + "values": [{"sql": "'1'", "cell": "1"}], + "edit": {"text": "", "cell": ""} + }, + { + "type": "UNIQUEIDENTIFIER", + "values": [{"sql": "'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'", "cell": "A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11"}], + "edit": {"text": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12", "cell": "A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A12"} + }, + { + "type": "GEOMETRY", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "geometry::STGeomFromText('POINT(1 2)', 0)", "cell": "POINT (1 2)"} + ], + "edit": {"text": "POINT (3 4)", "cell": "POINT (3 4)"}, + "known": { + "browse": {"defect": "a geometry reads as its CLR serialization in hex", "matches": "reads \"0x[0-9A-F]+\""}, + "edit": {"defect": "the saved geometry reads back as its CLR serialization in hex", "matches": "reads back \"0x[0-9A-F]+\""} + } + }, + { + "type": "GEOGRAPHY", + "values": [{"sql": "geography::Point(47.65, -122.34, 4326)", "cell": "POINT (-122.34 47.65)"}], + "edit": {"text": "POINT (1 2)", "cell": "POINT (1 2)"}, + "known": { + "browse": {"defect": "a geography reads as its CLR serialization in hex", "matches": "reads \"0x[0-9A-F]+\""}, + "edit": {"defect": "the saved geography reads back as its CLR serialization in hex", "matches": "reads back \"0x[0-9A-F]+\""} + } + }, + { + "type": "HIERARCHYID", + "values": [{"sql": "'/1/2/'", "cell": "/1/2/"}], + "edit": {"text": "/3/", "cell": "/3/"}, + "known": { + "browse": {"defect": "a hierarchyid reads as its CLR serialization in hex", "matches": "reads \"0x[0-9A-F]+\""}, + "edit": {"defect": "the saved hierarchyid reads back as its CLR serialization in hex", "matches": "reads back \"0x[0-9A-F]+\""} + } + }, + { + "type": "SQL_VARIANT", + "values": [{"sql": "CAST(1 AS INT)", "cell": "1"}, {"sql": "CAST(N'text' AS NVARCHAR(10))", "cell": "text"}], + "edit": {"text": "x", "cell": "x"}, + "known": { + "browse": {"defect": "a sql_variant reads as NULL", "matches": "reads NULL"}, + "edit": {"defect": "the grid binds the value as nvarchar(max), which a sql_variant column refuses (206)", "matches": "Operand type clash"} + } + }, + { + "type": "ROWVERSION", + "values": [{"sql": "DEFAULT", "matches": "^0x[0-9A-F]{16}$"}], + "skip": {"edit": "the server writes a rowversion", "sql": "a restore writes a new rowversion"} + }, + { + "type": "TIMESTAMP", + "values": [{"sql": "DEFAULT", "matches": "^0x[0-9A-F]{16}$"}], + "skip": { + "edit": "TIMESTAMP is ROWVERSION, which the server writes", + "sql": "a restore writes a new rowversion" + } + } + ] +} diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/mysql.json b/apple/Plugins/ColumnTypeCorpus/Corpus/mysql.json new file mode 100644 index 0000000000..2a84fa621a --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/mysql.json @@ -0,0 +1,284 @@ +{ + "engine": "mysql", + "databaseTypeId": "MySQL", + "typeList": "Plugins/MySQLDriverPlugin/MySQLPlugin.swift", + "createTable": "CREATE TABLE {table} ({id} INT PRIMARY KEY, {column} {type})", + "columns": [ + { + "type": "TINYINT", + "values": [{"sql": "NULL", "cell": null}, {"sql": "-128", "cell": "-128"}, {"sql": "127", "cell": "127"}], + "edit": {"text": "12", "cell": "12"} + }, + { + "type": "TINYINT UNSIGNED", + "covers": [], + "values": [{"sql": "255", "cell": "255"}], + "edit": {"text": "0", "cell": "0"} + }, + { + "type": "SMALLINT", + "values": [{"sql": "-32768", "cell": "-32768"}, {"sql": "32767", "cell": "32767"}], + "edit": {"text": "1", "cell": "1"} + }, + { + "type": "MEDIUMINT", + "values": [{"sql": "-8388608", "cell": "-8388608"}, {"sql": "16777215 - 8388608", "cell": "8388607"}], + "edit": {"text": "2", "cell": "2"} + }, + { + "type": "INT", + "values": [{"sql": "-2147483648", "cell": "-2147483648"}, {"sql": "2147483647", "cell": "2147483647"}], + "edit": {"text": "3", "cell": "3"} + }, + { + "type": "INTEGER", + "values": [{"sql": "NULL", "cell": null}, {"sql": "0", "cell": "0"}], + "edit": {"text": "-4", "cell": "-4"} + }, + { + "type": "BIGINT", + "values": [ + {"sql": "-9223372036854775808", "cell": "-9223372036854775808"}, + {"sql": "9223372036854775807", "cell": "9223372036854775807"} + ], + "edit": {"text": "42", "cell": "42"} + }, + { + "type": "BIGINT UNSIGNED", + "covers": [], + "values": [{"sql": "18446744073709551615", "cell": "18446744073709551615"}], + "edit": {"text": "18446744073709551614", "cell": "18446744073709551614"} + }, + { + "type": "FLOAT", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "1.5", "cell": "1.5"}, + {"sql": "-3.4e38", "cell": "-3.4e38"} + ], + "edit": {"text": "2.25", "cell": "2.25"} + }, + { + "type": "DOUBLE", + "values": [{"sql": "0.1", "cell": "0.1"}, {"sql": "1.7976931348623157e308", "cell": "1.7976931348623157e308"}], + "edit": {"text": "-0.5", "cell": "-0.5"} + }, + { + "type": "REAL", + "values": [{"sql": "2.5", "cell": "2.5"}], + "edit": {"text": "1e-10", "cell": "0.0000000001"} + }, + { + "type": "DECIMAL(65,30)", + "covers": ["DECIMAL"], + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "12345678901234567890.123456789", "cell": "12345678901234567890.123456789000000000000000000000"}, + {"sql": "-0.5", "cell": "-0.500000000000000000000000000000"} + ], + "edit": {"text": "1.1", "cell": "1.100000000000000000000000000000"} + }, + { + "type": "NUMERIC(10,2)", + "covers": ["NUMERIC"], + "values": [{"sql": "123.4", "cell": "123.40"}], + "edit": {"text": "99.99", "cell": "99.99"} + }, + { + "type": "CHAR(5)", + "covers": ["CHAR"], + "values": [{"sql": "''", "cell": ""}, {"sql": "'ab'", "cell": "ab"}], + "edit": {"text": "xyz", "cell": "xyz"} + }, + { + "type": "VARCHAR(50)", + "covers": ["VARCHAR"], + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "''", "cell": ""}, + {"sql": "'Ünïcödé ✓ 日本語 🦄'", "cell": "Ünïcödé ✓ 日本語 🦄"}, + {"sql": "'back\\\\slash ''quote'' \"double\"'", "cell": "back\\slash 'quote' \"double\""}, + {"sql": "'line one\\nline two'", "cell": "line one\nline two"}, + {"sql": "'true'", "cell": "true"}, + {"sql": "'=1+1'", "cell": "=1+1"} + ], + "edit": {"text": "edited, \"quoted\" \\ back", "cell": "edited, \"quoted\" \\ back"} + }, + { + "type": "TINYTEXT", + "values": [{"sql": "'tiny'", "cell": "tiny"}], + "edit": {"text": "t", "cell": "t"} + }, + { + "type": "TEXT", + "values": [{"sql": "'text'", "cell": "text"}], + "edit": {"text": "", "cell": ""} + }, + { + "type": "MEDIUMTEXT", + "values": [{"sql": "'medium'", "cell": "medium"}], + "edit": {"text": "m", "cell": "m"} + }, + { + "type": "LONGTEXT", + "values": [{"sql": "REPEAT('x', 70000)", "cell": "x", "repeat": 70000}], + "edit": {"text": "long", "cell": "long"} + }, + { + "type": "ENUM('small','large','it''s')", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'small'", "cell": "small"}, + {"sql": "'it''s'", "cell": "it's"} + ], + "edit": {"text": "large", "cell": "large"} + }, + { + "type": "SET('red','green','blue')", + "values": [{"sql": "''", "cell": ""}, {"sql": "'blue,red'", "cell": "red,blue"}], + "edit": {"text": "green,red", "cell": "red,green"} + }, + { + "type": "DATE", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'1000-01-01'", "cell": "1000-01-01"}, + {"sql": "'9999-12-31'", "cell": "9999-12-31"} + ], + "edit": {"text": "2000-02-29", "cell": "2000-02-29"} + }, + { + "type": "TIME(6)", + "covers": ["TIME"], + "values": [ + {"sql": "'-838:59:59'", "cell": "-838:59:59.000000"}, + {"sql": "'838:59:59'", "cell": "838:59:59.000000"}, + {"sql": "'12:34:56.789012'", "cell": "12:34:56.789012"} + ], + "edit": {"text": "01:02:03", "cell": "01:02:03.000000"} + }, + { + "type": "DATETIME(6)", + "covers": ["DATETIME"], + "values": [ + {"sql": "'1000-01-01 00:00:00'", "cell": "1000-01-01 00:00:00.000000"}, + {"sql": "'9999-12-31 23:59:59.999999'", "cell": "9999-12-31 23:59:59.999999"} + ], + "edit": {"text": "2026-10-20 15:22:20", "cell": "2026-10-20 15:22:20.000000"} + }, + { + "type": "TIMESTAMP", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'1970-01-01 00:00:01'", "cell": "1970-01-01 00:00:01"}, + {"sql": "'2038-01-19 03:14:07'", "cell": "2038-01-19 03:14:07"} + ], + "edit": {"text": "2026-10-20 15:22:20", "cell": "2026-10-20 15:22:20"} + }, + { + "type": "YEAR", + "values": [{"sql": "1901", "cell": "1901"}, {"sql": "2155", "cell": "2155"}], + "edit": {"text": "2026", "cell": "2026"} + }, + { + "type": "BINARY(4)", + "covers": ["BINARY"], + "values": [{"sql": "NULL", "cell": null}, {"sql": "X'00FF'", "cell": "0x00FF0000"}], + "edit": {"hex": "DEADBEEF", "cell": "0xDEADBEEF"} + }, + { + "type": "VARBINARY(16)", + "covers": ["VARBINARY"], + "values": [{"sql": "X''", "cell": "0x"}, {"sql": "X'00FF7F80'", "cell": "0x00FF7F80"}], + "edit": {"hex": "0102", "cell": "0x0102"} + }, + { + "type": "TINYBLOB", + "values": [{"sql": "X'01'", "cell": "0x01"}], + "edit": {"hex": "02", "cell": "0x02"} + }, + { + "type": "BLOB", + "values": [{"sql": "X'48656C6C6F'", "cell": "0x48656C6C6F"}], + "edit": {"hex": "FF", "cell": "0xFF"} + }, + { + "type": "MEDIUMBLOB", + "values": [{"sql": "X'00'", "cell": "0x00"}], + "edit": {"hex": "0A0D", "cell": "0x0A0D"} + }, + { + "type": "LONGBLOB", + "values": [{"sql": "X'CAFE'", "cell": "0xCAFE"}], + "edit": {"hex": "BABE", "cell": "0xBABE"} + }, + { + "type": "BIT(8)", + "covers": ["BIT"], + "values": [{"sql": "NULL", "cell": null}, {"sql": "b'10101010'", "cell": "170"}, {"sql": "b'1'", "cell": "1"}], + "edit": {"row": 2, "text": "5", "cell": "5"}, + "known": { + "edit": {"defect": "the grid saves a BIT cell as text, and the server stores the text's bytes: 5 saves as 53", "matches": "reads back \"53\""}, + "sql": {"defect": "the dump writes a BIT value as quoted text, which restores as the text's bytes", "matches": "the replayed dump reads"} + } + }, + { + "type": "BOOLEAN", + "values": [{"sql": "TRUE", "cell": "1"}, {"sql": "FALSE", "cell": "0"}], + "edit": {"text": "0", "cell": "0"} + }, + { + "type": "BOOL", + "values": [{"sql": "NULL", "cell": null}, {"sql": "1", "cell": "1"}], + "edit": {"text": "1", "cell": "1"} + }, + { + "type": "JSON", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'{\"b\": 1, \"a\": [2, 3]}'", "cell": "{\"a\": [2, 3], \"b\": 1}"}, + {"sql": "'\"text\"'", "cell": "\"text\""}, + {"sql": "'null'", "cell": "null"} + ], + "edit": {"text": "[1,2]", "cell": "[1, 2]"} + }, + { + "type": "GEOMETRY", + "values": [{"sql": "NULL", "cell": null}, {"sql": "ST_GeomFromText('POINT(1 2)')", "cell": "POINT(1.0 2.0)"}], + "edit": {"text": "POINT(3 4)", "cell": "POINT(3.0 4.0)"}, + "known": { + "edit": {"defect": "the grid saves a geometry as WKT bound as text, which the server refuses (1416)", "matches": "\\[1416\\]"}, + "sql": {"defect": "the dump writes a geometry as a WKT string, which the server refuses (1416)", "matches": "\\[1416\\]"} + } + }, + { + "type": "POINT", + "values": [{"sql": "ST_GeomFromText('POINT(1.5 -2)')", "cell": "POINT(1.5 -2.0)"}], + "edit": {"text": "POINT(0 0)", "cell": "POINT(0.0 0.0)"}, + "known": { + "edit": {"defect": "the grid saves a geometry as WKT bound as text, which the server refuses (1416)", "matches": "\\[1416\\]"}, + "sql": {"defect": "the dump writes a geometry as a WKT string, which the server refuses (1416)", "matches": "\\[1416\\]"} + } + }, + { + "type": "LINESTRING", + "values": [{"sql": "ST_GeomFromText('LINESTRING(0 0,1 1)')", "cell": "LINESTRING(0.0 0.0, 1.0 1.0)"}], + "edit": {"text": "LINESTRING(1 1,2 2)", "cell": "LINESTRING(1.0 1.0, 2.0 2.0)"}, + "known": { + "edit": {"defect": "the grid saves a geometry as WKT bound as text, which the server refuses (1416)", "matches": "\\[1416\\]"}, + "sql": {"defect": "the dump writes a geometry as a WKT string, which the server refuses (1416)", "matches": "\\[1416\\]"} + } + }, + { + "type": "POLYGON", + "values": [ + {"sql": "ST_GeomFromText('POLYGON((0 0,1 0,1 1,0 0))')", "cell": "POLYGON((0.0 0.0, 1.0 0.0, 1.0 1.0, 0.0 0.0))"} + ], + "edit": {"text": "POLYGON((0 0,2 0,2 2,0 0))", "cell": "POLYGON((0.0 0.0, 2.0 0.0, 2.0 2.0, 0.0 0.0))"}, + "known": { + "edit": {"defect": "the grid saves a geometry as WKT bound as text, which the server refuses (1416)", "matches": "\\[1416\\]"}, + "sql": {"defect": "the dump writes a geometry as a WKT string, which the server refuses (1416)", "matches": "\\[1416\\]"} + } + } + ] +} diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json b/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json new file mode 100644 index 0000000000..4a17f37b1c --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json @@ -0,0 +1,269 @@ +{ + "engine": "oracle", + "databaseTypeId": "Oracle", + "typeList": "Plugins/OracleDriverPlugin/OracleTypeCatalog.swift", + "createTable": "CREATE TABLE {table} ({id} NUMBER(10) PRIMARY KEY, {column} {type})", + "dropTable": "DROP TABLE {table} PURGE", + "columns": [ + { + "type": "NUMBER", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "0", "cell": "0"}, + {"sql": "-1.5", "cell": "-1.5"}, + {"sql": "12345678901234567890.123456789", "cell": "12345678901234567890.123456789"}, + {"sql": "1e40", "cell": "10000000000000000000000000000000000000000"} + ], + "edit": {"text": "42", "cell": "42"} + }, + { + "type": "NUMBER(10,2)", + "covers": [], + "values": [{"sql": "123.4", "cell": "123.4"}], + "edit": {"text": "99.999", "cell": "100"} + }, + { + "type": "INTEGER", + "values": [{"sql": "-2147483648", "cell": "-2147483648"}], + "edit": {"text": "7", "cell": "7"} + }, + { + "type": "INT", + "values": [{"sql": "1", "cell": "1"}], + "edit": {"text": "-1", "cell": "-1"} + }, + { + "type": "SMALLINT", + "values": [{"sql": "32767", "cell": "32767"}], + "edit": {"text": "2", "cell": "2"} + }, + { + "type": "FLOAT", + "values": [{"sql": "1.5", "cell": "1.5"}], + "edit": {"text": "2.25", "cell": "2.25"} + }, + { + "type": "BINARY_FLOAT", + "values": [{"sql": "1.5", "cell": "1.5"}, {"sql": "BINARY_FLOAT_INFINITY", "cell": "inf"}], + "edit": {"text": "-0.5", "cell": "-0.5"} + }, + { + "type": "BINARY_DOUBLE", + "values": [{"sql": "0.1", "cell": "0.1"}, {"sql": "BINARY_DOUBLE_NAN", "cell": "nan"}], + "edit": {"text": "3.5", "cell": "3.5"} + }, + { + "type": "DECIMAL(10,3)", + "covers": ["DECIMAL"], + "values": [{"sql": "1.25", "cell": "1.25"}], + "edit": {"text": "-3.125", "cell": "-3.125"} + }, + { + "type": "NUMERIC(5)", + "covers": ["NUMERIC"], + "values": [{"sql": "12345", "cell": "12345"}], + "edit": {"text": "1", "cell": "1"} + }, + { + "type": "REAL", + "values": [{"sql": "2.5", "cell": "2.5"}], + "edit": {"text": "0.125", "cell": "0.125"} + }, + { + "type": "DOUBLE PRECISION", + "values": [{"sql": "0.1", "cell": "0.1"}], + "edit": {"text": "8", "cell": "8"} + }, + { + "type": "VARCHAR2(100 CHAR)", + "covers": ["VARCHAR2"], + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "''", "cell": null}, + {"sql": "'Ünïcödé ✓ 日本語 🦄'", "cell": "Ünïcödé ✓ 日本語 🦄"}, + {"sql": "'line one' || CHR(10) || 'line two'", "cell": "line one\nline two"}, + {"sql": "'back\\slash ''quote'' \"double\"'", "cell": "back\\slash 'quote' \"double\""}, + {"sql": "'true'", "cell": "true"}, + {"sql": "'=1+1'", "cell": "=1+1"} + ], + "edit": {"text": "edited, \"quoted\"", "cell": "edited, \"quoted\""} + }, + { + "type": "NVARCHAR2(50)", + "covers": ["NVARCHAR2"], + "values": [{"sql": "N'ñandú 日本'", "cell": "ñandú 日本"}], + "edit": {"text": "ø", "cell": "ø"} + }, + { + "type": "CHAR(5)", + "covers": ["CHAR"], + "values": [{"sql": "'ab'", "cell": "ab "}], + "edit": {"text": "xyz", "cell": "xyz "} + }, + { + "type": "NCHAR(3)", + "covers": ["NCHAR"], + "values": [{"sql": "N'é'", "cell": "é "}], + "edit": {"text": "ü", "cell": "ü "} + }, + { + "type": "CLOB", + "values": [{"sql": "NULL", "cell": null}, {"sql": "'clob text'", "cell": "clob text"}], + "edit": {"text": "more", "cell": "more"} + }, + { + "type": "NCLOB", + "values": [{"sql": "N'nclob ✓'", "cell": "nclob ✓"}], + "edit": {"text": "n", "cell": "n"} + }, + { + "type": "LONG", + "values": [{"sql": "'long text'", "cell": "long text"}], + "edit": {"text": "l", "cell": "l"} + }, + { + "type": "DATE", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "TO_DATE('2026-10-20 15:22:20', 'YYYY-MM-DD HH24:MI:SS')", "cell": "2026-10-20 15:22:20"}, + {"sql": "TO_DATE('0001-01-01', 'YYYY-MM-DD')", "cell": "0001-01-01 00:00:00"} + ], + "edit": {"text": "2000-02-29 12:00:00", "cell": "2000-02-29 12:00:00"}, + "known": { + "sql": {"defect": "the dump writes the value as text in the driver's format, which Oracle parses with the session's NLS format and refuses (ORA-01861, ORA-01843)", "matches": "ORA-018(61|43)"} + } + }, + { + "type": "TIMESTAMP", + "values": [ + {"sql": "TO_TIMESTAMP('2026-10-06 15:22:20.050000', 'YYYY-MM-DD HH24:MI:SS.FF')", "cell": "2026-10-06 15:22:20.050000"} + ], + "edit": {"text": "1999-12-31 23:59:59.000000", "cell": "1999-12-31 23:59:59.000000"}, + "known": { + "sql": {"defect": "the dump writes the value as text in the driver's format, which Oracle parses with the session's NLS format and refuses (ORA-01861, ORA-01843)", "matches": "ORA-018(61|43)"} + } + }, + { + "type": "TIMESTAMP WITH TIME ZONE", + "values": [ + {"sql": "TO_TIMESTAMP_TZ('2026-10-06 15:22:20.000001 -05:30', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')", "cell": "2026-10-06 15:22:20.000001-05:30"} + ], + "edit": {"text": "2000-01-01 00:00:00.000000+00:00", "cell": "2000-01-01 00:00:00.000000+00:00"}, + "known": { + "sql": {"defect": "the dump writes the value as text in the driver's format, which Oracle parses with the session's NLS format and refuses (ORA-01861, ORA-01843)", "matches": "ORA-018(61|43)"} + } + }, + { + "type": "TIMESTAMP WITH LOCAL TIME ZONE", + "values": [ + {"sql": "TO_TIMESTAMP('2026-10-06 15:22:20', 'YYYY-MM-DD HH24:MI:SS')", "cell": "2026-10-06 15:22:20.000000+00:00"} + ], + "edit": {"text": "2000-01-01 00:00:00", "cell": "2000-01-01 00:00:00.000000+00:00"}, + "known": { + "sql": {"defect": "the dump writes the value as text in the driver's format, which Oracle parses with the session's NLS format and refuses (ORA-01861, ORA-01843)", "matches": "ORA-018(61|43)"} + } + }, + { + "type": "INTERVAL YEAR(4) TO MONTH", + "covers": ["INTERVAL YEAR TO MONTH"], + "values": [{"sql": "INTERVAL '1-2' YEAR TO MONTH", "cell": "1-02"}], + "edit": {"text": "3-04", "cell": "3-04"} + }, + { + "type": "INTERVAL DAY(3) TO SECOND(3)", + "covers": ["INTERVAL DAY TO SECOND"], + "values": [{"sql": "INTERVAL '3 04:05:06.789' DAY TO SECOND", "cell": "3 04:05:06.789"}], + "edit": {"text": "1 00:00:00", "cell": "1 00:00:00"} + }, + { + "type": "RAW(16)", + "covers": ["RAW"], + "values": [{"sql": "NULL", "cell": null}, {"sql": "HEXTORAW('00FF7F80')", "cell": "0x00FF7F80"}], + "edit": {"hex": "DEADBEEF", "cell": "0xDEADBEEF"} + }, + { + "type": "LONG RAW", + "values": [{"sql": "HEXTORAW('CAFE')", "cell": "0xCAFE"}], + "edit": {"hex": "BABE", "cell": "0xBABE"} + }, + { + "type": "BLOB", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "HEXTORAW('48656C6C6F')", "cell": "0x48656C6C6F"}, + {"sql": "EMPTY_BLOB()", "cell": "0x"} + ], + "edit": {"hex": "FF", "cell": "0xFF"}, + "known": { + "browse": {"defect": "an empty BLOB reads as NULL", "matches": "EMPTY_BLOB\\(\\) reads NULL"} + } + }, + { + "type": "BFILE", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "BFILENAME('TP_DIR', 'file.txt')", "cell": "BFILENAME('TP_DIR', 'file.txt')"} + ], + "skip": {"edit": "a BFILE is a locator to a server file, written with BFILENAME rather than a value"}, + "known": { + "sql": {"defect": "the dump writes a BFILE as quoted text, which Oracle refuses (ORA-00932)", "matches": "ORA-00932"} + } + }, + { + "type": "BOOLEAN", + "values": [{"sql": "NULL", "cell": null}, {"sql": "TRUE", "cell": "true"}, {"sql": "FALSE", "cell": "false"}], + "edit": {"text": "false", "cell": "false"} + }, + { + "type": "JSON", + "values": [{"sql": "JSON('{\"b\":1,\"a\":[2,3]}')", "cell": "{\"b\":1,\"a\":[2,3]}"}], + "edit": {"text": "{\"k\":\"v\"}", "cell": "{\"k\":\"v\"}"} + }, + { + "type": "VECTOR(3, FLOAT32)", + "covers": ["VECTOR"], + "values": [{"sql": "'[1.5, 2, 3]'", "cell": "[1.5E+000,2.0E+000,3.0E+000]"}], + "edit": {"text": "[4, 5, 6]", "cell": "[4.0E+000,5.0E+000,6.0E+000]"}, + "known": { + "browse": {"defect": "a VECTOR reads as the placeholder ", "matches": "reads \"\""}, + "edit": {"defect": "the saved VECTOR reads back as ", "matches": "reads back \"\""}, + "sql": {"defect": "the dump writes the placeholder , which Oracle refuses (ORA-51804)", "matches": "ORA-51804"} + } + }, + { + "type": "XMLTYPE", + "values": [{"sql": "XMLTYPE('1')", "cell": "1"}], + "edit": {"text": "2", "cell": "2"}, + "known": { + "browse": {"defect": "an XMLTYPE reads as ", "matches": "reads \"\""}, + "edit": {"defect": "the column reads as server-owned, so the grid refuses the save", "matches": "cannot be given a value"}, + "sql": {"defect": "the dump writes the placeholder, which Oracle refuses (ORA-31011)", "matches": "ORA-31011"} + } + }, + { + "type": "SDO_GEOMETRY", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "SDO_GEOMETRY(2001, NULL, SDO_POINT_TYPE(1, 2, NULL), NULL, NULL)", "cell": "POINT (1 2)"} + ], + "skip": {"edit": "a geometry is written through SDO_GEOMETRY, not a typed value"}, + "known": { + "browse": {"defect": "an SDO_GEOMETRY reads as ", "matches": "reads \"\""}, + "sql": {"defect": "the dump writes the placeholder, which Oracle refuses (ORA-00932)", "matches": "ORA-00932"} + } + }, + { + "type": "ROWID", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "CHARTOROWID('AAAR3sAAEAAAACXAAA')", "cell": "AAAR3sAAEAAAACXAAA"} + ], + "edit": {"text": "AAAR3sAAEAAAACXAAB", "cell": "AAAR3sAAEAAAACXAAB"} + }, + { + "type": "UROWID", + "values": [{"sql": "CHARTOROWID('AAAR3sAAEAAAACXAAA')", "cell": "AAAR3sAAEAAAACXAAA"}], + "edit": {"text": "AAAR3sAAEAAAACXAAC", "cell": "AAAR3sAAEAAAACXAAC"} + } + ] +} diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/postgresql.json b/apple/Plugins/ColumnTypeCorpus/Corpus/postgresql.json new file mode 100644 index 0000000000..871ea0e57a --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/postgresql.json @@ -0,0 +1,335 @@ +{ + "engine": "postgresql", + "databaseTypeId": "PostgreSQL", + "typeList": "Plugins/PostgreSQLDriverPlugin/PostgreSQLPlugin.swift", + "createTable": "CREATE TABLE {table} ({id} INTEGER PRIMARY KEY, {column} {type})", + "columns": [ + { + "type": "SMALLINT", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "-32768", "cell": "-32768"}, + {"sql": "32767", "cell": "32767"} + ], + "edit": {"text": "12", "cell": "12"} + }, + { + "type": "INTEGER", + "values": [{"sql": "-2147483648", "cell": "-2147483648"}, {"sql": "2147483647", "cell": "2147483647"}], + "edit": {"text": "-5", "cell": "-5"} + }, + { + "type": "BIGINT", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "-9223372036854775808", "cell": "-9223372036854775808"}, + {"sql": "9223372036854775807", "cell": "9223372036854775807"} + ], + "edit": {"text": "42", "cell": "42"} + }, + { + "type": "SERIAL", + "values": [{"sql": "1", "cell": "1"}, {"sql": "2147483647", "cell": "2147483647"}], + "edit": {"text": "5", "cell": "5"} + }, + { + "type": "BIGSERIAL", + "values": [{"sql": "9223372036854775807", "cell": "9223372036854775807"}], + "edit": {"text": "6", "cell": "6"} + }, + { + "type": "SMALLSERIAL", + "values": [{"sql": "32767", "cell": "32767"}], + "edit": {"text": "7", "cell": "7"} + }, + { + "type": "REAL", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "1.5", "cell": "1.5"}, + {"sql": "-0.000001", "cell": "-1e-06"}, + {"sql": "3.4028235e38", "cell": "3.4028235e+38"}, + {"sql": "'NaN'", "cell": "NaN"}, + {"sql": "'-Infinity'", "cell": "-Infinity"} + ], + "edit": {"text": "2.25", "cell": "2.25"} + }, + { + "type": "DOUBLE PRECISION", + "values": [ + {"sql": "0.1", "cell": "0.1"}, + {"sql": "1.7976931348623157e308", "cell": "1.7976931348623157e+308"}, + {"sql": "'Infinity'", "cell": "Infinity"} + ], + "edit": {"text": "-0.5", "cell": "-0.5"} + }, + { + "type": "NUMERIC", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "12345678901234567890.123456789", "cell": "12345678901234567890.123456789"}, + {"sql": "-0.00000001", "cell": "-0.00000001"}, + {"sql": "'NaN'", "cell": "NaN"}, + {"sql": "'Infinity'", "cell": "Infinity"} + ], + "edit": {"text": "1.10", "cell": "1.10"} + }, + { + "type": "DECIMAL(10,2)", + "covers": ["DECIMAL"], + "values": [{"sql": "123.4", "cell": "123.40"}, {"sql": "-0.5", "cell": "-0.50"}], + "edit": {"text": "99.999", "cell": "100.00"} + }, + { + "type": "MONEY", + "values": [{"sql": "12.34", "cell": "$12.34"}, {"sql": "-1", "cell": "-$1.00"}], + "edit": {"text": "5", "cell": "$5.00"} + }, + { + "type": "CHARACTER VARYING(10)", + "covers": ["CHARACTER VARYING"], + "values": [{"sql": "''", "cell": ""}, {"sql": "'ten chars!'", "cell": "ten chars!"}], + "edit": {"text": "x", "cell": "x"} + }, + { + "type": "VARCHAR(20)", + "covers": ["VARCHAR"], + "values": [{"sql": "'it''s'", "cell": "it's"}], + "edit": {"text": "", "cell": ""} + }, + { + "type": "CHARACTER(5)", + "covers": ["CHARACTER"], + "values": [{"sql": "'ab'", "cell": "ab "}], + "edit": {"text": "xyz", "cell": "xyz "} + }, + { + "type": "CHAR(3)", + "covers": ["CHAR"], + "values": [{"sql": "'a'", "cell": "a "}], + "edit": {"text": "é", "cell": "é "} + }, + { + "type": "TEXT", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "''", "cell": ""}, + {"sql": "'Ünïcödé ✓ 日本語 🦄'", "cell": "Ünïcödé ✓ 日本語 🦄"}, + {"sql": "E'line one\\nline two\\ttab'", "cell": "line one\nline two\ttab"}, + {"sql": "E'back\\\\slash and ''quote'' and \"double\"'", "cell": "back\\slash and 'quote' and \"double\""}, + {"sql": "'true'", "cell": "true"}, + {"sql": "'=1+1'", "cell": "=1+1"} + ], + "edit": {"text": "edited, \"quoted\"", "cell": "edited, \"quoted\""} + }, + { + "type": "NAME", + "values": [{"sql": "'pg_catalog'", "cell": "pg_catalog"}], + "edit": {"text": "public", "cell": "public"} + }, + { + "type": "DATE", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'2026-10-20'", "cell": "2026-10-20"}, + {"sql": "'0001-01-01'", "cell": "0001-01-01"}, + {"sql": "'4713-01-01 BC'", "cell": "4713-01-01 BC"}, + {"sql": "'infinity'", "cell": "infinity"} + ], + "edit": {"text": "2000-02-29", "cell": "2000-02-29"} + }, + { + "type": "TIME", + "values": [{"sql": "'23:59:59.999999'", "cell": "23:59:59.999999"}, {"sql": "'24:00:00'", "cell": "24:00:00"}], + "edit": {"text": "12:00:00", "cell": "12:00:00"} + }, + { + "type": "TIME WITH TIME ZONE", + "values": [{"sql": "'12:00:00+05:30'", "cell": "12:00:00+05:30"}], + "edit": {"text": "01:02:03-08", "cell": "01:02:03-08"} + }, + { + "type": "TIMESTAMP", + "values": [ + {"sql": "'2026-10-20 15:22:20.123456'", "cell": "2026-10-20 15:22:20.123456"}, + {"sql": "'0001-01-01 00:00:00'", "cell": "0001-01-01 00:00:00"}, + {"sql": "'infinity'", "cell": "infinity"} + ], + "edit": {"text": "1999-12-31 23:59:59", "cell": "1999-12-31 23:59:59"} + }, + { + "type": "TIMESTAMPTZ", + "values": [{"sql": "'2026-10-20 15:22:20+00'", "cell": "2026-10-20 15:22:20+00"}], + "edit": {"text": "2026-10-20 10:00:00-05", "cell": "2026-10-20 15:00:00+00"} + }, + { + "type": "TIMESTAMP WITH TIME ZONE", + "values": [{"sql": "'2026-10-20 15:22:20.5+02'", "cell": "2026-10-20 13:22:20.5+00"}], + "edit": {"text": "2000-01-01 00:00:00+00", "cell": "2000-01-01 00:00:00+00"} + }, + { + "type": "INTERVAL", + "values": [ + {"sql": "'1 year 2 mons 3 days 04:05:06.789'", "cell": "1 year 2 mons 3 days 04:05:06.789"}, + {"sql": "'-1 day'", "cell": "-1 days"} + ], + "edit": {"text": "2 hours", "cell": "02:00:00"} + }, + { + "type": "BYTEA", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "''::bytea", "cell": "0x"}, + {"sql": "'\\x00ff7f80'", "cell": "0x00FF7F80"} + ], + "edit": {"hex": "DEADBEEF", "cell": "0xDEADBEEF"} + }, + { + "type": "BOOLEAN", + "values": [{"sql": "NULL", "cell": null}, {"sql": "true", "cell": "true"}, {"sql": "false", "cell": "false"}], + "edit": {"text": "false", "cell": "false"} + }, + { + "type": "JSON", + "values": [ + {"sql": "'{\"b\": 1, \"a\": [2, 3]}'", "cell": "{\"b\": 1, \"a\": [2, 3]}"}, + {"sql": "'null'", "cell": "null"}, + {"sql": "'\"text\"'", "cell": "\"text\""} + ], + "edit": {"text": "{\"k\":\"v\"}", "cell": "{\"k\":\"v\"}"} + }, + { + "type": "JSONB", + "values": [ + {"sql": "'{\"b\": 1, \"a\": [2, 3]}'", "cell": "{\"a\": [2, 3], \"b\": 1}"}, + {"sql": "'[]'", "cell": "[]"} + ], + "edit": {"text": "[1,2]", "cell": "[1, 2]"} + }, + { + "type": "UUID", + "values": [{"sql": "'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'", "cell": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"}], + "edit": {"text": "A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A12", "cell": "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12"} + }, + { + "type": "INTEGER[]", + "covers": ["ARRAY"], + "values": [{"sql": "'{1,NULL,3}'", "cell": "{1,NULL,3}"}, {"sql": "'{}'", "cell": "{}"}], + "edit": {"text": "{4,5}", "cell": "{4,5}"} + }, + { + "type": "TEXT[]", + "covers": [], + "values": [{"sql": "ARRAY['a,b', 'c\"d', NULL, '']", "cell": "{\"a,b\",\"c\\\"d\",NULL,\"\"}"}], + "edit": {"text": "{x,\"y z\"}", "cell": "{x,\"y z\"}"} + }, + { + "type": "INTEGER[][]", + "covers": [], + "values": [{"sql": "'{{1,2},{3,4}}'", "cell": "{{1,2},{3,4}}"}], + "edit": {"text": "{{5,6}}", "cell": "{{5,6}}"} + }, + { + "type": "INET", + "values": [{"sql": "'192.168.0.1/24'", "cell": "192.168.0.1/24"}, {"sql": "'::1'", "cell": "::1"}], + "edit": {"text": "10.0.0.1", "cell": "10.0.0.1"} + }, + { + "type": "CIDR", + "values": [{"sql": "'10.0.0.0/8'", "cell": "10.0.0.0/8"}], + "edit": {"text": "192.168.0.0/16", "cell": "192.168.0.0/16"} + }, + { + "type": "MACADDR", + "values": [{"sql": "'08:00:2b:01:02:03'", "cell": "08:00:2b:01:02:03"}], + "edit": {"text": "08-00-2b-01-02-04", "cell": "08:00:2b:01:02:04"} + }, + { + "type": "MACADDR8", + "values": [{"sql": "'08:00:2b:01:02:03:04:05'", "cell": "08:00:2b:01:02:03:04:05"}], + "edit": {"text": "08:00:2b:01:02:03:04:06", "cell": "08:00:2b:01:02:03:04:06"} + }, + { + "type": "POINT", + "values": [{"sql": "'(1.5,2)'", "cell": "(1.5,2)"}], + "edit": {"text": "(0,-1)", "cell": "(0,-1)"} + }, + { + "type": "LINE", + "values": [{"sql": "'{1,-1,0}'", "cell": "{1,-1,0}"}], + "edit": {"text": "{2,-1,3}", "cell": "{2,-1,3}"} + }, + { + "type": "LSEG", + "values": [{"sql": "'[(0,0),(1,1)]'", "cell": "[(0,0),(1,1)]"}], + "edit": {"text": "[(1,1),(2,2)]", "cell": "[(1,1),(2,2)]"} + }, + { + "type": "BOX", + "values": [{"sql": "'(0,0),(1,1)'", "cell": "(1,1),(0,0)"}], + "edit": {"text": "(2,2),(0,0)", "cell": "(2,2),(0,0)"} + }, + { + "type": "PATH", + "values": [{"sql": "'[(0,0),(1,1),(2,0)]'", "cell": "[(0,0),(1,1),(2,0)]"}], + "edit": {"text": "((0,0),(1,1))", "cell": "((0,0),(1,1))"} + }, + { + "type": "POLYGON", + "values": [{"sql": "'((0,0),(1,1),(1,0))'", "cell": "((0,0),(1,1),(1,0))"}], + "edit": {"text": "((0,0),(2,2),(2,0))", "cell": "((0,0),(2,2),(2,0))"} + }, + { + "type": "CIRCLE", + "values": [{"sql": "'<(0,0),1>'", "cell": "<(0,0),1>"}], + "edit": {"text": "<(1,1),2>", "cell": "<(1,1),2>"} + }, + { + "type": "INT4RANGE", + "values": [{"sql": "'[1,10)'", "cell": "[1,10)"}, {"sql": "'empty'", "cell": "empty"}], + "edit": {"text": "(1,5]", "cell": "[2,6)"} + }, + { + "type": "INT8RANGE", + "values": [{"sql": "'[1,)'", "cell": "[1,)"}], + "edit": {"text": "[,5)", "cell": "(,5)"} + }, + { + "type": "NUMRANGE", + "values": [{"sql": "'(1.5,2.5]'", "cell": "(1.5,2.5]"}], + "edit": {"text": "[0,1)", "cell": "[0,1)"} + }, + { + "type": "TSRANGE", + "values": [ + {"sql": "'[2026-01-01 00:00,2026-01-02 00:00)'", "cell": "[\"2026-01-01 00:00:00\",\"2026-01-02 00:00:00\")"} + ], + "edit": {"text": "[2026-02-01,2026-03-01)", "cell": "[\"2026-02-01 00:00:00\",\"2026-03-01 00:00:00\")"} + }, + { + "type": "TSTZRANGE", + "values": [{"sql": "'[2026-01-01 00:00+00,)'", "cell": "[\"2026-01-01 00:00:00+00\",)"}], + "edit": {"text": "(,2026-01-01 00:00+00]", "cell": "(,\"2026-01-01 00:00:00+00\"]"} + }, + { + "type": "DATERANGE", + "values": [{"sql": "'[2026-01-01,2026-12-31]'", "cell": "[2026-01-01,2027-01-01)"}], + "edit": {"text": "[2026-02-01,2026-03-01)", "cell": "[2026-02-01,2026-03-01)"} + }, + { + "type": "TSVECTOR", + "values": [{"sql": "'a fat cat'", "cell": "'a' 'cat' 'fat'"}], + "edit": {"text": "rat", "cell": "'rat'"} + }, + { + "type": "TSQUERY", + "values": [{"sql": "'fat & rat'", "cell": "'fat' & 'rat'"}], + "edit": {"text": "cat | dog", "cell": "'cat' | 'dog'"} + }, + { + "type": "XML", + "values": [{"sql": "'1'", "cell": "1"}], + "edit": {"text": "", "cell": ""} + } + ] +} diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/sqlite.json b/apple/Plugins/ColumnTypeCorpus/Corpus/sqlite.json new file mode 100644 index 0000000000..3d276df2ed --- /dev/null +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/sqlite.json @@ -0,0 +1,169 @@ +{ + "engine": "sqlite", + "databaseTypeId": "SQLite", + "typeList": "Plugins/SQLiteDriverPlugin/SQLitePlugin.swift", + "createTable": "CREATE TABLE {table} ({id} INTEGER PRIMARY KEY, {column} {type})", + "columns": [ + { + "type": "INTEGER", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "0", "cell": "0"}, + {"sql": "-9223372036854775808", "cell": "-9223372036854775808"}, + {"sql": "9223372036854775807", "cell": "9223372036854775807"} + ], + "edit": {"text": "42", "cell": "42"} + }, + { + "type": "INT", + "values": [{"sql": "NULL", "cell": null}, {"sql": "-1", "cell": "-1"}], + "edit": {"text": "7", "cell": "7"} + }, + { + "type": "TINYINT", + "values": [{"sql": "NULL", "cell": null}, {"sql": "255", "cell": "255"}], + "edit": {"text": "-128", "cell": "-128"} + }, + { + "type": "SMALLINT", + "values": [{"sql": "-32768", "cell": "-32768"}, {"sql": "32767", "cell": "32767"}], + "edit": {"text": "12", "cell": "12"} + }, + { + "type": "MEDIUMINT", + "values": [{"sql": "8388607", "cell": "8388607"}], + "edit": {"text": "-8388608", "cell": "-8388608"} + }, + { + "type": "BIGINT", + "values": [{"sql": "NULL", "cell": null}, {"sql": "9223372036854775807", "cell": "9223372036854775807"}], + "edit": {"text": "-9223372036854775808", "cell": "-9223372036854775808"} + }, + { + "type": "REAL", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "1.5", "cell": "1.5"}, + {"sql": "-0.000001", "cell": "-1.0e-06"}, + {"sql": "1.7976931348623157e308", "cell": "1.7976931348623157e+308"} + ], + "edit": {"text": "3.25", "cell": "3.25"} + }, + { + "type": "DOUBLE", + "values": [{"sql": "2.5", "cell": "2.5"}], + "edit": {"text": "-2.5", "cell": "-2.5"} + }, + { + "type": "FLOAT", + "values": [{"sql": "0.1", "cell": "0.1"}], + "edit": {"text": "100", "cell": "100.0"} + }, + { + "type": "NUMERIC", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "12345.678", "cell": "12345.678"}, + {"sql": "'abc'", "cell": "abc"} + ], + "edit": {"text": "10", "cell": "10"} + }, + { + "type": "DECIMAL(10,2)", + "covers": ["DECIMAL"], + "values": [{"sql": "123.45", "cell": "123.45"}, {"sql": "-0.5", "cell": "-0.5"}], + "edit": {"text": "99.99", "cell": "99.99"} + }, + { + "type": "TEXT", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "''", "cell": ""}, + {"sql": "'it''s'", "cell": "it's"}, + {"sql": "'Ünïcödé ✓ 日本語 🦄'", "cell": "Ünïcödé ✓ 日本語 🦄"}, + {"sql": "'line one' || char(10) || 'line two'", "cell": "line one\nline two"}, + {"sql": "'true'", "cell": "true"}, + {"sql": "'=1+1'", "cell": "=1+1"} + ], + "edit": {"text": "edited, \"quoted\"", "cell": "edited, \"quoted\""} + }, + { + "type": "VARCHAR(20)", + "covers": ["VARCHAR"], + "values": [{"sql": "'short'", "cell": "short"}], + "edit": {"text": "", "cell": ""} + }, + { + "type": "CHARACTER(5)", + "covers": ["CHARACTER"], + "values": [{"sql": "'ab'", "cell": "ab"}], + "edit": {"text": "xyz", "cell": "xyz"} + }, + { + "type": "CHAR(5)", + "covers": ["CHAR"], + "values": [{"sql": "'ab'", "cell": "ab"}], + "edit": {"text": "xyz", "cell": "xyz"} + }, + { + "type": "CLOB", + "values": [{"sql": "'clob text'", "cell": "clob text"}], + "edit": {"text": "more", "cell": "more"} + }, + { + "type": "NVARCHAR(10)", + "covers": ["NVARCHAR"], + "values": [{"sql": "'ñ'", "cell": "ñ"}], + "edit": {"text": "ø", "cell": "ø"} + }, + { + "type": "NCHAR(3)", + "covers": ["NCHAR"], + "values": [{"sql": "'é'", "cell": "é"}], + "edit": {"text": "ü", "cell": "ü"} + }, + { + "type": "DATE", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "'2026-10-20'", "cell": "2026-10-20"}, + {"sql": "'0001-01-01'", "cell": "0001-01-01"}, + {"sql": "'9999-12-31'", "cell": "9999-12-31"} + ], + "edit": {"text": "2000-02-29", "cell": "2000-02-29"} + }, + { + "type": "TIME", + "values": [{"sql": "'23:59:59.999'", "cell": "23:59:59.999"}], + "edit": {"text": "00:00:00", "cell": "00:00:00"} + }, + { + "type": "DATETIME", + "values": [ + {"sql": "'2026-10-20 15:22:20'", "cell": "2026-10-20 15:22:20"}, + {"sql": "1729437740", "cell": "1729437740"} + ], + "edit": {"text": "1999-12-31 23:59:59", "cell": "1999-12-31 23:59:59"} + }, + { + "type": "TIMESTAMP", + "values": [{"sql": "'2026-10-20T15:22:20.123Z'", "cell": "2026-10-20T15:22:20.123Z"}], + "edit": {"text": "2026-01-01 00:00:00", "cell": "2026-01-01 00:00:00"} + }, + { + "type": "BLOB", + "values": [ + {"sql": "NULL", "cell": null}, + {"sql": "X''", "cell": "0x"}, + {"sql": "X'00FF7F80'", "cell": "0x00FF7F80"}, + {"sql": "X'48656C6C6F'", "cell": "0x48656C6C6F"} + ], + "edit": {"hex": "DEADBEEF", "cell": "0xDEADBEEF"} + }, + { + "type": "BOOLEAN", + "values": [{"sql": "NULL", "cell": null}, {"sql": "1", "cell": "1"}, {"sql": "0", "cell": "0"}], + "edit": {"text": "0", "cell": "0"} + } + ] +} diff --git a/apple/Plugins/FormatPluginTests/JSONExportBooleanTests.swift b/apple/Plugins/FormatPluginTests/JSONExportBooleanTests.swift new file mode 100644 index 0000000000..ef796f6336 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/JSONExportBooleanTests.swift @@ -0,0 +1,55 @@ +// +// JSONExportBooleanTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct JSONExportBooleanTests { + private final class Source: PluginExportDataSource, @unchecked Sendable { + struct NoSession: Error {} + + let databaseTypeId = "PostgreSQL" + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.yield(.header(PluginStreamHeader( + columns: ["flag", "nullable_flag", "label", "untyped", "choice"], + columnTypeNames: ["bool", "Nullable(Bool)", "text", "", "Enum8('true' = 1, 'bool' = 2)"] + ))) + continuation.yield(.rows([[.text("true"), .text("FALSE"), .text("true"), .text("false"), .text("true")]])) + continuation.finish() + } + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + func execute(query: String) async throws -> PluginQueryResult { throw NoSession() } + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + func escapeStringLiteral(_ value: String) -> String { value } + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { 1 } + } + + @Test("Only a boolean column writes true and false as JSON booleans; the same text elsewhere stays a string") + func booleansFollowTheColumn() async throws { + let plugin = JSONExportPlugin() + let stored = plugin.settings + plugin.settings = JSONExportOptions() + defer { plugin.settings = stored } + let destination = FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).json") + defer { try? FileManager.default.removeItem(at: destination) } + + _ = try await plugin.export( + tables: [PluginExportTable(name: "t", databaseName: "", tableType: "table", schema: nil, kind: .table)], + dataSource: Source(), + destination: destination, + progress: PluginExportProgress(progress: Progress(totalUnitCount: 1)) + ) + + let text = try String(contentsOf: destination, encoding: .utf8) + #expect(text.contains( + #"{"flag": true, "nullable_flag": false, "label": "true", "untyped": "false", "choice": "true"}"# + )) + } +} diff --git a/apple/Plugins/FormatPluginTests/SQLExportBinaryLiteralTests.swift b/apple/Plugins/FormatPluginTests/SQLExportBinaryLiteralTests.swift index 77a5ad6cf2..e0ca6019b4 100644 --- a/apple/Plugins/FormatPluginTests/SQLExportBinaryLiteralTests.swift +++ b/apple/Plugins/FormatPluginTests/SQLExportBinaryLiteralTests.swift @@ -16,7 +16,7 @@ struct SQLExportBinaryLiteralTests { @Test("MySQL and SQLite keep the hex literal they have always taken") func hexLiteralEnginesAreUnchanged() { - for typeId in ["MySQL", "MariaDB", "TiDB", "OceanBase", "SQLite", "libSQL", "Turso", "DuckDB", "Cloudflare D1"] { + for typeId in ["MySQL", "MariaDB", "TiDB", "OceanBase", "SQLite", "libSQL", "Turso", "Cloudflare D1"] { #expect(SQLExportBinaryLiteral.render(sample, databaseTypeId: typeId) == "X'414243'") } } @@ -32,6 +32,13 @@ struct SQLExportBinaryLiteralTests { } } + // Measured on DuckDB 1.5.2: X'414243' stores the bytes of the text x414243. + @Test("DuckDB takes from_hex, which is the BLOB itself") + func duckDBUsesFromHex() { + #expect(SQLExportBinaryLiteral.render(sample, databaseTypeId: "DuckDB") == "from_hex('414243')") + #expect(SQLExportBinaryLiteral.render(Data(), databaseTypeId: "DuckDB") == "from_hex('')") + } + @Test("SQL Server takes a 0x constant") func sqlServerUsesAHexConstant() { #expect(SQLExportBinaryLiteral.render(sample, databaseTypeId: "SQL Server") == "0x414243") diff --git a/apple/Plugins/FormatPluginTests/SQLExportNationalLiteralTests.swift b/apple/Plugins/FormatPluginTests/SQLExportNationalLiteralTests.swift new file mode 100644 index 0000000000..ab3832fe45 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/SQLExportNationalLiteralTests.swift @@ -0,0 +1,34 @@ +// +// SQLExportNationalLiteralTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +// Measured on SQL Server 2022: '日本' into an nvarchar column restores as ??, N'日本' as 日本. db-lib reports that +// column as varchar, so every text literal is national. +struct SQLExportNationalLiteralTests { + private func encoder(databaseTypeId: String) -> SQLExportRowValueEncoder { + SQLExportRowValueEncoder( + columns: ["n", "d", "i"], + columnTypeNames: ["varchar", "date", "int"], + excludedColumnNames: [], + databaseTypeId: databaseTypeId, + escapeStringLiteral: { $0.replacingOccurrences(of: "'", with: "''") } + ) + } + + @Test("Every SQL Server text literal is N''; a number stays bare") + func sqlServerTextIsNational() { + let row = encoder(databaseTypeId: "SQL Server").render([.text("日本'"), .text("2026-10-20"), .text("7")]) + #expect(row == " (N'日本''', N'2026-10-20', 7)") + } + + @Test("Other engines keep the plain literal") + func otherEnginesArePlain() { + let row = encoder(databaseTypeId: "PostgreSQL").render([.text("日本"), .text("2026-10-20"), .text("7")]) + #expect(row == " ('日本', '2026-10-20', 7)") + } +} diff --git a/apple/Plugins/SQLExportPlugin/SQLExportBinaryLiteral.swift b/apple/Plugins/SQLExportPlugin/SQLExportBinaryLiteral.swift index b63d092a2c..8fcff51093 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportBinaryLiteral.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportBinaryLiteral.swift @@ -36,8 +36,11 @@ import TableProPluginKit /// `max_string_size = EXTENDED` accepts up to 16,383 bytes, and `SQLExportRowValueEncoder` counts it /// so the export reports how many values it could not promise rather than claiming a clean dump. /// +/// DuckDB reads `X'4142'` as the text `x4142` and stores those bytes, so a restore wrote different +/// bytes and reported success. `from_hex('4142')` is the BLOB itself. Measured on DuckDB 1.5.2. +/// /// An engine this does not name keeps `X''`, which is what the export has always written, so no -/// engine's output changes except the two measured to have been wrong. +/// engine's output changes except the three measured to have been wrong. internal enum SQLExportBinaryLiteral { /// The most binary bytes Oracle can carry in one `HEXTORAW` literal: 4,000 characters of hex, /// two per byte. Measured, not inferred. @@ -56,6 +59,9 @@ internal enum SQLExportBinaryLiteral { if databaseTypeId == "Oracle" { return data.isEmpty ? "EMPTY_BLOB()" : "HEXTORAW('\(hex)')" } + if databaseTypeId == "DuckDB" { + return "from_hex('\(hex)')" + } switch SqlDialect.from(databaseTypeId: databaseTypeId) { case .postgres: return "decode('\(hex)', 'hex')" diff --git a/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift b/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift index 651d8456ab..ef9bdb9942 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift @@ -62,6 +62,13 @@ internal struct SQLExportRowValueEncoder { self.escapeStringLiteral = escapeStringLiteral } + // SQL Server reads '...' in the database's code page, so 日本 in an nvarchar restored as ??, and db-lib reports an + // nvarchar column as varchar, so the type cannot pick the columns. N'...' converts back into any text column. + // Measured on SQL Server 2022. + private var stringLiteralPrefix: String { + databaseTypeId == "SQL Server" ? "N" : "" + } + internal var writesNothing: Bool { includedColumnIndices.isEmpty } internal func columnNames(from columns: [String]) -> [String] { @@ -83,7 +90,7 @@ internal struct SQLExportRowValueEncoder { if numericIndices.contains(columnIndex), PluginNumericLiteral.isValid(value) { return value } - return "'\(escapeStringLiteral(value))'" + return "\(stringLiteralPrefix)'\(escapeStringLiteral(value))'" } } return " (\(values.joined(separator: ", ")))" diff --git a/apple/Plugins/SQLiteDriverTests/ColumnTypeCorpusTests.swift b/apple/Plugins/SQLiteDriverTests/ColumnTypeCorpusTests.swift new file mode 100644 index 0000000000..b95d469786 --- /dev/null +++ b/apple/Plugins/SQLiteDriverTests/ColumnTypeCorpusTests.swift @@ -0,0 +1,73 @@ +// +// ColumnTypeCorpusTests.swift +// SQLiteDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +// SQLite needs no server, so its corpus runs here; the server engines run theirs through +// scripts/probes/check-column-type-corpus.sh. +struct ColumnTypeCorpusTests { + private static let corpusFile: URL = { + var directory = URL(fileURLWithPath: #filePath) + for _ in 0..<2 { directory.deleteLastPathComponent() } + return directory.appendingPathComponent("ColumnTypeCorpus/Corpus/sqlite.json") + }() + + @Test("Every SQLite column type reads, saves, reads back and exports as the table tab shows it") + func sqliteCorpus() async throws { + let corpus = try ColumnTypeCorpus.load(Self.corpusFile) + let work = FileManager.default.temporaryDirectory.appendingPathComponent("sqlite-corpus-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: work, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: work) } + + let driver = SQLitePluginDriver(config: DriverConnectionConfig( + host: "", port: 0, username: "", password: "", database: work.appendingPathComponent("corpus.sqlite").path + )) + try await driver.connect() + defer { driver.disconnect() } + + // The exporters keep their options in the shared defaults, so each runs on the defaults and gets its own back. + let csv = CSVExportPlugin() + let json = JSONExportPlugin() + let sql = SQLExportPlugin() + let stored = (csv.settings, json.settings, sql.settings) + csv.settings = CSVExportOptions() + json.settings = JSONExportOptions() + sql.settings = SQLExportOptions() + defer { + csv.settings = stored.0 + json.settings = stored.1 + sql.settings = stored.2 + } + + let runner = ColumnTypeCorpusRunner( + corpus: corpus, + driver: driver, + listedTypes: SQLitePlugin.columnTypesByCategory, + dialect: SQLitePlugin.sqlDialect, + supportsCascadeDrop: SQLitePlugin.supportsCascadeDrop, + exports: [ + ColumnTypeCorpusExport(formatId: "csv", optionValues: []) { tables, source, destination in + _ = try await csv.export(tables: tables, dataSource: source, destination: destination, progress: Self.progress) + }, + ColumnTypeCorpusExport(formatId: "json", optionValues: []) { tables, source, destination in + _ = try await json.export(tables: tables, dataSource: source, destination: destination, progress: Self.progress) + }, + ColumnTypeCorpusExport(formatId: "sql", optionValues: [false, false, true]) { tables, source, destination in + _ = try await sql.export(tables: tables, dataSource: source, destination: destination, progress: Self.progress) + }, + ], + workDirectory: work + ) + let report = await runner.run { _ in } + #expect(report.failures.isEmpty, "\(report.failures.joined(separator: "\n"))") + #expect(report.entries.count == corpus.columns.count * ColumnTypeCorpus.Step.allCases.count + 1) + } + + private static var progress: PluginExportProgress { + PluginExportProgress(progress: Progress(totalUnitCount: 1)) + } +} diff --git a/apple/Plugins/TableProPluginKit/ClickHouseResponseClassifier.swift b/apple/Plugins/TableProPluginKit/ClickHouseResponseClassifier.swift index 9cd46409fa..7f03e3d74a 100644 --- a/apple/Plugins/TableProPluginKit/ClickHouseResponseClassifier.swift +++ b/apple/Plugins/TableProPluginKit/ClickHouseResponseClassifier.swift @@ -108,7 +108,8 @@ public enum ClickHouseResponseClassifier { row.append(.null) continue } - let value = ClickHouseTabSeparatedBytes.unescape(field) + let columnType = column < columnTypeNames.count ? columnTypeNames[column] : "" + let value = ClickHouseTabSeparatedBytes.value(field, columnType: columnType) guard let text = utf8Text(value) else { binaryColumns.insert(column) row.append(.bytes(Data(value))) diff --git a/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedBytes.swift b/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedBytes.swift index a71f741181..43df12f21a 100644 --- a/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedBytes.swift +++ b/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedBytes.swift @@ -26,6 +26,20 @@ internal enum ClickHouseTabSeparatedBytes { bytes.allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0A || $0 == 0x0D || $0 == 0x0B || $0 == 0x0C } } + // An Array, Map, Tuple or Nested value is written in ClickHouse's literal syntax with no TSV escaping over it, + // so `['it\'s']` is already the value's text and unescaping it reads `['it's']`. Measured on ClickHouse 25.8. + static func value(_ field: ArraySlice, columnType: String) -> [UInt8] { + writesLiteralSyntax(columnType) ? Array(field) : unescape(field) + } + + static func writesLiteralSyntax(_ columnType: String) -> Bool { + var type = Substring(columnType) + for wrapper in ["LowCardinality(", "Nullable("] where type.hasPrefix(wrapper) { + type = type.dropFirst(wrapper.count) + } + return ["Array(", "Map(", "Tuple(", "Nested("].contains { type.hasPrefix($0) } + } + /// The escape table ClickHouse's own `writeEscapedString` emits. An unrecognised escape keeps /// both of its bytes: a future addition then reads oddly rather than losing the character. static func unescape(_ field: ArraySlice) -> [UInt8] { diff --git a/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedRowDecoder.swift b/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedRowDecoder.swift index 66af57f23a..f4ff899bf2 100644 --- a/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedRowDecoder.swift +++ b/apple/Plugins/TableProPluginKit/ClickHouseTabSeparatedRowDecoder.swift @@ -66,13 +66,15 @@ public struct ClickHouseTabSeparatedRowDecoder: Sendable { ) return nil } - guard !line.isEmpty else { return nil } - return ClickHouseTabSeparatedBytes.fields(line).map(Self.cellValue) + guard !line.isEmpty, let types = header?.columnTypeNames else { return nil } + return ClickHouseTabSeparatedBytes.fields(line).enumerated().map { index, field in + Self.cellValue(field, columnType: index < types.count ? types[index] : "") + } } - private static func cellValue(_ field: ArraySlice) -> PluginCellValue { + private static func cellValue(_ field: ArraySlice, columnType: String) -> PluginCellValue { guard !ClickHouseTabSeparatedBytes.isNullMarker(field) else { return .null } - let value = ClickHouseTabSeparatedBytes.unescape(field) + let value = ClickHouseTabSeparatedBytes.value(field, columnType: columnType) guard let text = String(bytes: value, encoding: .utf8) else { return .bytes(Data(value)) } return .text(text) } diff --git a/apple/changelog.d/3710.fixed.1.md b/apple/changelog.d/3710.fixed.1.md new file mode 100644 index 0000000000..bde26765eb --- /dev/null +++ b/apple/changelog.d/3710.fixed.1.md @@ -0,0 +1 @@ +A DuckDB SQL export restored binary values as different bytes. diff --git a/apple/changelog.d/3710.fixed.2.md b/apple/changelog.d/3710.fixed.2.md new file mode 100644 index 0000000000..2bcc0e1935 --- /dev/null +++ b/apple/changelog.d/3710.fixed.2.md @@ -0,0 +1 @@ +ClickHouse dropped the backslash before a quote inside an Array, Map or Tuple value, and saving that row failed. diff --git a/apple/changelog.d/3710.fixed.3.md b/apple/changelog.d/3710.fixed.3.md new file mode 100644 index 0000000000..bc22c3e68d --- /dev/null +++ b/apple/changelog.d/3710.fixed.3.md @@ -0,0 +1 @@ +Saving a ClickHouse row that held a backslash, tab or newline changed the value or failed. diff --git a/apple/changelog.d/3710.fixed.4.md b/apple/changelog.d/3710.fixed.4.md new file mode 100644 index 0000000000..821cb91c2f --- /dev/null +++ b/apple/changelog.d/3710.fixed.4.md @@ -0,0 +1 @@ +SQL Server showed a date with a midnight time, and a time with the date 1900-01-01. diff --git a/apple/changelog.d/3710.fixed.5.md b/apple/changelog.d/3710.fixed.5.md new file mode 100644 index 0000000000..0688eda564 --- /dev/null +++ b/apple/changelog.d/3710.fixed.5.md @@ -0,0 +1 @@ +A SQL Server SQL export restored text outside the database code page as question marks. diff --git a/apple/project.yml b/apple/project.yml index 90ae04846f..9dff5bf9e6 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -757,6 +757,23 @@ targets: templates: [PluginTests, SQLiteLinkage] templateAttributes: folder: SQLiteDriverPlugin + # SQLite runs the column type corpus here, through the export formats a user picks. + sources: + - path: Plugins/ColumnTypeCorpus + excludes: + - Corpus + - path: Plugins/CSVExportPlugin + excludes: + - Info.plist + - path: Plugins/JSONExportPlugin + excludes: + - Info.plist + - path: Plugins/SQLExportPlugin + excludes: + - Info.plist + dependencies: + - package: TableProCore + product: TableProSQLGrammar ClickHouseDriverTests: templates: [PluginTests, ClickHouseLinkage] diff --git a/apple/scripts/ci/check-column-type-corpus.py b/apple/scripts/ci/check-column-type-corpus.py new file mode 100644 index 0000000000..7c671a65f8 --- /dev/null +++ b/apple/scripts/ci/check-column-type-corpus.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Fail when a driver lists a column type its corpus has no column for. + +Each Plugins/ColumnTypeCorpus/Corpus/.json names, under "typeList", the Swift file that holds its driver's +`columnTypesByCategory`, a dictionary literal or one built from a `categories` literal. A list this cannot read +fails, so a reshaped list never reads as empty. The probe, scripts/probes/check-column-type-corpus.sh, checks the +compiled list too. + +Usage: + check-column-type-corpus.py [--corpus ] [--root ] +""" + +import argparse +import json +import re +import sys +from pathlib import Path + +APPLE_DIR = Path(__file__).resolve().parents[2] +CORPUS_DIR = APPLE_DIR / "Plugins" / "ColumnTypeCorpus" / "Corpus" +STEPS = {"create", "insert", "columns", "browse", "stream", "edit", "csv", "json", "sql"} +DECLARATION = re.compile(r"\bstatic\s+let\s+(columnTypesByCategory|categories)\b[^=]*=\s*\[") + + +class Unreadable(Exception): + pass + + +def literal_after(text, start): + depth = 0 + index = start + while index < len(text): + character = text[index] + if character == '"': + index = text.index('"', index + 1) + while text[index - 1] == "\\": + index = text.index('"', index + 1) + elif character == "[": + depth += 1 + elif character == "]": + depth -= 1 + if depth == 0: + return text[start:index + 1] + index += 1 + raise Unreadable("an unterminated list") + + +# The strings inside an inner list are the types; the ones outside are category names. +def listed_types(source): + declarations = {match.group(1): match for match in DECLARATION.finditer(source)} + match = declarations.get("columnTypesByCategory") or declarations.get("categories") + if match is None: + raise Unreadable("no columnTypesByCategory or categories list literal") + literal = literal_after(source, match.end() - 1) + types = [] + depth = 0 + index = 0 + while index < len(literal): + character = literal[index] + if character == "[": + depth += 1 + elif character == "]": + depth -= 1 + elif character == '"': + end = literal.index('"', index + 1) + if depth >= 2: + types.append(literal[index + 1:end]) + index = end + index += 1 + if not types: + raise Unreadable("a type list with no types") + return types + + +def base_name(declaration): + return declaration.split("(", 1)[0].strip() + + +def problems_in(corpus_path, apple_dir): + try: + corpus = json.loads(corpus_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise Unreadable(f"does not parse: {error}") from error + type_list = apple_dir / corpus.get("typeList", "") + if not corpus.get("typeList") or not type_list.is_file(): + raise Unreadable(f"typeList {corpus.get('typeList')!r} is not a file") + + problems = [] + covered = set(corpus.get("notColumnTypes", {})) + for column in corpus.get("columns", []): + declaration = column.get("type", "") + covered.update(column["covers"] if "covers" in column else [base_name(declaration)]) + for key in ("skip", "known"): + for step in column.get(key, {}): + if step not in STEPS: + problems.append(f"{declaration}: {key} names {step!r}, not a step") + for step, issue in column.get("known", {}).items(): + if not (isinstance(issue, dict) and isinstance(issue.get("defect"), str) + and isinstance(issue.get("matches"), str)): + problems.append(f"{declaration}: known {step!r} needs a defect and the failure it matches") + if "edit" not in column and "edit" not in column.get("skip", {}): + problems.append(f"{declaration}: no edit and no skip reason for it") + + try: + listed = listed_types(type_list.read_text(encoding="utf-8")) + except Unreadable as error: + raise Unreadable(f"{corpus['typeList']}: {error}") from error + missing = sorted(set(listed) - covered) + if missing: + problems.append(f"{corpus['typeList']} lists {', '.join(missing)} with no corpus column") + return problems + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--corpus", type=Path, default=CORPUS_DIR) + parser.add_argument("--root", type=Path, default=APPLE_DIR) + args = parser.parse_args(argv) + + corpora = sorted(args.corpus.glob("*.json")) + if not corpora: + sys.exit(f"no corpus under {args.corpus}") + + failures = [] + for corpus in corpora: + try: + failures += [f"{corpus.name}: {problem}" for problem in problems_in(corpus, args.root)] + except Unreadable as error: + failures.append(f"{corpus.name}: {error}") + for failure in failures: + print(failure, file=sys.stderr) + if failures: + sys.exit(1) + print(f"{len(corpora)} corpora cover every listed type") + + +if __name__ == "__main__": + main() diff --git a/apple/scripts/ci/test_check_column_type_corpus.py b/apple/scripts/ci/test_check_column_type_corpus.py new file mode 100644 index 0000000000..ef4cd480ff --- /dev/null +++ b/apple/scripts/ci/test_check_column_type_corpus.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Fixture tests for check-column-type-corpus.py.""" + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "check_column_type_corpus", Path(__file__).with_name("check-column-type-corpus.py") +) +check = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(check) + +DICTIONARY = """final class ExamplePlugin: NSObject, DriverPlugin { + static let databaseGroupingStrategy: GroupingStrategy = .bySchema + static let columnTypesByCategory: [String: [String]] = [ + "Integer": ["INT", "BIGINT"], + "String": ["VARCHAR", "TEXT"], + "Binary": [] + ] + static let systemSchemaNames = ["pg_catalog"] +} +""" + +CATEGORIES = """internal enum ExampleTypeCatalog { + static let categories: [(name: String, types: [String])] = [ + ("Integer", ["NUMBER"]), + ("Date", [ + "DATE", "TIMESTAMP WITH TIME ZONE" + ]) + ] + + static let columnTypesByCategory: [String: [String]] = Dictionary( + uniqueKeysWithValues: categories.map { ($0.name, $0.types) } + ) +} +""" + +COMPUTED = """final class ExamplePlugin: NSObject, DriverPlugin { + static let columnTypesByCategory = ExampleMetadata.columnTypesByCategory +} +""" + + +def column(declaration, **extra): + return {"type": declaration, "values": [], "edit": {"text": "1", "cell": "1"}, **extra} + + +class CheckColumnTypeCorpusTests(unittest.TestCase): + def problems(self, source, columns, **extra): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "Plugins").mkdir() + (root / "Plugins" / "Example.swift").write_text(source, encoding="utf-8") + corpus = root / "example.json" + corpus.write_text( + json.dumps({"typeList": "Plugins/Example.swift", "columns": columns, **extra}), encoding="utf-8" + ) + return check.problems_in(corpus, root) + + def test_a_covered_dictionary_passes(self): + columns = [column("INT"), column("BIGINT"), column("VARCHAR(20)"), column("TEXT")] + self.assertEqual(self.problems(DICTIONARY, columns), []) + + def test_a_listed_type_without_a_column_fails(self): + problems = self.problems(DICTIONARY, [column("INT"), column("VARCHAR(20)"), column("TEXT")]) + self.assertEqual(problems, ["Plugins/Example.swift lists BIGINT with no corpus column"]) + + def test_covers_names_the_listed_type(self): + columns = [column("INTEGER[]", covers=["INT", "BIGINT"]), column("VARCHAR"), column("TEXT")] + self.assertEqual(self.problems(DICTIONARY, columns), []) + + def test_a_type_no_column_can_have_is_covered_by_its_reason(self): + columns = [column("INT"), column("VARCHAR"), column("TEXT")] + reasons = {"BIGINT": "not a column type"} + self.assertEqual(self.problems(DICTIONARY, columns, notColumnTypes=reasons), []) + + def test_a_list_built_from_categories_is_read(self): + problems = self.problems(CATEGORIES, [column("NUMBER(10)"), column("DATE")]) + self.assertEqual(problems, ["Plugins/Example.swift lists TIMESTAMP WITH TIME ZONE with no corpus column"]) + + def test_a_list_it_cannot_read_fails(self): + with self.assertRaises(check.Unreadable): + self.problems(COMPUTED, [column("INT")]) + + def test_a_column_needs_an_edit_or_a_reason(self): + columns = [column("INT"), column("BIGINT"), column("VARCHAR"), {"type": "TEXT", "values": []}] + self.assertEqual(self.problems(DICTIONARY, columns), ["TEXT: no edit and no skip reason for it"]) + + def test_an_unknown_step_fails(self): + known = {"grid": {"defect": "x", "matches": "x"}} + columns = [column("INT", known=known), column("BIGINT"), column("VARCHAR"), column("TEXT")] + self.assertEqual(self.problems(DICTIONARY, columns), ["INT: known names 'grid', not a step"]) + + def test_a_known_issue_names_the_failure_it_stands_for(self): + columns = [column("INT", known={"browse": "reads wrong"}), column("BIGINT"), column("VARCHAR"), column("TEXT")] + self.assertEqual( + self.problems(DICTIONARY, columns), ["INT: known 'browse' needs a defect and the failure it matches"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/apple/scripts/probes/check-column-type-corpus.sh b/apple/scripts/probes/check-column-type-corpus.sh new file mode 100755 index 0000000000..7f7eff47b7 --- /dev/null +++ b/apple/scripts/probes/check-column-type-corpus.sh @@ -0,0 +1,347 @@ +#!/usr/bin/env bash +# +# Take every column type an engine's driver lists through browse, edit and export against a real server. +# +# A type a driver has never seen can crash it or show the wrong value (#3241, #2454, #2130). This builds a harness +# from the driver's own sources, the CSV, JSON and SQL export plugins and the runner in Plugins/ColumnTypeCorpus, then +# runs Plugins/ColumnTypeCorpus/Corpus/.json: per column it creates a table, inserts the values, reads them +# through the table tab's query and the export's stream, saves one edit through the grid's statements, reads it +# back, and exports the table three ways. The SQL dump is replayed into the emptied table and has to read back the +# same. A listed type with no corpus column fails too. +# +# A corpus column is {"type", "covers", "values": [{"sql", "cell"}], "edit": {"text" | "hex", "cell"}, "skip", +# "known"}. "cell" is the text the driver hands the grid for the value, null for NULL, 0x-hex for bytes. "covers" +# names the listed types the column stands for, by default the type up to its "(". "skip" names a step that does not +# apply to the type and why; "known" names a step that fails today and the defect, and fails the run once it passes. +# +# Usage: +# scripts/probes/check-column-type-corpus.sh [--only ] [--keep] +# +# engine: postgresql, mysql, mariadb, clickhouse, duckdb, mssql, oracle, sqlite +# --only run the columns whose declared type contains +# --keep leave the tables behind for a look +# +# With no server listening on the engine's port, the script starts one in Docker as tablepro-corpus- and +# leaves it running for the next run. TP_CORPUS_HOST, TP_CORPUS_PORT, TP_CORPUS_USER, TP_CORPUS_PASSWORD and +# TP_CORPUS_DATABASE point it at another server instead. TP_CORPUS_BUILD_DIR keeps the harness between runs. +# Exits 1 when a check fails, 3 when it cannot run. + +set -uo pipefail + +ENGINE="${1:-}" +shift || true +ONLY="" +KEEP="" +while [ $# -gt 0 ]; do + case "$1" in + --only) + [ $# -ge 2 ] || { echo "--only needs a type" >&2; exit 3; } + ONLY="$2" + shift 2 + ;; + --keep) KEEP=1; shift ;; + *) echo "unknown argument $1" >&2; exit 3 ;; + esac +done + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PLUGINS="$ROOT/Plugins" +CORPUS="$PLUGINS/ColumnTypeCorpus/Corpus/$ENGINE.json" +PASSWORD_DEFAULT="TpCorpus#2026" + +# Per engine: plugin folder, principal class, C module, TableProCore products, link flags, image, port, user, +# database, and the command that answers once the server takes connections. +C_MODULE="" +PRODUCTS="TableProLogRedaction" +LINK="" +IMAGE="" +PORT_IN="" +USER_DEFAULT="" +DATABASE_DEFAULT="" +FIELDS="" +DOCKER_ENV=() +READY=() +case "$ENGINE" in + postgresql) + FOLDER=PostgreSQLDriverPlugin CLASS=PostgreSQLPlugin C_MODULE=CLibPQ + PRODUCTS="TableProLogRedaction TableProPostgreSQLCore" + LINK="$ROOT/Libs/libpq.a $ROOT/Libs/libpgcommon.a $ROOT/Libs/libpgport.a -lssl.3 -lcrypto.3 -lz" + IMAGE=postgres:17-alpine PORT_IN=5432 PORT=15467 USER_DEFAULT=postgres DATABASE_DEFAULT=postgres + DOCKER_ENV=(-e "POSTGRES_PASSWORD=$PASSWORD_DEFAULT") + READY=(pg_isready -h 127.0.0.1 -U postgres) + ;; + mysql | mariadb) + FOLDER=MySQLDriverPlugin CLASS=MySQLPlugin C_MODULE=CMariaDB + PRODUCTS="TableProGeometry TableProLogRedaction TableProMySQLCore" + LINK="$ROOT/Libs/libmariadb.a -lssl.3 -lcrypto.3 -liconv -lz" + USER_DEFAULT=root DATABASE_DEFAULT=tp_corpus PORT_IN=3306 + if [ "$ENGINE" = mysql ]; then + IMAGE=mysql:8.4 PORT=13367 + DOCKER_ENV=(-e "MYSQL_ROOT_PASSWORD=$PASSWORD_DEFAULT" -e MYSQL_DATABASE=tp_corpus) + READY=(mysqladmin ping -h 127.0.0.1 -uroot "-p$PASSWORD_DEFAULT" --silent) + else + IMAGE=mariadb:11.8 PORT=13368 + DOCKER_ENV=(-e "MARIADB_ROOT_PASSWORD=$PASSWORD_DEFAULT" -e MARIADB_DATABASE=tp_corpus) + READY=(mariadb-admin ping -h 127.0.0.1 -uroot "-p$PASSWORD_DEFAULT" --silent) + fi + ;; + clickhouse) + FOLDER=ClickHouseDriverPlugin CLASS=ClickHousePlugin + PRODUCTS="TableProNumberFormatting TableProTLSClientIdentity" + IMAGE=clickhouse/clickhouse-server:25.8 PORT_IN=8123 PORT=18123 USER_DEFAULT=default DATABASE_DEFAULT=tp_corpus + DOCKER_ENV=(-e "CLICKHOUSE_PASSWORD=$PASSWORD_DEFAULT" -e CLICKHOUSE_DB=tp_corpus) + READY=(clickhouse-client --password "$PASSWORD_DEFAULT" --query "SELECT 1") + ;; + duckdb) + FOLDER=DuckDBDriverPlugin CLASS=DuckDBPlugin C_MODULE=CDuckDB PRODUCTS="" + LINK="$ROOT/Libs/libduckdb.a -lc++" + ;; + mssql) + FOLDER=MSSQLDriverPlugin CLASS=MSSQLPlugin C_MODULE=CFreeTDS + PRODUCTS="TableProCoreTypes TableProLogRedaction TableProMSSQLCore" + LINK="$ROOT/Libs/libsybdb.a -lssl.3 -lcrypto.3 -liconv -framework GSS -lcom_err" + # SQL Server proper rather than Azure SQL Edge, which has no CLR types (geometry, geography, hierarchyid). + IMAGE=mcr.microsoft.com/mssql/server:2022-latest PORT_IN=1433 PORT=14367 USER_DEFAULT=sa DATABASE_DEFAULT=tp_corpus + DOCKER_ENV=(--platform linux/amd64 -e ACCEPT_EULA=Y -e "MSSQL_SA_PASSWORD=$PASSWORD_DEFAULT") + READY=(/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P "$PASSWORD_DEFAULT" + -Q "IF DB_ID('tp_corpus') IS NULL CREATE DATABASE tp_corpus") + ;; + oracle) + FOLDER=OracleDriverPlugin CLASS=OraclePlugin + # Not a slim image: those leave out Spatial, so SDO_GEOMETRY does not exist. + IMAGE=gvenzl/oracle-free:23-faststart PORT_IN=1521 PORT=15421 USER_DEFAULT=probe DATABASE_DEFAULT=FREEPDB1 + DOCKER_ENV=(-e "ORACLE_PASSWORD=$PASSWORD_DEFAULT" -e APP_USER=probe -e "APP_USER_PASSWORD=$PASSWORD_DEFAULT") + READY=(healthcheck.sh) + FIELDS="oracleServiceName=FREEPDB1" + ;; + sqlite) + FOLDER=SQLiteDriverPlugin CLASS=SQLitePlugin PRODUCTS="CSQLite TableProSQLiteCore" + LINK="$ROOT/Libs/libsqlite3_vendored.a" + ;; + *) + echo "usage: $0 [--only ] [--keep]" >&2 + exit 3 + ;; +esac + +[ -f "$CORPUS" ] || { echo "no corpus at $CORPUS" >&2; exit 3; } +command -v swift > /dev/null || { echo "swift not found" >&2; exit 3; } +for library in $LINK; do + case "$library" in + /*) [ -f "$library" ] || { echo "not found: $library (run scripts/download-libs.sh)" >&2; exit 3; } ;; + esac +done + +HOST="${TP_CORPUS_HOST:-127.0.0.1}" +PORT="${TP_CORPUS_PORT:-${PORT:-0}}" +USER_NAME="${TP_CORPUS_USER:-$USER_DEFAULT}" +PASSWORD="${TP_CORPUS_PASSWORD:-$PASSWORD_DEFAULT}" +DATABASE="${TP_CORPUS_DATABASE:-$DATABASE_DEFAULT}" +CONTAINER="tablepro-corpus-$ENGINE" + +if [ -n "$IMAGE" ] && ! nc -z "$HOST" "$PORT" > /dev/null 2>&1; then + command -v docker > /dev/null || { echo "no server at $HOST:$PORT and no docker to start one" >&2; exit 3; } + if docker ps -a --format '{{.Names}}' | grep -qx "$CONTAINER"; then + echo "starting container $CONTAINER" + docker start "$CONTAINER" > /dev/null || exit 3 + else + echo "starting $IMAGE as $CONTAINER on $HOST:$PORT" + docker run -d --name "$CONTAINER" ${DOCKER_ENV[@]+"${DOCKER_ENV[@]}"} -p "$HOST:$PORT:$PORT_IN" "$IMAGE" \ + > /dev/null || exit 3 + fi +fi +if [ ${#READY[@]} -gt 0 ] && docker ps --format '{{.Names}}' 2> /dev/null | grep -qx "$CONTAINER"; then + for _ in $(seq 1 120); do + docker exec "$CONTAINER" "${READY[@]}" > /dev/null 2>&1 && break + sleep 2 + done +fi + +WORK="${TP_CORPUS_BUILD_DIR:-}" +if [ -z "$WORK" ]; then + WORK="$(mktemp -d)" + trap 'rm -rf "$WORK"' EXIT +fi +SOURCES="$WORK/Sources/Probe" +rm -rf "$SOURCES" "$WORK/CModule" +mkdir -p "$SOURCES" + +link_sources() { + local prefix="$1" directory="$2" source + while IFS= read -r source; do + ln -s "$source" "$SOURCES/${prefix}_$(basename "$source")" + done < <(find "$directory" -name '*.swift' -not -path '*/Tests/*') +} +link_sources driver "$PLUGINS/$FOLDER" +link_sources corpus "$PLUGINS/ColumnTypeCorpus" +link_sources csv "$PLUGINS/CSVExportPlugin" +link_sources json "$PLUGINS/JSONExportPlugin" +link_sources sql "$PLUGINS/SQLExportPlugin" +for shared in PluginConnectTimeout PluginQueryTimeout PluginURLSessionTaskBox; do + ln -s "$PLUGINS/Shared/$shared.swift" "$SOURCES/shared_$shared.swift" +done + +dependencies='.product(name: "TableProPluginKit", package: "TableProCore"), + .product(name: "TableProSQLGrammar", package: "TableProCore"),' +for product in $PRODUCTS; do + dependencies="$dependencies + .product(name: \"$product\", package: \"TableProCore\")," +done +packages=".package(path: \"$ROOT/Packages/TableProCore\")," +if [ "$ENGINE" = oracle ]; then + packages="$packages + .package(path: \"$ROOT/Packages/TableProOracle\")," + dependencies="$dependencies + .product(name: \"TableProOracleCore\", package: \"TableProOracle\")," +fi +system_target="" +swift_flags="" +if [ -n "$C_MODULE" ]; then + swift_flags=", .unsafeFlags([\"-Xcc\", \"-I$PLUGINS/$FOLDER/$C_MODULE/include\"])" + ln -s "$PLUGINS/$FOLDER/$C_MODULE" "$WORK/CModule" + system_target=".systemLibrary(name: \"$C_MODULE\", path: \"CModule\")," + dependencies="$dependencies + \"$C_MODULE\"," +fi +flags="\"-L$ROOT/Libs\", \"-L$ROOT/Libs/dylibs\", \"-Xlinker\", \"-rpath\", \"-Xlinker\", \"$ROOT/Libs/dylibs\"" +for flag in $LINK; do + flags="$flags, \"$flag\"" +done + +cat > "$WORK/Package.swift" << MANIFEST +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "ColumnTypeCorpusProbe", + platforms: [.macOS(.v14)], + dependencies: [ + $packages + ], + targets: [ + $system_target + .executableTarget( + name: "Probe", + dependencies: [ + $dependencies + ], + path: "Sources/Probe", + swiftSettings: [.swiftLanguageMode(.v6), .enableUpcomingFeature("MemberImportVisibility")$swift_flags], + linkerSettings: [.unsafeFlags([$flags])] + ), + ] +) +MANIFEST + +cat > "$SOURCES/probe_Engine.swift" << SWIFT +typealias EnginePlugin = $CLASS +SWIFT + +cat > "$SOURCES/probe_Probe.swift" << 'SWIFT' +import Foundation +import TableProPluginKit + +@main +enum Probe { + static func main() async { + let arguments = CommandLine.arguments + guard arguments.count >= 10 else { + print("usage: Probe ") + exit(3) + } + let corpus: ColumnTypeCorpus + do { + corpus = try ColumnTypeCorpus.load(URL(fileURLWithPath: arguments[1])) + } catch { + print("cannot read the corpus: \(error)") + exit(3) + } + var fields: [String: String] = [:] + for pair in arguments[7].split(separator: ";") { + let parts = pair.split(separator: "=", maxSplits: 1).map(String.init) + if parts.count == 2 { fields[parts[0]] = parts[1] } + } + if let variant = EnginePlugin.driverVariant(for: corpus.databaseTypeId) { + fields["driverVariant"] = variant + } + let driver = EnginePlugin().createDriver(config: DriverConnectionConfig( + host: arguments[2], port: Int(arguments[3]) ?? 0, username: arguments[4], password: arguments[5], + database: arguments[6], additionalFields: fields + )) + do { + try await driver.connect() + } catch { + print("cannot connect: \(error)") + exit(3) + } + print("\(corpus.engine): \(driver.serverVersion ?? "server version unknown")") + + let csv = CSVExportPlugin() + let json = JSONExportPlugin() + let sql = SQLExportPlugin() + let exports = [ + ColumnTypeCorpusExport(formatId: "csv", optionValues: []) { tables, source, destination in + _ = try await csv.export(tables: tables, dataSource: source, destination: destination, progress: PluginExportProgress(progress: Progress(totalUnitCount: 1))) + }, + ColumnTypeCorpusExport(formatId: "json", optionValues: []) { tables, source, destination in + _ = try await json.export(tables: tables, dataSource: source, destination: destination, progress: PluginExportProgress(progress: Progress(totalUnitCount: 1))) + }, + ColumnTypeCorpusExport(formatId: "sql", optionValues: [false, false, true]) { tables, source, destination in + _ = try await sql.export(tables: tables, dataSource: source, destination: destination, progress: PluginExportProgress(progress: Progress(totalUnitCount: 1))) + }, + ] + let work = FileManager.default.temporaryDirectory.appendingPathComponent("tp-corpus-\(getpid())") + try? FileManager.default.createDirectory(at: work, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: work) } + + let only = arguments[8] + var runner = ColumnTypeCorpusRunner( + corpus: corpus, + driver: driver, + listedTypes: EnginePlugin.columnTypesByCategory, + dialect: EnginePlugin.sqlDialect, + supportsCascadeDrop: EnginePlugin.supportsCascadeDrop, + exports: exports, + workDirectory: work + ) + runner.tablePrefix = "tpc\(getpid() % 10_000)_" + runner.keepTables = arguments[9] == "1" + if !only.isEmpty { + runner.includes = { $0.type.localizedCaseInsensitiveContains(only) } + } + let report = await runner.run { line in + print(line) + fflush(stdout) + } + driver.disconnect() + let failures = report.failures + print(failures.isEmpty ? "\nall checks hold" : "\n\(failures.count) failing check(s)") + exit(failures.isEmpty ? 0 : 1) + } +} +SWIFT + +# A database file the caller names only gets the runner's tables; a scratch one is made fresh, and outlives the run +# when --keep asks for the tables. +case "$ENGINE" in + duckdb | sqlite) + if [ -z "${TP_CORPUS_DATABASE:-}" ]; then + if [ -n "$KEEP" ]; then + DATABASE="$(mktemp -d)/corpus.$ENGINE" + echo "keeping the tables in $DATABASE" + else + DATABASE="$WORK/corpus.$ENGINE" + rm -f "$DATABASE" "$DATABASE.wal" + fi + fi + ;; +esac + +echo "building the $ENGINE harness (the first build takes a few minutes)" +(cd "$WORK" && swift build -c debug > "$WORK/build.log" 2>&1) || { + grep -E "error:" "$WORK/build.log" | head -30 >&2 + exit 3 +} +# A session takes the client's zone for TIMESTAMP WITH LOCAL TIME ZONE and the like, so the cells read the same on +# every Mac. +TZ=UTC "$WORK/.build/debug/Probe" "$CORPUS" "$HOST" "$PORT" "$USER_NAME" "$PASSWORD" "$DATABASE" "$FIELDS" "$ONLY" "$KEEP"