test(plugins): run every listed column type through browse, edit and export

.claude/rules/plugin-system.md

@@ -55,6 +55,12 @@ TableProPluginKit is built with Library Evolution, so an already-built plugin ke
 - The `TablePro` scheme builds only the bundled plugins, and a registry plugin's compile error still ends in `BUILD SUCCEEDED`. Run `verify.sh plugins` (the `AllPlugins` aggregate) after any change under `apple/Plugins/`.
 - `apple/scripts/ci/verify-plugin-loads.sh <dir>...` dlopens every built `.tableplugin` and names any that will not load. It does not instantiate the principal class.
 
+## Column type corpus
+
+- PostgreSQL, MySQL, MariaDB, ClickHouse, DuckDB, SQL Server, Oracle and SQLite each have a corpus in `apple/Plugins/ColumnTypeCorpus/Corpus/`: a column per type the driver lists, its values and the cell each reads as. A type added to `columnTypesByCategory` needs a column there; Repo Hygiene checks it.
+- After changing how one of those drivers reads, writes or exports a value, run `apple/scripts/probes/check-column-type-corpus.sh <engine>`. SQLite's corpus runs in `SQLiteDriverTests`.
+- A wrong cell is fixed in the driver, never in the corpus. A defect left for later is a `known` entry naming it, which fails the run once it passes.
+
 ## Structure
 
 - PluginKit-free logic a plugin shares with the iOS app lives in a TableProCore product, such as `TableProMySQLCore`, `TableProPostgreSQLCore` or `TableProRedisCore`.

.github/workflows/repo-hygiene.yml

@@ -117,6 +117,11 @@ jobs:
       - name: Check every test target is in its test plan
         run: python3 apple/scripts/ci/check-test-plans.py
 
+      # The corpora run by hand against servers (scripts/probes/check-column-type-corpus.sh), so a type a driver
+      # starts listing is caught here.
+      - name: Check every listed column type has a corpus column
+        run: python3 apple/scripts/ci/check-column-type-corpus.py
+
       - name: Set up Go
         if: inputs.hana
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0

apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDatetimeFormatter.swift

@@ -1,11 +1,22 @@
 import Foundation
 
 public enum MSSQLDatetimeFormatter {
+    // db-lib converts a `date` and a `time` as a datetime, so a date reads with midnight and a time with 1900-01-01:
+    // parts the column does not hold. Measured on SQL Server 2022.
     public static func reformat(_ raw: String, type: MSSQLColumnType) -> String? {
-        guard type.isDateOrTime else { return nil }
-        return parse(raw)
+        guard type.isDateOrTime, let text = parse(raw) else { return nil }
+        switch type {
+        case .date:
+            return String(text.prefix(10))
+        case .time where text.hasPrefix(timeBaseDate):
+            return String(text.dropFirst(timeBaseDate.count))
+        default:
+            return text
+        }
     }
 
+    private static let timeBaseDate = "1900-01-01 "
+
     /// db-lib's text for a `datetimeoffset` is the local wall clock with no offset, which SQL Server reads back as
     /// +00:00. The ISO 8601 form carries the offset, so the value reads back as the same instant.
     public static func dateTimeOffset(_ raw: String, offsetMinutes: Int) -> String? {

apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDatetimeFormatterTests.swift

@@ -49,6 +49,21 @@ struct MSSQLDatetimeFormatterTests {
         #expect(MSSQLDatetimeFormatter.reformat("Dec 31 2025 11:59:59:999PM", type: .dateTime2) == "2025-12-31 23:59:59.999")
     }
 
+    // Measured on SQL Server 2022: dbconvert writes a time(7) 23:59:59.9999999 as 1900-01-01 23:59:59.9999999.
+    @Test("A time reads as the time of day, without the date db-lib converts it on")
+    func timeDropsTheBaseDate() {
+        #expect(MSSQLDatetimeFormatter.reformat("1900-01-01 23:59:59.9999999", type: .time) == "23:59:59.9999999")
+        #expect(MSSQLDatetimeFormatter.reformat("Jan  1 1900 12:00:00:0000000AM", type: .time) == "00:00:00.0000000")
+        #expect(MSSQLDatetimeFormatter.reformat("1900-01-01 08:00:00", type: .dateTime) == "1900-01-01 08:00:00")
+    }
+
+    // Measured on SQL Server 2022: dbconvert writes the date 0001-01-01 as Jan  1 0001 12:00:00:0000000AM.
+    @Test("A date reads as the date alone, without the midnight db-lib adds")
+    func dateDropsTheTime() {
+        #expect(MSSQLDatetimeFormatter.reformat("Jan  1 0001 12:00:00:0000000AM", type: .date) == "0001-01-01")
+        #expect(MSSQLDatetimeFormatter.reformat("9999-12-31 00:00:00", type: .date) == "9999-12-31")
+    }
+
     @Test func testDateTimeOffsetKeepsAPositiveOffset() {
         #expect(MSSQLDatetimeFormatter.dateTimeOffset("Jan  2 2024  3:04:05:1234567AM", offsetMinutes: 330) == "2024-01-02T03:04:05.1234567+05:30")
     }

apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseResponseClassifierTests.swift

@@ -98,6 +98,19 @@ struct ClickHouseResponseClassifierTests {
         #expect(outcome.rows == [[.text("a\tb\nc\\d")]])
     }
 
+    // Measured on ClickHouse 25.8: the String column is TSV-escaped, the compound ones are not escaped again.
+    @Test("A quote or backslash inside an Array, Map or Tuple keeps the escape ClickHouse wrote")
+    func compoundValuesKeepTheirEscapes() {
+        let outcome = classify(
+            headers: matchingFormatHeaders,
+            bodyText: "a\tm\tt\ts\nArray(Nullable(String))\tMap(String, String)\tTuple(UInt8, String)\tString\n"
+                + #"['a',NULL,'it\'s']"# + "\t" + #"{'k\'':'v\\'}"# + "\t" + #"(1,'x\'y')"# + "\t" + #"it\'s"# + "\n"
+        )
+        #expect(outcome.rows == [[
+            .text(#"['a',NULL,'it\'s']"#), .text(#"{'k\'':'v\\'}"#), .text(#"(1,'x\'y')"#), .text("it's")
+        ]])
+    }
+
     // MARK: - The Full Escape Table ClickHouse Writes
 
     @Test("Every escape the server emits is decoded, not left as two characters")

apple/Packages/TableProCore/Tests/TableProPluginKitTests/ClickHouseTabSeparatedRowDecoderTests.swift

@@ -127,6 +127,20 @@ struct ClickHouseTabSeparatedRowDecoderTests {
         #expect(outcome.rows == [[.text("['a','b']"), .text("{'k':'v'}"), .text("(7,'x')")]])
     }
 
+    // Measured on ClickHouse 25.8: the String column is TSV-escaped, the compound ones are not escaped again.
+    @Test("A quote or backslash inside an Array, Map or Tuple keeps the escape ClickHouse wrote")
+    func keepsEscapesInsideCompoundValues() {
+        var body = Data(
+            "tags\tattrs\tpair\tname\nArray(Nullable(String))\tMap(String, String)\tTuple(UInt8, String)\tString\n".utf8
+        )
+        let row = #"['a',NULL,'it\'s']"# + "\t" + #"{'k\'':'v\\'}"# + "\t" + #"(1,'x\'y')"# + "\t" + #"it\'s"# + "\n"
+        body.append(contentsOf: Data(row.utf8))
+        let outcome = decodeAll(body)
+        #expect(outcome.rows == [[
+            .text(#"['a',NULL,'it\'s']"#), .text(#"{'k\'':'v\\'}"#), .text(#"(1,'x\'y')"#), .text("it's")
+        ]])
+    }
+
     // MARK: - Chunk boundaries
 
     @Test("A row split across two chunks decodes once it is whole")

apple/Plugins/ClickHouseDriverPlugin/ClickHouseParameterBinding.swift

@@ -56,12 +56,30 @@ nonisolated internal enum ClickHouseParameterBinding {
             namedCount += 1
             let name = "p\(namedCount)"
             converted.append("{\(name):String}")
-            paramMap[name] = parameter.asText
+            paramMap.updateValue(parameter.asText.map(escaped), forKey: name)
         }
 
         return Bound(query: converted, params: paramMap)
     }
 
+    // ClickHouse reads an HTTP parameter in TabSeparated's escaped form: a backslash starts an escape and a raw tab or
+    // newline is refused, so `C:\new` arrived as `C:`, a newline and `ew`. Measured on ClickHouse 25.8.
+    static func escaped(_ value: String) -> String {
+        var result = ""
+        result.unicodeScalars.reserveCapacity(value.unicodeScalars.count)
+        for scalar in value.unicodeScalars {
+            switch scalar {
+            case "\\": result += "\\\\"
+            case "\t": result += "\\t"
+            case "\n": result += "\\n"
+            case "\r": result += "\\r"
+            case "\0": result += "\\0"
+            default: result.unicodeScalars.append(scalar)
+            }
+        }
+        return result
+    }
+
     static func hexLiteral(_ data: Data) -> String {
         let hex = data.map { String(format: "%02X", $0) }.joined()
         return "unhex('\(hex)')"

apple/Plugins/ClickHouseDriverTests/ClickHouseParameterBindingTests.swift

@@ -19,6 +19,17 @@ struct ClickHouseParameterBindingTests {
         #expect(bound.params["p2"] == "two")
     }
 
+    // Measured on ClickHouse 25.8: `C:\new` sent as it is arrived as `C:`, a newline and `ew`, and a raw tab was refused.
+    @Test("A text parameter is sent in the escaped form ClickHouse reads it in")
+    func textParametersAreEscaped() {
+        let bound = ClickHouseParameterBinding.bind(
+            query: "SELECT ?, ?",
+            parameters: [.text(#"C:\new\table"#), .text("a\tb\nc\rd\u{0}e")]
+        )
+        #expect(bound.params["p1"] == #"C:\\new\\table"#)
+        #expect(bound.params["p2"] == #"a\tb\nc\rd\0e"#)
+    }
+
     @Test("A null parameter keeps its slot with no value")
     func nullParameterKeepsItsSlot() {
         let bound = ClickHouseParameterBinding.bind(

apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpus.swift

@@ -0,0 +1,135 @@
+//
+//  ColumnTypeCorpus.swift
+//  ColumnTypeCorpus
+//
+
+import Foundation
+
+struct ColumnTypeCorpus: Decodable, Sendable {
+    enum Step: String, CaseIterable, Sendable {
+        case create, insert, columns, browse, stream, edit, csv, json, sql
+    }
+
+    struct Value: Decodable, Sendable {
+        let sql: String
+        // nil is SQL NULL. The key is required, so a forgotten expectation never reads as NULL.
+        let cell: String?
+        // For a value the server makes up, such as a rowversion: the cell has to match this instead.
+        let matches: String?
+
+        private enum CodingKeys: String, CodingKey { case sql, cell, `repeat`, matches }
+
+        init(from decoder: any Decoder) throws {
+            let container = try decoder.container(keyedBy: CodingKeys.self)
+            sql = try container.decode(String.self, forKey: .sql)
+            matches = try container.decodeIfPresent(String.self, forKey: .matches)
+            if matches != nil {
+                cell = nil
+                return
+            }
+            guard container.contains(.cell) else {
+                throw DecodingError.keyNotFound(
+                    CodingKeys.cell,
+                    DecodingError.Context(codingPath: container.codingPath, debugDescription: "\(sql) has no cell")
+                )
+            }
+            let written = try container.decodeIfPresent(String.self, forKey: .cell)
+            // A value too long to write out: `cell` repeated, as REPEAT('x', 70000) reads.
+            let count = try container.decodeIfPresent(Int.self, forKey: .repeat) ?? 1
+            cell = written.map { String(repeating: $0, count: count) }
+        }
+    }
+
+    struct Edit: Decodable, Sendable {
+        let row: Int?
+        let text: String?
+        let hex: String?
+        let cell: String?
+    }
+
+    // A defect left for later, and what its failure looks like: any other failure at that step still fails the run.
+    struct KnownIssue: Decodable, Sendable {
+        let defect: String
+        let matches: String
+    }
+
+    struct Column: Decodable, Sendable {
+        let type: String
+        let covers: [String]
+        let values: [Value]
+        let edit: Edit?
+        let skip: [Step: String]
+        let known: [Step: KnownIssue]
+
+        private enum CodingKeys: String, CodingKey { case type, covers, values, edit, skip, known }
+
+        init(from decoder: any Decoder) throws {
+            let container = try decoder.container(keyedBy: CodingKeys.self)
+            type = try container.decode(String.self, forKey: .type)
+            covers = try container.decodeIfPresent([String].self, forKey: .covers) ?? [Self.baseName(of: type)]
+            values = try container.decode([Value].self, forKey: .values)
+            edit = try container.decodeIfPresent(Edit.self, forKey: .edit)
+            skip = try Self.steps(container.decodeIfPresent([String: String].self, forKey: .skip), in: container, key: .skip)
+            known = try Self.steps(
+                container.decodeIfPresent([String: KnownIssue].self, forKey: .known), in: container, key: .known
+            )
+            if edit == nil, skip[.edit] == nil {
+                throw DecodingError.dataCorruptedError(
+                    forKey: .edit, in: container, debugDescription: "\(type) needs an edit or a skip reason for it"
+                )
+            }
+        }
+
+        // The listed name a declaration stands for: `NUMERIC(10,2)` covers `NUMERIC`. Kept in step with
+        // apple/scripts/ci/check-column-type-corpus.py.
+        static func baseName(of type: String) -> String {
+            let name = type.split(separator: "(", maxSplits: 1).first.map(String.init) ?? type
+            return name.trimmingCharacters(in: .whitespaces)
+        }
+
+        private static func steps<Reason>(
+            _ raw: [String: Reason]?,
+            in container: KeyedDecodingContainer<CodingKeys>,
+            key: CodingKeys
+        ) throws -> [Step: Reason] {
+            var steps: [Step: Reason] = [:]
+            for (name, reason) in raw ?? [:] {
+                guard let step = Step(rawValue: name) else {
+                    throw DecodingError.dataCorruptedError(
+                        forKey: key, in: container, debugDescription: "\(name) is not a step"
+                    )
+                }
+                steps[step] = reason
+            }
+            return steps
+        }
+    }
+
+    let engine: String
+    let databaseTypeId: String
+    let typeList: String
+    let createTable: String
+    let dropTable: String?
+    let clearTable: String?
+    // Listed types no column can have, such as SQL Server's CURSOR, with why.
+    let notColumnTypes: [String: String]?
+    // Polled after a save until it reads 0, for an engine that applies a save after answering it.
+    let settleAfterSave: String?
+    let columns: [Column]
+
+    static func load(_ url: URL) throws -> ColumnTypeCorpus {
+        try JSONDecoder().decode(ColumnTypeCorpus.self, from: Data(contentsOf: url))
+    }
+
+    func uncoveredTypes(listed: [String: [String]]) -> [String] {
+        let covered = Set(columns.flatMap(\.covers)).union((notColumnTypes ?? [:]).keys)
+        return listed.values.flatMap { $0 }.filter { !covered.contains($0) }.sorted()
+    }
+
+    // By what the column covers, else by the first word of its type: `TINYINT UNSIGNED` is an Integer.
+    func category(of column: Column, listed: [String: [String]]) -> String? {
+        let firstWord = Column.baseName(of: column.type).split(separator: " ").first.map(String.init) ?? column.type
+        let names = column.covers + [firstWord]
+        return listed.first { _, types in names.contains(where: types.contains) }?.key
+    }
+}

apple/Plugins/ColumnTypeCorpus/ColumnTypeCorpusExportSource.swift

@@ -0,0 +1,73 @@
+//
+//  ColumnTypeCorpusExportSource.swift
+//  ColumnTypeCorpus
+//
+
+import Foundation
+import TableProPluginKit
+
+// What the host's ExportDataSourceAdapter hands an export plugin for one table of the connected database.
+final class ColumnTypeCorpusExportSource: PluginExportDataSource, @unchecked Sendable {
+    let databaseTypeId: String
+    let lexicalFeatures: SQLLexicalFeatures
+    let supportsCascadeDrop: Bool
+    private let driver: any PluginDatabaseDriver
+
+    init(
+        driver: any PluginDatabaseDriver,
+        databaseTypeId: String,
+        lexicalFeatures: SQLLexicalFeatures,
+        supportsCascadeDrop: Bool
+    ) {
+        self.driver = driver
+        self.databaseTypeId = databaseTypeId
+        self.lexicalFeatures = lexicalFeatures
+        self.supportsCascadeDrop = supportsCascadeDrop
+    }
+
+    static func readQuery(table: String, driver: any PluginDatabaseDriver) -> (sql: String, context: PluginStatementContext) {
+        var context = PluginStatementContext()
+        context.readOnly = true
+        if let custom = driver.defaultExportQuery(table: table, schema: nil) {
+            return (custom, context)
+        }
+        return ("SELECT * FROM \(driver.quoteIdentifier(table))", context)
+    }
+
+    func streamRows(table: String, databaseName: String) -> AsyncThrowingStream<PluginStreamElement, Error> {
+        let read = Self.readQuery(table: table, driver: driver)
+        return driver.streamRows(query: read.sql, context: read.context)
+    }
+
+    func fetchTableDDL(table: String, databaseName: String) async throws -> String {
+        try await driver.fetchTableDDL(table: table, schema: nil)
+    }
+
+    func execute(query: String) async throws -> PluginQueryResult {
+        try await driver.execute(query: query)
+    }
+
+    func quoteIdentifier(_ identifier: String) -> String {
+        driver.quoteIdentifier(identifier)
+    }
+
+    func escapeStringLiteral(_ value: String) -> String {
+        driver.escapeStringLiteral(value)
+    }
+
+    func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? {
+        try await driver.fetchApproximateRowCount(table: table, schema: nil)
+    }
+
+    func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] {
+        try await driver.fetchColumns(table: table, schema: nil)
+    }
+
+    func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? {
+        try await driver.openSessionScope(for: purpose)
+    }
+
+    func closeSessionScope(_ scope: PluginSessionScope) async throws {
+        try await driver.closeSessionScope(scope)
+    }
+}