fix: confirm more destructive SQL and fix five driver bugs (#3907)

apple/Packages/TableProCore/Package.swift

@@ -278,6 +278,11 @@ let package = Package(
             dependencies: ["TableProSpannerCore"],
             path: "Tests/TableProSpannerCoreTests"
         ),
+        .testTarget(
+            name: "TableProTeradataCoreTests",
+            dependencies: ["TableProTeradataCore"],
+            path: "Tests/TableProTeradataCoreTests"
+        ),
         .testTarget(
             name: "TableProTrinoCoreTests",
             dependencies: ["TableProTrinoCore"],

apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift

@@ -15,14 +15,16 @@ public struct BigtableColumn: Hashable, Sendable, Comparable {
         family + ":" + BigtableBytes.display(qualifier)
     }
 
+    // By scalar: a qualifier that starts with a combining mark joins the colon into one Character.
     public init(name: String) throws {
-        guard let colon = name.firstIndex(of: ":"), colon != name.startIndex else {
+        guard let colon = name.unicodeScalars.firstIndex(of: ":"), colon != name.startIndex else {
             throw BigtableError.configuration(String(
                 format: String(localized: "\"%@\" is not a column. Name it as family:qualifier."), name
             ))
         }
-        self.family = String(name[..<colon])
-        self.qualifier = try BigtableBytes.parse(String(name[name.index(after: colon)...]))
+        self.family = String(name.unicodeScalars[..<colon])
+        let scalars = name.unicodeScalars
+        self.qualifier = try BigtableBytes.parse(String(scalars[scalars.index(after: colon)...]))
     }
 
     public static func < (lhs: BigtableColumn, rhs: BigtableColumn) -> Bool {

apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift

@@ -157,7 +157,7 @@ public struct BigtableRequest: Sendable, Equatable {
 
     /// Text that would not read back as one bare word goes in JSON quotes.
     static func token(_ text: String, reserved: String) -> String {
-        let plain = !text.isEmpty && !text.hasPrefix("\"") && text.unicodeScalars.allSatisfy { scalar in
+        let plain = !text.isEmpty && text.unicodeScalars.first != "\"" && text.unicodeScalars.allSatisfy { scalar in
             !CharacterSet.whitespacesAndNewlines.contains(scalar) && !reserved.unicodeScalars.contains(scalar)
         }
         if plain { return text }
@@ -173,38 +173,39 @@ public struct BigtableRequest: Sendable, Equatable {
     static let missingTable = BigtableError.configuration(String(localized: "Name the table before the key, as in GET events/user#42."))
 }
 
-/// Reads a request line one token at a time.
+// Read by scalar, as `token` writes: a key that starts with a combining mark joins the quote or slash before it
+// into one Character.
 private struct RequestScanner {
-    private var text: Substring
+    private var text: Substring.UnicodeScalarView
     private var query: [String: String] = [:]
 
     init(_ text: String) {
-        self.text = Substring(text)
+        self.text = Substring(text).unicodeScalars
     }
 
     var isAtEnd: Bool {
-        text.allSatisfy(\.isWhitespace)
+        text.allSatisfy(\.properties.isWhitespace)
     }
 
-    mutating func word(stoppingAt stop: Character? = nil) -> String? {
+    mutating func word(stoppingAt stop: Unicode.Scalar? = nil) -> String? {
         skipSpace()
-        let word = text.prefix { !$0.isWhitespace && $0 != stop }
+        let word = text.prefix { !$0.properties.isWhitespace && $0 != stop }
         guard !word.isEmpty else { return nil }
         text = text.dropFirst(word.count)
         return String(word)
     }
 
     mutating func rest() -> String {
         let remaining = String(text).trimmingCharacters(in: .whitespacesAndNewlines)
-        text = ""
+        text.removeAll()
         return remaining
     }
 
     /// `table/key`, with an optional `?name=value` query after it. With `tableOnly`, a target with
     /// no slash is a table alone, as in `SCAN events`.
     mutating func target(tableOnly: Bool) throws -> (table: String, key: Data?) {
         skipSpace()
-        let head = text.prefix { !$0.isWhitespace && $0 != "\"" && $0 != "?" }
+        let head = text.prefix { !$0.properties.isWhitespace && $0 != "\"" && $0 != "?" }
         let table: String
         if let slash = head.firstIndex(of: "/") {
             table = String(text[..<slash])
@@ -221,12 +222,12 @@ private struct RequestScanner {
         if text.first == "\"" {
             keyText = try quoted()
         } else {
-            let raw = text.prefix { !$0.isWhitespace && $0 != "?" }
+            let raw = text.prefix { !$0.properties.isWhitespace && $0 != "?" }
             text = text.dropFirst(raw.count)
             keyText = raw.isEmpty ? nil : String(raw)
         }
         if text.first == "?" {
-            let raw = text.dropFirst().prefix { !$0.isWhitespace }
+            let raw = text.dropFirst().prefix { !$0.properties.isWhitespace }
             text = text.dropFirst(raw.count + 1)
             for pair in raw.split(separator: "&") {
                 let parts = pair.split(separator: "=", maxSplits: 1).map(String.init)
@@ -259,9 +260,9 @@ private struct RequestScanner {
         var index = text.index(after: text.startIndex)
         var escaped = false
         while index < text.endIndex {
-            let character = text[index]
-            if character == "\"" && !escaped { break }
-            escaped = character == "\\" && !escaped
+            let scalar = text[index]
+            if scalar == "\"" && !escaped { break }
+            escaped = scalar == "\\" && !escaped
             index = text.index(after: index)
         }
         guard index < text.endIndex else {
@@ -277,6 +278,6 @@ private struct RequestScanner {
     }
 
     private mutating func skipSpace() {
-        text = text.drop { $0.isWhitespace }
+        text = text.drop { $0.properties.isWhitespace }
     }
 }

apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift

@@ -9,7 +9,8 @@ public enum R2SQLResponseDecoder {
             throw R2SQLError.malformedResponse(status: response.statusCode, detail: snippet(response.body))
         }
 
-        guard envelope.success else {
+        // A refused redirect arrives as its 3xx, whatever the body says.
+        guard envelope.success, (200 ..< 300).contains(response.statusCode) else {
             switch response.statusCode {
             case 401, 403:
                 throw R2SQLError.authentication(status: response.statusCode, errors: envelope.errors)

apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift

@@ -75,6 +75,17 @@ private final class TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Se
         transport?.register(task)
     }
 
+    // A redirect would carry the API token to wherever it points.
+    func urlSession(
+        _ session: URLSession,
+        task: URLSessionTask,
+        willPerformHTTPRedirection response: HTTPURLResponse,
+        newRequest request: URLRequest,
+        completionHandler: @escaping (URLRequest?) -> Void
+    ) {
+        completionHandler(nil)
+    }
+
     func finish() {
         guard let task = lock.withLock({ task }) else { return }
         transport?.unregister(task)

apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift

@@ -58,11 +58,15 @@ public enum MongoStatementClassifier {
     // Server-side JavaScript, plus the two stages that write to a collection the statement names.
     private static let codeExecutionKeys: Set<String> = ["$where", "$function", "$accumulator", "$out", "$merge"]
 
+    // The server accepts its own operators from any client, and rewrites `mapReduce` into two of them that run a
+    // function: `$_internalJsEmit` and `$_internalJsReduce`. Nothing a person writes names one.
+    private static let internalOperatorPrefix = "$_internal"
+
     private static let codeExecutionCalls: Set<String> = ["mapreduce", "eval"]
 
     // For text the grammar rejects, where a false alarm only stops an external client sending code nobody could read.
     private static let codeExecutionMarkers: [String] = [
-        "$where", "$function", "$accumulator", "mapreduce", ".eval(", "$out", "$merge"
+        "$where", "$function", "$accumulator", "$_internal", "mapreduce", ".eval(", "$out", "$merge"
     ]
 
     private static func statementVerdict(
@@ -78,6 +82,7 @@ public enum MongoStatementClassifier {
 
     private static func verdict(of statement: MongoStatement) -> MongoStatementVerdict {
         let reachesCode = !statement.objectKeys.isDisjoint(with: codeExecutionKeys)
+            || statement.objectKeys.contains { $0.hasPrefix(internalOperatorPrefix) }
             || statement.calls.contains { call in
                 codeExecutionCalls.contains(call.name.lowercased())
                     || call.commandName.map { codeExecutionCalls.contains($0.lowercased()) } == true

apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnfilteredWrite.swift

@@ -16,6 +16,8 @@ public enum SQLUnfilteredWrite: Sendable, Equatable {
 
     private static let period = UInt16(UnicodeScalar(".").value)
 
+    private static let whereLength = "WHERE".utf16.count
+
     private static let servedStatements: Set<String> = ["SELECT", "INSERT", "UPDATE", "DELETE", "MERGE"]
 
     // `:where` and `$where` are bind parameters on SQLite.
@@ -30,58 +32,65 @@ public enum SQLUnfilteredWrite: Sendable, Equatable {
         }
         let depth = tokens[start].depth
         var found: [SQLUnfilteredWrite] = []
-        let verb: Int?
+        let verbs: [Int]
         switch first {
         case "UPDATE", "DELETE":
-            verb = start
+            verbs = [start]
         case "WITH":
-            // A CTE may be named `delete` or `merge`, and on PostgreSQL its body may itself write. A list that does
-            // not parse is searched whole, which can only find a write that is not there.
-            let list = commonTableExpressions(in: tokens, with: start)
-            for body in list?.bodies ?? [] {
+            // A CTE may be named `delete` or `merge`, and on PostgreSQL its body may itself write. Every body is read
+            // whether or not the list around it can be followed. When it cannot, every word at the statement's depth
+            // is a candidate, which can only find a write that is not there.
+            for body in commonTableExpressionBodies(in: tokens, depth: depth) {
                 found += writes(inStatement: source.substring(with: body) as NSString, grammar: grammar)
             }
-            verb = tokens.indices.dropFirst(list?.main ?? start + 1).first { index in
-                tokens[index].depth == depth && servedStatements.contains(tokens[index].word ?? "")
-            }
+            verbs = mainStatement(in: tokens, with: start).map { [$0] }
+                ?? tokens.indices.dropFirst(start + 1).filter { tokens[$0].depth == depth }
         case "ALTER":
             // ClickHouse writes rows with `ALTER TABLE t UPDATE ... WHERE` and `ALTER TABLE t DELETE WHERE`.
             guard tokens.indices.contains(start + 1), tokens[start + 1].word == "TABLE" else { return [] }
-            verb = tokens.indices.dropFirst(start + 2).first { index in
+            let verb = tokens.indices.dropFirst(start + 2).first { index in
                 let word = tokens[index].word
                 return tokens[index].depth == depth && (word == "UPDATE" || word == "DELETE")
                     && tokens[index - 1].word != "ON"
             }
+            verbs = verb.map { [$0] } ?? []
         default:
-            verb = nil
+            verbs = []
         }
-        if let verb, let write = write(at: verb, in: tokens), !hasOwnWhere(tokens, depth: depth, in: source) {
-            found.append(write)
+        let writes = verbs.compactMap { write(at: $0, in: tokens) }
+        if !writes.isEmpty, !hasOwnWhere(tokens, depth: depth, in: source) {
+            found += writes
         }
         return found
     }
 
-    private static func commonTableExpressions(
-        in tokens: [TSQLTokens.Token],
-        with start: Int
-    ) -> (bodies: [NSRange], main: Int)? {
-        let depth = tokens[start].depth
+    private static let bodyOpeners: Set<String> = ["AS", "MATERIALIZED"]
+
+    // Found by scanning rather than by walking the list, so a body is read when a name is several tokens, as
+    // `U&"gone"` is, or a SEARCH or CYCLE clause follows an earlier body.
+    private static func commonTableExpressionBodies(in tokens: [TSQLTokens.Token], depth: Int) -> [NSRange] {
+        tokens.indices.dropFirst().compactMap { index in
+            guard tokens[index].depth == depth, bodyOpeners.contains(tokens[index - 1].word ?? ""),
+                  let close = close(of: index, in: tokens)
+            else { return nil }
+            let inside = tokens[index].location + 1
+            return NSRange(location: inside, length: tokens[close].location - inside)
+        }
+    }
+
+    // Nil when no statement begins right after the last body: a name T-SQL reads as several words where the engine
+    // reads one, as at a euro sign, or PostgreSQL's `SEARCH ... SET name` and `CYCLE ... USING name` after a body.
+    private static func mainStatement(in tokens: [TSQLTokens.Token], with start: Int) -> Int? {
         var index = start + 1
-        var bodies: [NSRange] = []
         func word(_ expected: String) -> Bool {
             guard tokens.indices.contains(index), tokens[index].word == expected else { return false }
             index += 1
             return true
         }
-        func parentheses() -> NSRange? {
-            guard tokens.indices.contains(index), tokens[index].kind == .open,
-                  let close = tokens.indices.dropFirst(index + 1).first(where: {
-                      tokens[$0].kind == .close && tokens[$0].depth == depth
-                  })
-            else { return nil }
-            let inside = tokens[index].location + 1
+        func parentheses() -> Bool {
+            guard let close = close(of: index, in: tokens) else { return false }
             index = close + 1
-            return NSRange(location: inside, length: tokens[close].location - inside)
+            return true
         }
         _ = word("RECURSIVE")
         while tokens.indices.contains(index) {
@@ -90,12 +99,19 @@ public enum SQLUnfilteredWrite: Sendable, Equatable {
             guard word("AS") else { return nil }
             _ = word("NOT")
             _ = word("MATERIALIZED")
-            guard let body = parentheses() else { return nil }
-            bodies.append(body)
+            guard parentheses() else { return nil }
             guard tokens.indices.contains(index), tokens[index].kind == .comma else { break }
             index += 1
         }
-        return tokens.indices.contains(index) ? (bodies, index) : nil
+        guard tokens.indices.contains(index), servedStatements.contains(tokens[index].word ?? "") else { return nil }
+        return index
+    }
+
+    private static func close(of open: Int, in tokens: [TSQLTokens.Token]) -> Int? {
+        guard tokens.indices.contains(open), tokens[open].kind == .open else { return nil }
+        return tokens.indices.dropFirst(open + 1).first {
+            tokens[$0].kind == .close && tokens[$0].depth == tokens[open].depth
+        }
     }
 
     private static func write(at index: Int, in tokens: [TSQLTokens.Token]) -> SQLUnfilteredWrite? {
@@ -115,13 +131,23 @@ public enum SQLUnfilteredWrite: Sendable, Equatable {
 
     private static func hasOwnWhere(_ tokens: [TSQLTokens.Token], depth: Int, in source: NSString) -> Bool {
         tokens.indices.contains { index in
-            guard tokens[index].depth == depth, tokens[index].word == "WHERE" else { return false }
-            guard index > 0 else { return true }
-            let location = tokens[index].location
-            if location > 0, parameterPrefixes.contains(source.character(at: location - 1)) { return false }
-            // `t.where` is a column.
-            let previous = tokens[index - 1]
-            return previous.kind != .other || source.character(at: previous.location) != period
+            tokens[index].depth == depth && tokens[index].word == "WHERE" && !isName(tokens, at: index, in: source)
         }
     }
+
+    // The tokens end a word where T-SQL does. SQLite, PostgreSQL and MySQL read every character outside ASCII as part
+    // of a name, a no-break space included, and MySQL lets a digit begin one. Taking a real filter for a name only
+    // asks about a statement that had one.
+    private static func isName(_ tokens: [TSQLTokens.Token], at index: Int, in source: NSString) -> Bool {
+        guard index > 0 else { return false }
+        let location = tokens[index].location
+        let before = source.character(at: location - 1)
+        if parameterPrefixes.contains(before) || SqlDollarQuote.isIdentifierContinuation(before) { return true }
+        let end = location + whereLength
+        if end < source.length, source.character(at: end) >= 0x80 { return true }
+        // `t.where` is a column, and so is PostgreSQL's `RETURNING id AS where`.
+        let previous = tokens[index - 1]
+        if previous.word == "AS" { return true }
+        return previous.kind == .other && source.character(at: previous.location) == period
+    }
 }

apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift

@@ -73,6 +73,9 @@ public enum SQLUnterminatedStatements {
             return tokens[index].depth == 0 || (next?.word != "HINT" && next?.word != "PLAN")
         case "END":
             return next?.word == "CONVERSATION"
+        case "BULK":
+            // `ADMINISTER BULK OPERATIONS` is a permission, and neither `BULK INSERT` nor `OPENROWSET(BULK ...)` goes on so.
+            return next?.word != "OPERATIONS"
         default:
             return true
         }