diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index 2ee21b30d2..89cd97b446 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -415,6 +415,20 @@ "category": "database-driver", "homepage": "https://docs.tablepro.app/databases/typesense" }, + "weaviate": { + "target": "WeaviateDriverPlugin", + "bundleName": "WeaviateDriverPlugin", + "bundleId": "com.TablePro.WeaviateDriverPlugin", + "bundled": false, + "displayName": "Weaviate Driver", + "summary": "Weaviate driver over the REST API with a GraphQL console", + "databaseTypeIds": [ + "Weaviate" + ], + "icon": "weaviate-icon", + "category": "database-driver", + "homepage": "https://docs.tablepro.app/databases/weaviate" + }, "xlsx": { "target": "XLSXExport", "bundleName": "XLSXExport", diff --git a/CHANGELOG.md b/CHANGELOG.md index c1e1e9b898..de7de9ec4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Google Cloud Spanner as a registry plugin over the REST API. (#1226, #2480) +- Weaviate as a registry REST plugin, collections as tables. (#1724) - TiDB and Databend connection types on the MySQL driver. (#1066, #2514) - Empty state in the inspector and the assistant for a connection that is not up. - **Check connections** in Settings > General, including Only when I use the connection. (#2700) @@ -20,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Remove Invisible Characters** in the Query menu. (#2717) - **Show invisible characters** in Settings > Editor. (#2717) - Warnings in the SQL editor for full-width punctuation, curly quotes and non-ASCII spaces. (#2717) +- Highlight rules that color data grid rows or cells by value. (#2723) +- **Encoding** option for MySQL and MariaDB connections, with **UTF-8 via Latin 1** for databases written through a Latin 1 client. (#2725) ### Changed @@ -32,9 +35,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Open Project Folder… in File > Import. - First-launch tour replaced by a one-page welcome sheet, shown again from Help > Getting Started. - Beancount connections held at Safe Mode Read-Only. (#2030) +- MySQL sessions on the server's default `utf8mb4` collation. ### Fixed +- Wrong SQLSTATE code in PostgreSQL, Redshift, CockroachDB and PGlite error messages. +- Read-only write explanation never shown on PostgreSQL servers. - Safe Mode minimum from a configuration profile missing from the toolbar, the Database menu and the connection form. (#2030) - Stop not ending queries on MySQL and MariaDB servers without TLS. - Users & Roles failing, Stop not ending queries and sequences listed as tables on TiDB servers opened as MySQL. @@ -42,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Blank welcome window list when a search matched nothing and a favorite existed. - Welcome window reading No Connections while a tag filter hid every connection. - Favorited connection inside a group listed twice on the welcome window. +- Missing red wash on a row deleted together with a new, unsaved row. - Welcome window tag filter stuck on a tag no connection carries any more, hiding every connection. - Collapsing every group on the welcome window undone at the next launch. - Linked Folders and Team Library connections ignoring the welcome window search, with no context menu. @@ -121,6 +128,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Line and paragraph separators (U+2028, U+2029) shown as line breaks the database does not see. (#2717) - Stop on Cloudflare D1, libSQL and Trino cancelling a sidebar read instead of the running query. - Numeric-looking filter values sent unquoted to text columns when a table first opens or after a foreign key jump. +- Garbled non-Latin text saved to MySQL and MariaDB servers that force a Latin 1 session. (#2725) +- Garbled non-Latin text when restoring a MySQL or MariaDB SQL export through a Latin 1 client. (#2725) +- Curly quotes, € and other Windows-1252 symbols shown as invisible characters after `SET NAMES latin1`. +- `Illegal mix of collations` comparing a column with a user variable on MySQL 8. +- GEOMETRY values from a parameterized MySQL query shown as raw bytes. +- Earlier row's text repeated in later rows of a parameterized MySQL query once a value passed 64 KB. ### Security diff --git a/CLAUDE.md b/CLAUDE.md index 171a618c95..4dab1c7726 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -225,6 +225,8 @@ To ship one: add the record type or field in CloudKit Console (or `xcrun cktool **A pooled metadata read assumes a second connection reaches the same database, and an embedded engine breaks that assumption**: `MetadataConnectionPool` builds a whole new driver, so it is only correct when the database lives on a server the driver reconnects to. When the database lives *inside* the driver instance, the pool gets a different database: a second `duckdb_open(":memory:")` is a fresh empty database, and a second `duckdb_open` on the same *file* is a second independent read-write instance that the first never sees (DuckDB's file lock does not conflict within one process). The failure is silent, because an empty catalog is indistinguishable from "no tables", which is why #2108 survived a manual refresh. `supportsConnectionPooling` is the opt-out, and it is read only by `DatabaseManager.canPool`; DuckDB and PGlite set it `false`. SQLite-family engines keep pooling, because multi-connection access to one file is what they are built for. Two rules follow. First, every metadata read goes through `DatabaseManager.withMetadataDriver` so `metadataRoute` can apply the rule; reaching for `MetadataConnectionPool.shared.withDriver` directly bypasses it, which is how routines kept pooling after the sidebar stopped. Second, a capability with no `DriverPlugin` static is curated per type and `buildMetadataSnapshot` must carry it over from the built-in snapshot, or `register(snapshot:forTypeId:)` resets it to the struct default the moment the plugin loads. That is not hypothetical: it silently disabled MongoDB's `authenticationIsDatabaseScoped` (#1970) for every build that had the plugin installed. `registerVariant` already treats the curated entry as authoritative, which is the only reason PGlite's flag ever worked. +**A MySQL session's character set is server state, and the driver sets it rather than asking for it**: `MYSQL_SET_CHARSET_NAME` only puts a collation byte in the handshake, and the server is free to ignore it. `init_connect='SET NAMES latin1'` for any user without `CONNECTION_ADMIN`, MariaDB's `skip-character-set-client-handshake`, and a server with no `utf8mb4` all leave the session latin1 while libmariadb still reports `utf8mb4`. The driver sends UTF-8 either way, so every comment and value it wrote there was stored double-encoded, which is the `メール` of #2725, and a user-variable comparison on MySQL 8 failed with `ERROR 1267` under the handshake's `utf8mb4_general_ci`. So `MariaDBPluginConnection.establishSessionCharacterSet` calls `mysql_set_character_set` on every connect path, falls back to a plain `SET NAMES utf8` (libmariadb itself rewrites `utf8` to `utf8mb3`, which a pre-5.5.3 server does not know), and keeps the server's own session when it refuses both rather than failing the connect: a MySQL-protocol engine that rejects `SET NAMES` must still connect. Two rules follow. Outgoing SQL stays the UTF-8 bytes of the Swift string and never follows a mid-session `SET NAMES`: a legacy dump that declares `SET NAMES latin1` over UTF-8 bytes restores byte-for-byte only because nothing re-encodes it, which is also what the `mysql` client does. And every result cell decodes through `MySQLColumnDecoding` by its field's own `charsetnr`, never as "UTF-8 and hope": a latin1 field reads UTF-8 first and then MySQL's latin1, which is cp1252 plus five C1 pass-through bytes and not Foundation's `.isoLatin1` or `.windowsCP1252`. The Foundation encodings in `MySQLCharacterSet` are a hand transcription of the server's tables; `scripts/check-mysql-charset-decoding.sh` diffs them against a live server, and a charset that disagrees stays out of the table. **UTF-8 via Latin 1** (`mysqlConnectionEncoding`) sets only `character_set_client` to latin1 and repairs the text client-side, so correctly stored text is never read back as `?`, which a latin1 results charset would do and a structure edit would then write back. + **A MongoDB update or delete is anchored on `_id` or it does not run**: `generateDelete` used to fall back to a filter built from the remaining columns, which silently dropped every value it could not stringify (all binary) and then `deleteOne`d the first partial match, so a document with a binary `_id` could delete a different document. Both paths now skip with a logged warning instead, matching what `generateUpdate` already did. **Redis Cluster routing follows the server's own answer, and the curated table is only a fallback**: `RedisCommandRouting` fetches `COMMAND` once at connect, which supplies key positions on every Redis and, from Redis 7, the `request_policy` / `response_policy` tips that say which commands fan out and how their replies combine. A policy lives on the *subcommand* entry, not the container (`COMMAND INFO config` carries no tips at all; `config|set` is what says `all_nodes`), so the table is keyed `container|sub`. Redis 6 reports no tips, so a parsed reply is merged *over* the curated table rather than replacing it, or `DBSIZE`, `KEYS` and `FLUSHDB` would each go to one shard of a cluster and report success. The curated table is a hand-written list that has to agree with Redis and that nothing at runtime checks, so `scripts/check-redis-command-routing.sh [host] [port]` diffs it against a live Redis 7+; it found 32 disagreements the first time it ran, including a container command hashed on its literal subcommand name and `MSETNX` marked splittable when splitting it breaks the guarantee it exists for. Two rules follow. A container command takes no key of its own, so `OBJECT`, `MEMORY` and `CONFIG` declare no key positions and their keyed subcommands are listed separately, at the index the key sits in the *full* argument list (`OBJECT ENCODING k` puts it at 2, not 1). And an unknown command is routed as keyless rather than by hashing `argv[1]`: a keyless container like `SCRIPT LOAD` answers `+OK` from one node and never sends a `MOVED` to correct the guess. diff --git a/Packages/TableProCore/Package.swift b/Packages/TableProCore/Package.swift index 12982aeb23..c853d1efb3 100644 --- a/Packages/TableProCore/Package.swift +++ b/Packages/TableProCore/Package.swift @@ -23,6 +23,7 @@ let package = Package( .library(name: "TableProTrinoCore", targets: ["TableProTrinoCore"]), .library(name: "TableProGoogleCloud", targets: ["TableProGoogleCloud"]), .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]), + .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), .library(name: "TableProR2SQLCore", targets: ["TableProR2SQLCore"]) ], @@ -102,6 +103,11 @@ let package = Package( dependencies: ["TableProGoogleCloud"], path: "Sources/TableProSpannerCore" ), + .target( + name: "TableProWeaviateCore", + dependencies: [], + path: "Sources/TableProWeaviateCore" + ), .target( name: "TableProR2SQLCore", dependencies: [], @@ -162,6 +168,11 @@ let package = Package( dependencies: ["TableProSpannerCore", "TableProGoogleCloud"], path: "Tests/TableProSpannerCoreTests" ), + .testTarget( + name: "TableProWeaviateCoreTests", + dependencies: ["TableProWeaviateCore"], + path: "Tests/TableProWeaviateCoreTests" + ), .testTarget( name: "TableProR2SQLCoreTests", dependencies: ["TableProR2SQLCore"], diff --git a/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift b/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift index 211eda5ad6..75a4803a4b 100644 --- a/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift +++ b/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift @@ -40,12 +40,13 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let trino = DatabaseType(rawValue: "Trino") public static let kafka = DatabaseType(rawValue: "Kafka") public static let cloudflareR2SQL = DatabaseType(rawValue: "Cloudflare R2 SQL") + public static let weaviate = DatabaseType(rawValue: "Weaviate") public static let allKnownTypes: [DatabaseType] = [ .mysql, .mariadb, .tidb, .databend, .postgresql, .sqlite, .redis, .mongodb, .clickhouse, .mssql, .oracle, .dameng, .duckdb, .cassandra, .redshift, .etcd, .cloudflareD1, .dynamodb, .bigquery, .spanner, .snowflake, .libsql, .beancount, - .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL + .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate ] /// Icon name for this database type — asset catalog name (e.g. "mysql-icon") or SF Symbol fallback @@ -79,6 +80,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .trino: return "trino-icon" case .kafka: return "kafka-icon" case .cloudflareR2SQL: return "cloudflare-r2-sql-icon" + case .weaviate: return "weaviate-icon" default: return "externaldrive" } } diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateAuth.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateAuth.swift new file mode 100644 index 0000000000..bf2a74a611 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateAuth.swift @@ -0,0 +1,93 @@ +import Foundation + +public enum WeaviateFieldID { + public static let authMethod = "wvAuthMethod" + public static let apiKey = "wvApiKey" + public static let skipTLSVerify = "wvSkipTLSVerify" +} + +public enum WeaviateAuthMethod: String, Sendable, Equatable { + case none + case apiKey +} + +public struct WeaviateAuth: Sendable, Equatable { + public let method: WeaviateAuthMethod + public let apiKey: String + + public init(method: WeaviateAuthMethod, apiKey: String = "") { + self.method = method + self.apiKey = apiKey.trimmingCharacters(in: .whitespacesAndNewlines) + } + + public static func parse(fields: [String: String]) -> WeaviateAuth { + let raw = fields[WeaviateFieldID.authMethod] ?? WeaviateAuthMethod.none.rawValue + let method = WeaviateAuthMethod(rawValue: raw) ?? .none + return WeaviateAuth(method: method, apiKey: fields[WeaviateFieldID.apiKey] ?? "") + } + + /// A key left in the form after the user switches back to None must not be sent: the form + /// keeps the field's text, and only the method says whether the connection is authenticated. + public var authorizationHeader: String? { + guard method == .apiKey, !apiKey.isEmpty else { return nil } + return "Bearer \(apiKey)" + } +} + +public struct WeaviateConnectionSettings: Sendable, Equatable { + public static let defaultPort = 8_080 + + public let host: String + public let port: Int + public let usesTLS: Bool + public let auth: WeaviateAuth + public let skipTLSVerify: Bool + + public init( + host: String, + port: Int, + usesTLS: Bool, + auth: WeaviateAuth, + skipTLSVerify: Bool + ) { + self.host = host.trimmingCharacters(in: .whitespacesAndNewlines) + self.port = port + self.usesTLS = usesTLS + self.auth = auth + self.skipTLSVerify = skipTLSVerify + } + + public static func parse( + host: String, + port: Int, + usesTLS: Bool, + fields: [String: String] + ) throws -> WeaviateConnectionSettings { + let resolvedHost = host.trimmingCharacters(in: .whitespacesAndNewlines) + let resolvedPort = port > 0 ? port : defaultPort + let skipTLS = fields[WeaviateFieldID.skipTLSVerify] == "true" + let settings = WeaviateConnectionSettings( + host: resolvedHost.isEmpty ? "localhost" : resolvedHost, + port: resolvedPort, + usesTLS: usesTLS, + auth: WeaviateAuth.parse(fields: fields), + skipTLSVerify: skipTLS + ) + _ = try settings.baseURL() + if settings.auth.method == .apiKey, settings.auth.apiKey.isEmpty { + throw WeaviateError.configuration(String(localized: "Enter a Weaviate API key.")) + } + return settings + } + + public func baseURL() throws -> URL { + var components = URLComponents() + components.scheme = usesTLS ? "https" : "http" + components.host = host + components.port = port + guard let url = components.url else { + throw WeaviateError.configuration(String(localized: "The host is not valid in a URL.")) + } + return url + } +} diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateError.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateError.swift new file mode 100644 index 0000000000..854bfa4951 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateError.swift @@ -0,0 +1,57 @@ +import Foundation + +public enum WeaviateError: Error, LocalizedError, Equatable, Sendable { + case configuration(String) + case notConnected + case transport(String) + case authentication(String) + case api(status: Int, message: String) + case malformedResponse(String) + case cancelled + + public var errorDescription: String? { + switch self { + case .configuration(let detail), .transport(let detail), .malformedResponse(let detail): + return detail + case .notConnected: + return String(localized: "Not connected to Weaviate.") + case .authentication(let detail): + return detail + case .api(_, let message): + return message + case .cancelled: + return String(localized: "The request was cancelled.") + } + } + + public static func from(status: Int, body: Data) -> WeaviateError { + let message = apiMessage(from: body) + ?? String(format: String(localized: "Weaviate returned HTTP %d."), status) + if status == 401 || status == 403 { + return .authentication(message) + } + return .api(status: status, message: message) + } + + public static func apiMessage(from body: Data) -> String? { + guard let json = try? JSONSerialization.jsonObject(with: body) else { + let text = String(data: body, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) + return (text?.isEmpty ?? true) ? nil : text + } + if let object = json as? [String: Any] { + if let errors = object["error"] as? [[String: Any]] { + let messages = errors.compactMap { $0["message"] as? String }.filter { !$0.isEmpty } + if !messages.isEmpty { + return messages.joined(separator: "\n") + } + } + if let error = object["error"] as? String, !error.isEmpty { + return error + } + if let message = object["message"] as? String, !message.isEmpty { + return message + } + } + return nil + } +} diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateFilterBuilder.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateFilterBuilder.swift new file mode 100644 index 0000000000..d9235f1578 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateFilterBuilder.swift @@ -0,0 +1,309 @@ +import Foundation + +public enum WeaviateFilterError: Error, LocalizedError, Equatable { + case unsupportedOperator(String) + case missingUpperBound(column: String) + case notANumber(column: String, value: String) + case notABoolean(column: String, value: String) + case notADate(column: String, value: String) + case emptyList(column: String) + case comparisonNeedsNumberOrDate(column: String, op: String) + case textMatchNeedsText(column: String, op: String) + case vectorNotFilterable(column: String) + + public var errorDescription: String? { + switch self { + case .unsupportedOperator(let op): + return String(format: String(localized: "Weaviate cannot filter with %@."), op) + case .missingUpperBound(let column): + return String(format: String(localized: "BETWEEN on %@ needs an upper bound."), column) + case .notANumber(let column, let value): + return String( + format: String(localized: "%@ is a numeric property, and %@ is not a number."), + column, value + ) + case .notABoolean(let column, let value): + return String( + format: String(localized: "%@ is a boolean property, so it matches only true or false, not %@."), + column, value + ) + case .notADate(let column, let value): + return String( + format: String(localized: "%@ is a date property, and %@ is not an RFC 3339 timestamp such as 2024-01-31T00:00:00Z."), + column, value + ) + case .emptyList(let column): + return String(format: String(localized: "IN on %@ needs at least one value."), column) + case .comparisonNeedsNumberOrDate(let column, let op): + return String( + format: String(localized: "Weaviate compares only numbers and dates with %@, and %@ is neither."), + op, column + ) + case .textMatchNeedsText(let column, let op): + return String( + format: String(localized: "Weaviate matches text with %@, and %@ is not a text property."), + op, column + ) + case .vectorNotFilterable(let column): + return String(format: String(localized: "Weaviate cannot filter on %@."), column) + } + } +} + +/// Weaviate has no array value field: an `int[]` property filters with `valueInt`, so the kind is +/// always the element's. +public enum WeaviateValueKind: String, Sendable, Equatable { + case text + case uuid + case int + case number + case boolean + case date + + public var graphQLField: String { + switch self { + case .text, .uuid: return "valueText" + case .int: return "valueInt" + case .number: return "valueNumber" + case .boolean: return "valueBoolean" + case .date: return "valueDate" + } + } + + public var isOrdered: Bool { + self == .int || self == .number || self == .date + } + + /// `Like` compiles to a regex over the inverted index, which Weaviate only builds for text. + public var acceptsPatternMatch: Bool { + self == .text + } + + public static func forDataType(_ dataType: String) -> WeaviateValueKind { + var name = dataType.trimmingCharacters(in: .whitespaces).lowercased() + while name.hasSuffix("[]") { + name = String(name.dropLast(2)) + } + switch name { + case "int": return .int + case "number": return .number + case "boolean", "bool": return .boolean + case "date": return .date + case "uuid": return .uuid + default: return .text + } + } +} + +public enum WeaviateFilterBuilder { + public static func graphQLWhere( + filters: [WeaviateFilterSpec], + logicMode: String, + types: [String: String] + ) throws -> String? { + guard !filters.isEmpty else { return nil } + let operands = try filters.map { try operand(for: $0, types: types) } + guard operands.count > 1 else { return operands[0] } + let op = logicMode.uppercased() == "OR" ? "Or" : "And" + return "{ operator: \(op) operands: [\(operands.joined(separator: " "))] }" + } + + public static func operand(for filter: WeaviateFilterSpec, types: [String: String]) throws -> String { + let column = filter.column + guard column != WeaviateSchema.vectorColumn else { + throw WeaviateFilterError.vectorNotFilterable(column: column) + } + let path = column == WeaviateSchema.uuidColumn ? WeaviateSchema.uuidGraphQLPath : column + let kind = column == WeaviateSchema.uuidColumn + ? WeaviateValueKind.text + : WeaviateValueKind.forDataType(types[column] ?? "text") + let op = filter.op.uppercased() + + switch op { + case "=": + return try comparison("Equal", path: path, column: column, value: filter.value, kind: kind) + case "!=", "<>": + return try comparison("NotEqual", path: path, column: column, value: filter.value, kind: kind) + case ">", ">=", "<", "<=": + guard kind.isOrdered else { + throw WeaviateFilterError.comparisonNeedsNumberOrDate(column: column, op: op) + } + return try comparison(orderedOperator(op), path: path, column: column, value: filter.value, kind: kind) + case "CONTAINS": + return try like(pattern: "*\(filter.value)*", path: path, column: column, kind: kind, op: op) + case "NOT CONTAINS": + return negated(try like(pattern: "*\(filter.value)*", path: path, column: column, kind: kind, op: op)) + case "STARTS WITH": + return try like(pattern: "\(filter.value)*", path: path, column: column, kind: kind, op: op) + case "ENDS WITH": + return try like(pattern: "*\(filter.value)", path: path, column: column, kind: kind, op: op) + case "IN": + return try containsList("ContainsAny", filter.value, path: path, column: column, kind: kind) + case "NOT IN": + return try containsList("ContainsNone", filter.value, path: path, column: column, kind: kind) + case "BETWEEN": + return try between(filter, path: path, column: column, kind: kind) + case "IS NULL": + return "{ path: [\"\(escape(path))\"] operator: IsNull valueBoolean: true }" + case "IS NOT NULL": + return "{ path: [\"\(escape(path))\"] operator: IsNull valueBoolean: false }" + case "IS EMPTY": + return try emptiness("Equal", path: path, column: column, kind: kind, op: op) + case "IS NOT EMPTY": + return try emptiness("GreaterThan", path: path, column: column, kind: kind, op: op) + default: + throw WeaviateFilterError.unsupportedOperator(op) + } + } + + private static func orderedOperator(_ op: String) -> String { + switch op { + case ">": return "GreaterThan" + case ">=": return "GreaterThanEqual" + case "<": return "LessThan" + default: return "LessThanEqual" + } + } + + private static func comparison( + _ operatorName: String, + path: String, + column: String, + value: String, + kind: WeaviateValueKind + ) throws -> String { + let literal = try literal(value, column: column, kind: kind) + return "{ path: [\"\(escape(path))\"] operator: \(operatorName) \(kind.graphQLField): \(literal) }" + } + + /// Weaviate compiles a `Like` value into a regex over the inverted index. An int, number or + /// date property crashes that compile, and a uuid property is refused outright. + private static func like( + pattern: String, + path: String, + column: String, + kind: WeaviateValueKind, + op: String + ) throws -> String { + guard kind.acceptsPatternMatch else { + throw WeaviateFilterError.textMatchNeedsText(column: column, op: op) + } + return "{ path: [\"\(escape(path))\"] operator: Like valueText: \"\(escape(pattern))\" }" + } + + private static func containsList( + _ operatorName: String, + _ value: String, + path: String, + column: String, + kind: WeaviateValueKind + ) throws -> String { + let parts = value + .split(separator: ",") + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty } + guard !parts.isEmpty else { + throw WeaviateFilterError.emptyList(column: column) + } + let literals = try parts.map { try literal($0, column: column, kind: kind) } + return "{ path: [\"\(escape(path))\"] operator: \(operatorName) \(kind.graphQLField): [\(literals.joined(separator: ", "))] }" + } + + /// Weaviate has no "is empty": it counts with `len(prop)`, which needs `indexPropertyLength` on + /// the collection and answers with what to turn on when it is off. There is no `len(id)`, and a + /// uuid property has no length either. + private static func emptiness( + _ operatorName: String, + path: String, + column: String, + kind: WeaviateValueKind, + op: String + ) throws -> String { + guard kind == .text, column != WeaviateSchema.uuidColumn else { + throw WeaviateFilterError.textMatchNeedsText(column: column, op: op) + } + return "{ path: [\"len(\(escape(path)))\"] operator: \(operatorName) valueInt: 0 }" + } + + private static func between( + _ filter: WeaviateFilterSpec, + path: String, + column: String, + kind: WeaviateValueKind + ) throws -> String { + guard kind.isOrdered else { + throw WeaviateFilterError.comparisonNeedsNumberOrDate(column: column, op: "BETWEEN") + } + guard let upperBound = filter.secondValue, !upperBound.trimmingCharacters(in: .whitespaces).isEmpty else { + throw WeaviateFilterError.missingUpperBound(column: column) + } + let lower = try comparison( + "GreaterThanEqual", path: path, column: column, value: filter.value, kind: kind + ) + let upper = try comparison( + "LessThanEqual", path: path, column: column, value: upperBound, kind: kind + ) + return "{ operator: And operands: [\(lower) \(upper)] }" + } + + private static func negated(_ operand: String) -> String { + "{ operator: Not operands: [\(operand)] }" + } + + private static func literal(_ value: String, column: String, kind: WeaviateValueKind) throws -> String { + let trimmed = value.trimmingCharacters(in: .whitespaces) + switch kind { + case .text, .uuid: + return "\"\(escape(value))\"" + case .int: + guard let number = Int(trimmed) else { + throw WeaviateFilterError.notANumber(column: column, value: value) + } + return String(number) + case .number: + guard let number = Double(trimmed) else { + throw WeaviateFilterError.notANumber(column: column, value: value) + } + return String(number) + case .boolean: + switch trimmed.lowercased() { + case "true", "1": return "true" + case "false", "0": return "false" + default: throw WeaviateFilterError.notABoolean(column: column, value: value) + } + case .date: + guard WeaviateDateLiteral.isRFC3339(trimmed) else { + throw WeaviateFilterError.notADate(column: column, value: value) + } + return "\"\(escape(trimmed))\"" + } + } + + static func escape(_ value: String) -> String { + var result = "" + result.reserveCapacity(value.count) + for character in value { + switch character { + case "\\": result += "\\\\" + case "\"": result += "\\\"" + case "\n": result += "\\n" + case "\r": result += "\\r" + case "\t": result += "\\t" + default: result.append(character) + } + } + return result + } +} + +/// Weaviate parses a `valueDate` with Go's RFC 3339 layout and answers +/// `trying parse time as RFC3339 string` for anything else, including a bare `2024-01-31`. +public enum WeaviateDateLiteral { + public static func isRFC3339(_ value: String) -> Bool { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime] + if formatter.date(from: value) != nil { return true } + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter.date(from: value) != nil + } +} diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift new file mode 100644 index 0000000000..0705167481 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift @@ -0,0 +1,140 @@ +import Foundation + +public enum WeaviateGraphQL { + public static func getQuery( + collection: String, + properties: [String], + limit: Int, + offset: Int, + sorts: [WeaviateSortSpec], + filters: [WeaviateFilterSpec], + logicMode: String, + schema: [String: WeaviateProperty], + includeVector: Bool = true + ) throws -> String { + let types = schema.mapValues(\.dataType) + let fields = properties + .filter { $0 != WeaviateSchema.uuidColumn && $0 != WeaviateSchema.vectorColumn } + .compactMap { selection(for: $0, schema: schema) } + .joined(separator: " ") + var args: [String] = ["limit: \(max(limit, 0))", "offset: \(max(offset, 0))"] + if let whereClause = try WeaviateFilterBuilder.graphQLWhere( + filters: filters, logicMode: logicMode, types: types + ) { + args.append("where: \(whereClause)") + } + let sortArgs = sorts.compactMap { sort -> String? in + guard let path = sortPath(for: sort) else { return nil } + let order = sort.ascending ? "asc" : "desc" + return "{ path: [\"\(WeaviateFilterBuilder.escape(path))\"] order: \(order) }" + } + if !sortArgs.isEmpty { + args.append("sort: [\(sortArgs.joined(separator: " "))]") + } + let argumentList = args.joined(separator: ", ") + let additional = includeVector ? "id vector" : "id" + return """ + { Get { \(collection)(\(argumentList)) { \(fields) _additional { \(additional) } } } } + """ + } + + /// A structured property needs its own sub-selection, and an `object` with no declared nested + /// properties has nothing to select, so it is left out rather than failing the query. + private static func selection(for name: String, schema: [String: WeaviateProperty]) -> String? { + guard let property = schema[name] else { return name } + switch WeaviatePropertyShape.of(property) { + case .scalar: + return name + case .geoCoordinates: + return "\(name) { latitude longitude }" + case .phoneNumber: + return "\(name) { input internationalFormatted nationalFormatted countryCode national valid defaultCountry }" + case .object: + let nested = property.nestedProperties + .compactMap { selection(for: $0.name, schema: [$0.name: $0]) } + .joined(separator: " ") + return nested.isEmpty ? nil : "\(name) { \(nested) }" + case .crossReference(let targets): + let fragments = targets.map { "... on \($0) { _additional { id } }" }.joined(separator: " ") + return "\(name) { \(fragments) }" + } + } + + /// `vector` is a grid column rather than a property, so Weaviate has nothing to sort on. Every + /// real property is passed through: a type it cannot sort, such as uuid, is reported by the + /// server, which beats painting a sort chevron over rows in insertion order. + private static func sortPath(for sort: WeaviateSortSpec) -> String? { + if sort.column == WeaviateSchema.uuidColumn { + return WeaviateSchema.uuidGraphQLPath + } + return sort.column == WeaviateSchema.vectorColumn ? nil : sort.column + } + + public static func looksLikeGraphQL(_ text: String) -> Bool { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.hasPrefix("{") { return true } + let lowered = trimmed.lowercased() + return lowered.hasPrefix("query") || lowered.hasPrefix("mutation") || lowered.hasPrefix("subscription") + || lowered.hasPrefix("fragment") + } + + public static func requestBody(query: String) throws -> Data { + let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + if let data = trimmed.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + object["query"] != nil { + return try WeaviateJSON.data(object) + } + return try WeaviateJSON.data(["query": trimmed]) + } +} + +public struct WeaviateConsoleRequest: Sendable, Equatable { + public let method: String + public let path: String + public let body: String? + + public init(method: String, path: String, body: String?) { + self.method = method + self.path = path + self.body = body + } +} + +public enum WeaviateConsoleParser { + public static func parse(_ input: String) -> WeaviateConsoleRequest? { + let trimmed = input.trimmingCharacters(in: .whitespacesAndNewlines) + guard let newline = trimmed.firstIndex(where: \.isNewline) else { + return parseHeader(trimmed, body: nil) + } + let header = String(trimmed[.. WeaviateConsoleRequest? { + let parts = header.split(maxSplits: 2, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) + guard parts.count >= 2 else { return nil } + let method = String(parts[0]).uppercased() + guard ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"].contains(method) else { + return nil + } + let rawPath = String(parts[1]) + guard rawPath.hasPrefix("/") else { return nil } + var path = rawPath + if path != "/", path != "/v1", !path.hasPrefix("/v1/"), !path.hasPrefix("/v1?") { + path = "/v1" + path + } + let inlineBody = parts.count > 2 + ? String(parts[2]).trimmingCharacters(in: .whitespacesAndNewlines) + : "" + return WeaviateConsoleRequest( + method: method, + path: path, + body: body ?? (inlineBody.isEmpty ? nil : inlineBody) + ) + } +} diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateJSON.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateJSON.swift new file mode 100644 index 0000000000..6ae697efdb --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateJSON.swift @@ -0,0 +1,73 @@ +import Foundation + +public enum WeaviateJSON { + public static func dictionary(_ value: Any?) -> [String: Any]? { + value as? [String: Any] + } + + public static func data(_ object: Any, pretty: Bool = false) throws -> Data { + guard JSONSerialization.isValidJSONObject(object) else { + throw WeaviateError.malformedResponse(String(localized: "Request body is not valid JSON.")) + } + var options: JSONSerialization.WritingOptions = [.sortedKeys] + if pretty { + options.insert(.prettyPrinted) + } + return try JSONSerialization.data(withJSONObject: object, options: options) + } + + public static func text(_ object: Any, pretty: Bool = false) throws -> String { + let encoded = try data(object, pretty: pretty) + return String(data: encoded, encoding: .utf8) ?? "{}" + } + + public static func displayText(_ value: Any?) -> String? { + switch value { + case nil, is NSNull: + return nil + case let text as String: + return text + case let number as NSNumber: + if CFGetTypeID(number) == CFBooleanGetTypeID() { + return number.boolValue ? "true" : "false" + } + return number.stringValue + case let object as [String: Any]: + return (try? text(object)) ?? nil + case let object as [Any]: + return (try? text(object)) ?? nil + default: + return String(describing: value as Any) + } + } + + /// Only a property the grid renders as JSON is parsed back as JSON. Running the parser over + /// every type sends a `text` cell holding `{"a":1}` as an object, which Weaviate rejects while + /// the grid reports the save. + public static func parsedValue(_ text: String, typeName: String) -> Any { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + let declared = typeName.trimmingCharacters(in: .whitespaces) + let isArray = declared.hasSuffix("[]") + if !isArray { + switch WeaviateValueKind.forDataType(declared) { + case .boolean: + if trimmed.lowercased() == "true" { return true } + if trimmed.lowercased() == "false" { return false } + return text + case .int: + return Int(trimmed) ?? text + case .number: + return Double(trimmed) ?? text + case .text, .uuid, .date: + break + } + } + let shape = WeaviatePropertyShape.of(WeaviateProperty(name: "", dataType: declared)) + guard isArray || shape != .scalar else { return text } + guard let data = trimmed.data(using: .utf8), + let parsed = try? JSONSerialization.jsonObject(with: data), + parsed is [Any] || parsed is [String: Any] + else { return text } + return parsed + } +} diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateObjectCodec.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateObjectCodec.swift new file mode 100644 index 0000000000..d202dfba72 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateObjectCodec.swift @@ -0,0 +1,141 @@ +import Foundation + +public struct WeaviateObject: Sendable, Equatable { + public let uuid: String + public let className: String + public let properties: [String: String?] + public let vector: [Double]? + + public init(uuid: String, className: String, properties: [String: String?], vector: [Double]?) { + self.uuid = uuid + self.className = className + self.properties = properties + self.vector = vector + } + + public static func parse(_ json: [String: Any]) -> WeaviateObject? { + let uuid = (json["id"] as? String) ?? "" + let className = (json["class"] as? String) ?? "" + let raw = json["properties"] as? [String: Any] ?? [:] + var properties: [String: String?] = [:] + for (key, value) in raw { + properties[key] = WeaviateJSON.displayText(value) + } + let vector = vector(from: json["vector"]) + if uuid.isEmpty, properties.isEmpty, vector == nil { + return nil + } + return WeaviateObject(uuid: uuid, className: className, properties: properties, vector: vector) + } + + public static func parseList(_ json: Any) -> [WeaviateObject] { + let objects: [[String: Any]] + if let object = json as? [String: Any] { + objects = object["objects"] as? [[String: Any]] ?? [] + } else if let array = json as? [[String: Any]] { + objects = array + } else { + return [] + } + return objects.compactMap(parse) + } + + public var vectorText: String? { + guard let vector, !vector.isEmpty else { + return vector == nil ? nil : "[]" + } + return "[" + vector.map(Self.numberText).joined(separator: ",") + "]" + } + + private static func numberText(_ value: Double) -> String { + if value.rounded() == value, let whole = Int(exactly: value) { + return String(whole) + } + return String(value) + } + + static func vector(from value: Any?) -> [Double]? { + if let numbers = value as? [Double] { + return numbers + } + if let numbers = value as? [NSNumber] { + return numbers.map(\.doubleValue) + } + return nil + } +} + +public enum WeaviateObjectCodec { + static let additionalKey = "_additional" + + public static func row(for object: WeaviateObject, columns: [String]) -> [String?] { + return columns.map { column in + switch column { + case WeaviateSchema.uuidColumn: + return object.uuid.isEmpty ? nil : object.uuid + case WeaviateSchema.vectorColumn: + return object.vectorText + default: + if let value = object.properties[column] { + return value + } + return nil + } + } + } + + public static func objects(fromGraphQL json: Any) -> [WeaviateObject] { + guard let root = json as? [String: Any] else { return [] } + if let errors = root["errors"] as? [[String: Any]], !errors.isEmpty { + return [] + } + guard let data = root["data"] as? [String: Any] else { return [] } + if let get = data["Get"] as? [String: Any] { + return objects(fromGet: get) + } + return [] + } + + public static func graphQLErrors(from json: Any) -> [String] { + guard let root = json as? [String: Any], + let errors = root["errors"] as? [[String: Any]] + else { return [] } + return errors.compactMap { $0["message"] as? String }.filter { !$0.isEmpty } + } + + private static func objects(fromGet get: [String: Any]) -> [WeaviateObject] { + var result: [WeaviateObject] = [] + for (className, value) in get { + guard let rows = value as? [[String: Any]] else { continue } + for row in rows { + result.append(object(fromGetRow: row, className: className)) + } + } + return result + } + + /// `_additional` is where a vector search puts `distance`, `score` and `certainty`, which are + /// the whole point of the query the user wrote. Only `id` and `vector` have a column of their + /// own; the rest become properties so they reach the grid. + private static func object(fromGetRow row: [String: Any], className: String) -> WeaviateObject { + var properties: [String: String?] = [:] + var additional: [String: Any] = [:] + var uuid = "" + var vector: [Double]? + for (key, value) in row { + if key == additionalKey, let fields = value as? [String: Any] { + additional = fields + continue + } + properties[key] = WeaviateJSON.displayText(value) + } + uuid = (additional["id"] as? String) ?? uuid + vector = WeaviateObject.vector(from: additional["vector"]) + for (key, value) in additional where key != "id" && key != WeaviateSchema.vectorColumn { + let name = properties[key] == nil ? key : "\(additionalKey).\(key)" + properties[name] = WeaviateJSON.displayText(value) + } + return WeaviateObject(uuid: uuid, className: className, properties: properties, vector: vector) + } +} + diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviatePathEncoding.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviatePathEncoding.swift new file mode 100644 index 0000000000..0d6ffcd3c4 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviatePathEncoding.swift @@ -0,0 +1,28 @@ +import Foundation + +public enum WeaviatePathEncoding { + private static let allowed: CharacterSet = { + var set = CharacterSet.alphanumerics + set.insert(charactersIn: "-_") + return set + }() + + public static func segment(_ value: String) -> String { + value.addingPercentEncoding(withAllowedCharacters: allowed) ?? value + } + + /// A console line carries its own query string (`GET /v1/objects?class=Article`), so the path + /// is parsed rather than assigned whole: `URLComponents.path` percent-encodes `?` into `%3F` + /// and sends the request to a path that does not exist. + public static func resolve(_ path: String, query: [String: String] = [:], against base: URL) -> URL? { + guard path.hasPrefix("/"), !path.contains("://") else { return nil } + guard var components = URLComponents(url: base, resolvingAgainstBaseURL: false), + let requested = URLComponents(string: path) + else { return nil } + components.percentEncodedPath = requested.percentEncodedPath + var items = requested.queryItems ?? [] + items += query.keys.sorted().map { URLQueryItem(name: $0, value: query[$0]) } + components.queryItems = items.isEmpty ? nil : items + return components.url + } +} diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateQuery.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateQuery.swift new file mode 100644 index 0000000000..773b8cf34d --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateQuery.swift @@ -0,0 +1,360 @@ +import Foundation + +public struct WeaviateFilterSpec: Codable, Sendable, Equatable { + public let column: String + public let op: String + public let value: String + + /// The upper bound of `BETWEEN`, carried apart from `value` so a value holding a comma cannot + /// be mistaken for the separator between the two bounds. + public let secondValue: String? + + public init(column: String, op: String, value: String, secondValue: String? = nil) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + } +} + +public struct WeaviateSortSpec: Codable, Sendable, Equatable { + public let column: String + public let ascending: Bool + + public init(column: String, ascending: Bool) { + self.column = column + self.ascending = ascending + } +} + +public struct WeaviateParsedSearch: Sendable, Equatable { + public let collection: String + public let offset: Int + public let limit: Int + public let sorts: [WeaviateSortSpec] + public let filters: [WeaviateFilterSpec] + public let logicMode: String + public let propertyNames: [String] + + public init( + collection: String, + offset: Int, + limit: Int, + sorts: [WeaviateSortSpec], + filters: [WeaviateFilterSpec], + logicMode: String, + propertyNames: [String] + ) { + self.collection = collection + self.offset = offset + self.limit = limit + self.sorts = sorts + self.filters = filters + self.logicMode = logicMode + self.propertyNames = propertyNames + } + + /// `GET /v1/objects` sorts, but only on a property: the grid's `vector` column is not one, and + /// Weaviate answers `no such prop with name 'vector'`. Everything else goes through GraphQL, + /// whose `sort` argument takes the object id as well. + public var sortableSorts: [WeaviateSortSpec] { + sorts.filter { $0.column != WeaviateSchema.vectorColumn } + } + + public var usesGraphQL: Bool { + !filters.isEmpty || !sortableSorts.isEmpty + } +} + +public enum WeaviateBrowseQuery { + public static let searchTag = "WEAVIATE_SEARCH:" + + public static func encode( + collection: String, + offset: Int, + limit: Int, + sorts: [WeaviateSortSpec], + filters: [WeaviateFilterSpec], + logicMode: String, + propertyNames: [String] + ) -> String { + let payload: [String: Any] = [ + "collection": collection, + "offset": offset, + "limit": limit, + "logicMode": logicMode, + "sorts": sorts.map { ["column": $0.column, "ascending": $0.ascending] }, + "filters": filters.map { filter -> [String: Any] in + var encoded: [String: Any] = ["column": filter.column, "op": filter.op, "value": filter.value] + if let secondValue = filter.secondValue { + encoded["secondValue"] = secondValue + } + return encoded + }, + "properties": propertyNames + ] + let body = (try? WeaviateJSON.data(payload)) ?? Data() + return searchTag + body.base64EncodedString() + } + + public static func parse(_ query: String) -> WeaviateParsedSearch? { + guard query.hasPrefix(searchTag) else { return nil } + let encoded = String(query.dropFirst(searchTag.count)) + guard let data = Data(base64Encoded: encoded), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let collection = json["collection"] as? String + else { return nil } + let sortsJSON = json["sorts"] as? [[String: Any]] ?? [] + let filtersJSON = json["filters"] as? [[String: Any]] ?? [] + let properties = json["properties"] as? [String] ?? [] + return WeaviateParsedSearch( + collection: collection, + offset: json["offset"] as? Int ?? 0, + limit: json["limit"] as? Int ?? 25, + sorts: sortsJSON.compactMap { item in + guard let column = item["column"] as? String else { return nil } + return WeaviateSortSpec(column: column, ascending: item["ascending"] as? Bool ?? true) + }, + filters: filtersJSON.compactMap { item in + guard let column = item["column"] as? String, let op = item["op"] as? String else { + return nil + } + return WeaviateFilterSpec( + column: column, + op: op, + value: item["value"] as? String ?? "", + secondValue: item["secondValue"] as? String + ) + }, + logicMode: json["logicMode"] as? String ?? "AND", + propertyNames: properties + ) + } + + public static func isTagged(_ query: String) -> Bool { + query.hasPrefix(searchTag) + } +} + +public struct WeaviateWriteRequest: Sendable, Equatable { + public let method: String + public let path: String + public let query: [String: String] + public let body: String? + + public init(method: String, path: String, query: [String: String] = [:], body: String?) { + self.method = method + self.path = path + self.query = query + self.body = body + } +} + +public enum WeaviateWriteCodec { + public static let writeTag = "WEAVIATE_WRITE:" + + public static func encode(_ request: WeaviateWriteRequest) -> String { + let payload: [String: Any] = [ + "method": request.method, + "path": request.path, + "query": request.query, + "body": request.body ?? "" + ] + let data = (try? WeaviateJSON.data(payload)) ?? Data() + return writeTag + data.base64EncodedString() + } + + public static func decode(_ statement: String) -> WeaviateWriteRequest? { + guard statement.hasPrefix(writeTag) else { return nil } + let encoded = String(statement.dropFirst(writeTag.count)) + guard let data = Data(base64Encoded: encoded), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let method = json["method"] as? String, + let path = json["path"] as? String + else { return nil } + let query = json["query"] as? [String: String] ?? [:] + let body = json["body"] as? String + return WeaviateWriteRequest( + method: method, + path: path, + query: query, + body: (body?.isEmpty ?? true) ? nil : body + ) + } + + public static func isTagged(_ statement: String) -> Bool { + statement.hasPrefix(writeTag) + } +} + +public struct WeaviateCellChange: Sendable, Equatable { + public let column: String + public let newText: String? + + public init(column: String, newText: String?) { + self.column = column + self.newText = newText + } +} + +public struct WeaviateTrackedChange: Sendable, Equatable { + public enum Kind: String, Sendable, Equatable { + case insert + case update + case delete + } + + public let kind: Kind + public let uuid: String? + public let values: [String: String?] + public let cellChanges: [WeaviateCellChange] + + public init( + kind: Kind, + uuid: String?, + values: [String: String?], + cellChanges: [WeaviateCellChange] + ) { + self.kind = kind + self.uuid = uuid + self.values = values + self.cellChanges = cellChanges + } +} + +public enum WeaviateSkipReason: String, Sendable, Equatable { + case missingUUID + case noEditableColumns + case payloadNotEncodable +} + +public struct WeaviateSkippedChange: Sendable, Equatable { + public let kind: WeaviateTrackedChange.Kind + public let reason: WeaviateSkipReason + + public init(kind: WeaviateTrackedChange.Kind, reason: WeaviateSkipReason) { + self.kind = kind + self.reason = reason + } +} + +/// A skipped change writes nothing while the grid reports the save succeeded, so the driver has to +/// be able to say what it dropped. Same reason the MongoDB generator logs its own skips. +public struct WeaviateWriteBatch: Sendable, Equatable { + public let requests: [WeaviateWriteRequest] + public let skipped: [WeaviateSkippedChange] + + public init(requests: [WeaviateWriteRequest], skipped: [WeaviateSkippedChange]) { + self.requests = requests + self.skipped = skipped + } +} + +public enum WeaviateStatementGenerator { + public static func generate( + collection: String, + columns: [String], + typeNames: [String], + changes: [WeaviateTrackedChange] + ) -> WeaviateWriteBatch { + let types = Dictionary(zip(columns, typeNames), uniquingKeysWith: { first, _ in first }) + var requests: [WeaviateWriteRequest] = [] + var skipped: [WeaviateSkippedChange] = [] + for change in changes { + let request: WeaviateWriteRequest? + switch change.kind { + case .insert: + request = insert(collection: collection, types: types, change: change) + case .update: + request = update(collection: collection, types: types, change: change) + case .delete: + request = delete(collection: collection, change: change) + } + if let request { + requests.append(request) + } else { + skipped.append(WeaviateSkippedChange(kind: change.kind, reason: reason(for: change))) + } + } + return WeaviateWriteBatch(requests: requests, skipped: skipped) + } + + private static func reason(for change: WeaviateTrackedChange) -> WeaviateSkipReason { + if change.kind != .insert, change.uuid?.isEmpty ?? true { + return .missingUUID + } + if change.kind == .update, editablePatch(from: change).isEmpty { + return .noEditableColumns + } + return .payloadNotEncodable + } + + private static func editablePatch(from change: WeaviateTrackedChange) -> [String: String?] { + var patch: [String: String?] = [:] + for cell in change.cellChanges where !WeaviateSchema.immutableColumns.contains(cell.column) { + patch[cell.column] = cell.newText + } + return patch + } + + private static func insert( + collection: String, + types: [String: String], + change: WeaviateTrackedChange + ) -> WeaviateWriteRequest? { + var payload: [String: Any] = [ + "class": collection, + "properties": properties(from: change.values, types: types) + ] + if let uuid = change.uuid, !uuid.isEmpty { + payload["id"] = uuid + } + guard let body = try? WeaviateJSON.text(payload) else { return nil } + return WeaviateWriteRequest(method: "POST", path: "/v1/objects", body: body) + } + + private static func update( + collection: String, + types: [String: String], + change: WeaviateTrackedChange + ) -> WeaviateWriteRequest? { + guard let uuid = change.uuid, !uuid.isEmpty else { return nil } + let patch = editablePatch(from: change) + guard !patch.isEmpty else { return nil } + let payload: [String: Any] = [ + "class": collection, + "properties": properties(from: patch, types: types) + ] + guard let body = try? WeaviateJSON.text(payload) else { return nil } + return WeaviateWriteRequest( + method: "PATCH", + path: "/v1/objects/\(WeaviatePathEncoding.segment(uuid))", + query: ["class": collection], + body: body + ) + } + + private static func delete(collection: String, change: WeaviateTrackedChange) -> WeaviateWriteRequest? { + guard let uuid = change.uuid, !uuid.isEmpty else { return nil } + return WeaviateWriteRequest( + method: "DELETE", + path: "/v1/objects/\(WeaviatePathEncoding.segment(uuid))", + query: ["class": collection], + body: nil + ) + } + + private static func properties(from values: [String: String?], types: [String: String]) -> [String: Any] { + var result: [String: Any] = [:] + for (column, text) in values { + if WeaviateSchema.immutableColumns.contains(column) { continue } + if column.hasPrefix("\(WeaviateObjectCodec.additionalKey).") { continue } + if let text { + result[column] = WeaviateJSON.parsedValue(text, typeName: types[column] ?? "text") + } else { + result[column] = NSNull() + } + } + return result + } +} diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateSchema.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateSchema.swift new file mode 100644 index 0000000000..b1478e6ec8 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateSchema.swift @@ -0,0 +1,119 @@ +import Foundation + +public struct WeaviateProperty: Sendable, Equatable { + public let name: String + public let dataTypes: [String] + public let nestedProperties: [WeaviateProperty] + + public var dataType: String { dataTypes.first ?? "text" } + + public init(name: String, dataType: String) { + self.init(name: name, dataTypes: [dataType], nestedProperties: []) + } + + public init(name: String, dataTypes: [String], nestedProperties: [WeaviateProperty]) { + self.name = name + self.dataTypes = dataTypes.isEmpty ? ["text"] : dataTypes + self.nestedProperties = nestedProperties + } + + public static func parse(_ json: [String: Any]) -> WeaviateProperty? { + guard let name = json["name"] as? String, !name.isEmpty else { return nil } + let nested = json["nestedProperties"] as? [[String: Any]] ?? [] + return WeaviateProperty( + name: name, + dataTypes: json["dataType"] as? [String] ?? [], + nestedProperties: nested.compactMap(WeaviateProperty.parse) + ) + } +} + +/// A GraphQL field with a structured type is refused without a sub-selection: asking for a +/// `geoCoordinates` property by name answers `Field "place" ... must have a sub selection` and the +/// whole query fails, taking the filtered browse with it. +public enum WeaviatePropertyShape: Sendable, Equatable { + case scalar + case geoCoordinates + case phoneNumber + case object + case crossReference([String]) + + private static let scalarTypes: Set = [ + "text", "string", "int", "number", "boolean", "bool", "date", "uuid", "blob" + ] + + public static func of(_ property: WeaviateProperty) -> WeaviatePropertyShape { + let names = property.dataTypes.map { element($0) } + guard let first = names.first else { return .scalar } + if scalarTypes.contains(first.lowercased()) { return .scalar } + switch first { + case "geoCoordinates": return .geoCoordinates + case "phoneNumber": return .phoneNumber + case "object": return .object + default: + return first.first?.isUppercase == true ? .crossReference(names) : .scalar + } + } + + private static func element(_ dataType: String) -> String { + var name = dataType.trimmingCharacters(in: .whitespaces) + while name.hasSuffix("[]") { + name = String(name.dropLast(2)) + } + return name + } +} + +public struct WeaviateCollection: Sendable, Equatable { + public let name: String + public let properties: [WeaviateProperty] + public let vectorizer: String? + + public init(name: String, properties: [WeaviateProperty], vectorizer: String? = nil) { + self.name = name + self.properties = properties + self.vectorizer = vectorizer + } + + public static func parse(_ json: [String: Any]) -> WeaviateCollection? { + guard let name = json["class"] as? String, !name.isEmpty else { return nil } + let rawProperties = json["properties"] as? [[String: Any]] ?? [] + let properties = rawProperties.compactMap(WeaviateProperty.parse) + return WeaviateCollection( + name: name, + properties: properties, + vectorizer: json["vectorizer"] as? String + ) + } +} + +public enum WeaviateSchema { + public static let uuidColumn = "uuid" + public static let vectorColumn = "vector" + public static let immutableColumns: [String] = [uuidColumn, vectorColumn] + + /// Weaviate names the object id `id` inside a `where` or `sort` argument, and normalizes it to + /// `_id` in its own errors. The grid calls the same column `uuid`. + public static let uuidGraphQLPath = "id" + + public static func collections(from json: Any) -> [WeaviateCollection] { + let classes: [[String: Any]] + if let object = json as? [String: Any] { + classes = object["classes"] as? [[String: Any]] ?? [] + } else if let array = json as? [[String: Any]] { + classes = array + } else { + return [] + } + return classes.compactMap(WeaviateCollection.parse) + } + + public static func columns(for collection: WeaviateCollection) -> [(name: String, type: String, isPrimaryKey: Bool)] { + var result: [(name: String, type: String, isPrimaryKey: Bool)] = [ + (uuidColumn, "uuid", true) + ] + result += collection.properties.map { ($0.name, $0.dataType, false) } + result.append((vectorColumn, "vector", false)) + return result + } +} diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateClient.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateClient.swift new file mode 100644 index 0000000000..cb56e8146b --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateClient.swift @@ -0,0 +1,140 @@ +import Foundation + +public final class WeaviateClient: @unchecked Sendable { + public let settings: WeaviateConnectionSettings + private let transport: WeaviateTransport + private let timeout: @Sendable () -> TimeInterval + private let lock = NSLock() + private var _version: String? + + public init( + settings: WeaviateConnectionSettings, + transport: WeaviateTransport, + timeout: @escaping @Sendable () -> TimeInterval + ) { + self.settings = settings + self.transport = transport + self.timeout = timeout + } + + public var serverVersion: String? { + lock.withLock { _version } + } + + public func cancelAll() { + transport.cancelAll() + } + + public func connect() async throws { + let ready = try await send(method: "GET", path: "/v1/.well-known/ready") + try throwIfFailed(ready) + let meta = try await send(method: "GET", path: "/v1/meta") + try throwIfFailed(meta) + if let json = WeaviateJSON.dictionary(meta.json), let version = json["version"] as? String { + lock.withLock { _version = version } + } + } + + /// `/v1/.well-known/ready` is the readiness probe and answers 200 with no key at all, so a + /// revoked key would leave the health monitor reporting a session every query then fails on. + public func ping() async throws { + let response = try await send(method: "GET", path: "/v1/meta") + try throwIfFailed(response) + } + + public func schema() async throws -> [WeaviateCollection] { + let response = try await send(method: "GET", path: "/v1/schema") + try throwIfFailed(response) + guard let json = response.json else { + throw WeaviateError.malformedResponse(String(localized: "Schema response was empty.")) + } + return WeaviateSchema.collections(from: json) + } + + public func objects( + collection: String, + limit: Int, + offset: Int, + includeVector: Bool = true + ) async throws -> [WeaviateObject] { + var query = [ + "class": collection, + "limit": String(max(limit, 0)), + "offset": String(max(offset, 0)) + ] + if includeVector { + query["include"] = "vector" + } + let response = try await send(method: "GET", path: "/v1/objects", query: query) + try throwIfFailed(response) + guard let json = response.json else { return [] } + return WeaviateObject.parseList(json) + } + + public func graphql(_ query: String) async throws -> WeaviateHTTPResponse { + let body = try WeaviateGraphQL.requestBody(query: query) + let response = try await send(method: "POST", path: "/v1/graphql", body: body) + try throwIfFailed(response) + if let json = response.json { + let errors = WeaviateObjectCodec.graphQLErrors(from: json) + if !errors.isEmpty { + throw WeaviateError.api(status: response.statusCode, message: errors.joined(separator: "\n")) + } + } + return response + } + + public func execute(write request: WeaviateWriteRequest) async throws -> WeaviateHTTPResponse { + let body = request.body.flatMap { $0.data(using: .utf8) } + let response = try await send( + method: request.method, + path: request.path, + query: request.query, + body: body + ) + try throwIfFailed(response) + return response + } + + public func execute(console request: WeaviateConsoleRequest) async throws -> WeaviateHTTPResponse { + let body = request.body.flatMap { $0.data(using: .utf8) } + let response = try await send(method: request.method, path: request.path, body: body) + try throwIfFailed(response) + return response + } + + public func send( + method: String, + path: String, + query: [String: String] = [:], + body: Data? = nil + ) async throws -> WeaviateHTTPResponse { + let base = try settings.baseURL() + guard let url = WeaviatePathEncoding.resolve(path, query: query, against: base) else { + throw WeaviateError.configuration(String(format: String(localized: "Invalid path: %@"), path)) + } + var headers = [ + "Accept": "application/json" + ] + if body != nil { + headers["Content-Type"] = "application/json" + } + if let authorization = settings.auth.authorizationHeader { + headers["Authorization"] = authorization + } + let request = WeaviateHTTPRequest( + method: method, + url: url, + headers: headers, + body: body, + timeoutInterval: timeout() + ) + return try await transport.send(request) + } + + public func throwIfFailed(_ response: WeaviateHTTPResponse) throws { + guard (200..<300).contains(response.statusCode) else { + throw WeaviateError.from(status: response.statusCode, body: response.body) + } + } +} diff --git a/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift b/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift new file mode 100644 index 0000000000..f7695710e2 --- /dev/null +++ b/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift @@ -0,0 +1,174 @@ +import Foundation + +public struct WeaviateHTTPRequest: Sendable, Equatable { + public let method: String + public let url: URL + public let headers: [String: String] + public let body: Data? + public let timeoutInterval: TimeInterval + + public init( + method: String, + url: URL, + headers: [String: String], + body: Data?, + timeoutInterval: TimeInterval + ) { + self.method = method + self.url = url + self.headers = headers + self.body = body + self.timeoutInterval = timeoutInterval + } +} + +public struct WeaviateHTTPResponse: Sendable, Equatable { + public let statusCode: Int + public let body: Data + private let parsed: ParsedResponseJSON + + public init(statusCode: Int, body: Data) { + self.statusCode = statusCode + self.body = body + self.parsed = ParsedResponseJSON(body) + } + + /// A filtered browse reads this two or three times, and a page of 1536-dimension vectors is + /// several megabytes, so the body is parsed once and the result held. + public var json: Any? { + parsed.value + } + + public var text: String { + String(data: body, encoding: .utf8) ?? "" + } + + public static func == (lhs: WeaviateHTTPResponse, rhs: WeaviateHTTPResponse) -> Bool { + lhs.statusCode == rhs.statusCode && lhs.body == rhs.body + } +} + +private final class ParsedResponseJSON: @unchecked Sendable { + private let body: Data + private let lock = NSLock() + private var value_: Any? + private var hasParsed = false + + init(_ body: Data) { + self.body = body + } + + var value: Any? { + lock.withLock { + if !hasParsed { + value_ = try? JSONSerialization.jsonObject(with: body, options: [.fragmentsAllowed]) + hasParsed = true + } + return value_ + } + } +} + +public protocol WeaviateTransport: Sendable { + func send(_ request: WeaviateHTTPRequest) async throws -> WeaviateHTTPResponse + func cancelAll() +} + +public final class URLSessionWeaviateTransport: WeaviateTransport, @unchecked Sendable { + private let session: URLSession + private let lock = NSLock() + private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] + + public init( + configuration: URLSessionConfiguration = .ephemeral, + resourceTimeout: TimeInterval, + skipTLSVerify: Bool = false + ) { + configuration.timeoutIntervalForResource = resourceTimeout + if skipTLSVerify { + let delegate = InsecureTLSDelegate() + session = URLSession(configuration: configuration, delegate: delegate, delegateQueue: nil) + } else { + session = URLSession(configuration: configuration) + } + } + + deinit { + session.invalidateAndCancel() + } + + public func cancelAll() { + let tasks = lock.withLock { Array(inFlight.values) } + tasks.forEach { $0.cancel() } + } + + public func send(_ request: WeaviateHTTPRequest) async throws -> WeaviateHTTPResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = request.timeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + + let tracker = TaskTracker(transport: self) + defer { tracker.finish() } + do { + let (data, response) = try await session.data(for: urlRequest, delegate: tracker) + guard let httpResponse = response as? HTTPURLResponse else { + throw WeaviateError.transport(String(localized: "Weaviate answered with something other than HTTP.")) + } + return WeaviateHTTPResponse(statusCode: httpResponse.statusCode, body: data) + } catch let error as URLError where error.code == .cancelled { + throw WeaviateError.cancelled + } catch let error as WeaviateError { + throw error + } catch let error as URLError { + throw WeaviateError.transport(error.localizedDescription) + } + } + + fileprivate func register(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = task } + } + + fileprivate func unregister(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = nil } + } +} + +private final class InsecureTLSDelegate: NSObject, URLSessionDelegate { + func urlSession( + _ session: URLSession, + didReceive challenge: URLAuthenticationChallenge, + completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void + ) { + guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust, + let trust = challenge.protectionSpace.serverTrust + else { + completionHandler(.performDefaultHandling, nil) + return + } + completionHandler(.useCredential, URLCredential(trust: trust)) + } +} + +private final class TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { + private weak var transport: URLSessionWeaviateTransport? + private let lock = NSLock() + private var task: URLSessionTask? + + init(transport: URLSessionWeaviateTransport) { + self.transport = transport + } + + func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { + lock.withLock { self.task = task } + transport?.register(task) + } + + func finish() { + guard let task = lock.withLock({ task }) else { return } + transport?.unregister(task) + } +} diff --git a/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift b/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift index f0d7b2e80f..afa059d747 100644 --- a/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift +++ b/Packages/TableProCore/Tests/TableProModelsTests/DatabaseTypeTests.swift @@ -59,7 +59,7 @@ struct DatabaseTypeTests { @Test("allKnownTypes contains all expected types") func allKnownTypesComplete() { - #expect(DatabaseType.allKnownTypes.count == 28) + #expect(DatabaseType.allKnownTypes.count == 29) #expect(DatabaseType.allKnownTypes.contains(.mysql)) #expect(DatabaseType.allKnownTypes.contains(.tidb)) #expect(DatabaseType.allKnownTypes.contains(.databend)) @@ -74,6 +74,7 @@ struct DatabaseTypeTests { #expect(DatabaseType.allKnownTypes.contains(.dameng)) #expect(DatabaseType.allKnownTypes.contains(.kafka)) #expect(DatabaseType.allKnownTypes.contains(.cloudflareR2SQL)) + #expect(DatabaseType.allKnownTypes.contains(.weaviate)) } /// The list has no duplicates, which a count alone would not catch: adding a type twice @@ -91,6 +92,13 @@ struct DatabaseTypeTests { #expect(DatabaseType.cloudflareR2SQL.pluginTypeId == "Cloudflare R2 SQL") } + @Test("Weaviate resolves its icon and plugin type id") + func weaviateIdentity() { + #expect(DatabaseType.weaviate.rawValue == "Weaviate") + #expect(DatabaseType.weaviate.iconName == "weaviate-icon") + #expect(DatabaseType.weaviate.pluginTypeId == "Weaviate") + } + @Test("Hashable conformance") func hashableConformance() { var set: Set = [.mysql, .postgresql, .mysql] diff --git a/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateAuthAndQueryTests.swift b/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateAuthAndQueryTests.swift new file mode 100644 index 0000000000..e11c48cd81 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateAuthAndQueryTests.swift @@ -0,0 +1,195 @@ +import Foundation +import Testing +@testable import TableProWeaviateCore + +@Suite("Weaviate auth and settings") +struct WeaviateAuthTests { + @Test("Field ids stay Weaviate-prefixed") + func fieldIdsArePrefixed() { + #expect(WeaviateFieldID.authMethod == "wvAuthMethod") + #expect(WeaviateFieldID.apiKey == "wvApiKey") + #expect(WeaviateFieldID.skipTLSVerify == "wvSkipTLSVerify") + #expect(WeaviateFieldID.authMethod != "esAuthMethod") + #expect(WeaviateFieldID.apiKey != "esApiKey") + } + + @Test("API key mode without a key is refused") + func apiKeyRequiresValue() { + #expect(throws: WeaviateError.configuration("Enter a Weaviate API key.")) { + _ = try WeaviateConnectionSettings.parse( + host: "localhost", + port: 8_080, + usesTLS: false, + fields: [WeaviateFieldID.authMethod: "apiKey"] + ) + } + } + + @Test("A pasted key is not sent when Auth Method is None") + func noneModeIgnoresAPastedKey() throws { + let settings = try WeaviateConnectionSettings.parse( + host: "localhost", + port: 8_080, + usesTLS: false, + fields: [ + WeaviateFieldID.authMethod: "none", + WeaviateFieldID.apiKey: "wv-secret" + ] + ) + #expect(settings.auth.authorizationHeader == nil) + } + + @Test("None mode does not require a key") + func noneModeConnects() throws { + let settings = try WeaviateConnectionSettings.parse( + host: "localhost", + port: 0, + usesTLS: false, + fields: [WeaviateFieldID.authMethod: "none"] + ) + #expect(settings.port == 8_080) + #expect(settings.auth.authorizationHeader == nil) + #expect(try settings.baseURL().absoluteString == "http://localhost:8080") + } + + @Test("TLS uses https") + func tlsUsesHTTPS() throws { + let settings = try WeaviateConnectionSettings.parse( + host: "example.weaviate.cloud", + port: 443, + usesTLS: true, + fields: [ + WeaviateFieldID.authMethod: "apiKey", + WeaviateFieldID.apiKey: "key" + ] + ) + #expect(try settings.baseURL().absoluteString == "https://example.weaviate.cloud:443") + #expect(settings.auth.authorizationHeader == "Bearer key") + } +} + +@Suite("Weaviate query tags") +struct WeaviateQueryTests { + @Test("Browse tags round-trip") + func browseRoundTrip() throws { + let encoded = WeaviateBrowseQuery.encode( + collection: "Article", + offset: 10, + limit: 25, + sorts: [WeaviateSortSpec(column: "title", ascending: true)], + filters: [WeaviateFilterSpec(column: "title", op: "=", value: "Hello")], + logicMode: "AND", + propertyNames: ["uuid", "title"] + ) + #expect(WeaviateBrowseQuery.isTagged(encoded)) + let parsed = try #require(WeaviateBrowseQuery.parse(encoded)) + #expect(parsed.collection == "Article") + #expect(parsed.offset == 10) + #expect(parsed.limit == 25) + #expect(parsed.usesGraphQL) + #expect(parsed.filters.first?.value == "Hello") + } + + @Test("An unfiltered browse stays on REST objects") + func unfilteredUsesREST() throws { + let encoded = WeaviateBrowseQuery.encode( + collection: "Article", + offset: 0, + limit: 25, + sorts: [], + filters: [], + logicMode: "AND", + propertyNames: ["uuid", "title"] + ) + let parsed = try #require(WeaviateBrowseQuery.parse(encoded)) + #expect(!parsed.usesGraphQL) + } + + @Test("Write tags round-trip") + func writeRoundTrip() throws { + let original = WeaviateWriteRequest( + method: "PATCH", + path: "/v1/objects/abc", + query: ["class": "Article"], + body: "{\"title\":\"x\"}" + ) + let encoded = WeaviateWriteCodec.encode(original) + #expect(WeaviateWriteCodec.isTagged(encoded)) + #expect(WeaviateWriteCodec.decode(encoded) == original) + } +} + +@Suite("Weaviate GraphQL and console") +struct WeaviateGraphQLTests { + @Test("A Get query asks for _additional id and vector") + func getQueryIncludesAdditional() throws { + let query = try WeaviateGraphQL.getQuery( + collection: "Article", + properties: ["uuid", "title", "vector"], + limit: 10, + offset: 0, + sorts: [], + filters: [WeaviateFilterSpec(column: "title", op: "=", value: "Hello")], + logicMode: "AND", + schema: ["title": WeaviateProperty(name: "title", dataType: "text")] + ) + #expect(query.contains("Get")) + #expect(query.contains("Article")) + #expect(query.contains("title")) + #expect(query.contains("_additional { id vector }")) + #expect(query.contains("operator: Equal")) + #expect(query.contains("valueText: \"Hello\"")) + #expect(!query.contains(" uuid ")) + } + + @Test("GraphQL detection") + func detection() { + #expect(WeaviateGraphQL.looksLikeGraphQL("{ Get { Article { title } } }")) + #expect(WeaviateGraphQL.looksLikeGraphQL("query { Get { Article { title } } }")) + #expect(!WeaviateGraphQL.looksLikeGraphQL("GET /v1/schema")) + } + + @Test("Console parser reads a method and path") + func consoleParser() throws { + let request = try #require(WeaviateConsoleParser.parse("GET /v1/schema")) + #expect(request.method == "GET") + #expect(request.path == "/v1/schema") + #expect(request.body == nil) + + let withBody = try #require(WeaviateConsoleParser.parse("POST /v1/graphql\n{ \"query\": \"{ Get { Article { title } } }\" }")) + #expect(withBody.method == "POST") + #expect(withBody.body?.contains("Get") == true) + } + + @Test("A SQL delete is not a console request") + func sqlIsNotConsole() { + #expect(WeaviateConsoleParser.parse("DELETE FROM Article") == nil) + #expect(WeaviateConsoleParser.parse("UPDATE Article SET title = 'x'") == nil) + #expect(WeaviateConsoleParser.parse("GET schema") == nil) + } +} + +@Suite("Weaviate columns") +struct WeaviateSchemaTests { + @Test("uuid leads and vector trails, and both are the primary key surface") + func columnOrder() { + let collection = WeaviateCollection( + name: "Article", + properties: [WeaviateProperty(name: "title", dataType: "text")] + ) + let columns = WeaviateSchema.columns(for: collection) + #expect(columns.map(\.name) == ["uuid", "title", "vector"]) + #expect(columns.first?.isPrimaryKey == true) + #expect(WeaviateSchema.immutableColumns == ["uuid", "vector"]) + } + + @Test("Object and array properties display as JSON") + func displayTextEncodesCollections() { + #expect(WeaviateJSON.displayText(["title": "Hello"]) == "{\"title\":\"Hello\"}") + let vector = WeaviateJSON.displayText([0.1, 0.2]) + #expect(vector?.hasPrefix("[") == true) + #expect(vector?.hasSuffix("]") == true) + #expect(WeaviateJSON.displayText(true) == "true") + #expect(WeaviateJSON.displayText(NSNull()) == nil) + } +} diff --git a/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateClientTests.swift b/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateClientTests.swift new file mode 100644 index 0000000000..2ea2b8300b --- /dev/null +++ b/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateClientTests.swift @@ -0,0 +1,279 @@ +import Foundation +import Testing +@testable import TableProWeaviateCore + +@Suite("Weaviate client") +struct WeaviateClientTests { + @Test("Connect reads ready and meta, and stores the version") + func connectStoresVersion() async throws { + let transport = FakeWeaviateTransport() + transport.respond(method: "GET", path: "/v1/.well-known/ready", status: 200, body: ".") + transport.respond(method: "GET", path: "/v1/meta", status: 200, json: WeaviateFixtures.meta) + let client = testClient(transport: transport) + + try await client.connect() + + #expect(client.serverVersion == "1.27.0") + #expect(transport.requests.map { $0.url.path } == ["/v1/.well-known/ready", "/v1/meta"]) + } + + @Test("An API key is sent as a Bearer header") + func apiKeyIsBearer() async throws { + let transport = FakeWeaviateTransport() + transport.respond(method: "GET", path: "/v1/meta", status: 200, json: WeaviateFixtures.meta) + let client = testClient( + transport: transport, + auth: WeaviateAuth(method: .apiKey, apiKey: "wv-secret") + ) + + try await client.ping() + + #expect(transport.requests.first?.headers["Authorization"] == "Bearer wv-secret") + } + + @Test("A ping asks an endpoint that checks the key") + func pingIsAuthenticated() async throws { + let transport = FakeWeaviateTransport() + transport.respond(method: "GET", path: "/v1/meta", status: 200, json: WeaviateFixtures.meta) + let client = testClient(transport: transport) + + try await client.ping() + + #expect(transport.requests.map { $0.url.path } == ["/v1/meta"]) + } + + @Test("Anonymous auth sends no Authorization header") + func anonymousHasNoAuthorization() async throws { + let transport = FakeWeaviateTransport() + transport.respond(method: "GET", path: "/v1/meta", status: 200, json: WeaviateFixtures.meta) + let client = testClient(transport: transport) + + try await client.ping() + + #expect(transport.requests.first?.headers["Authorization"] == nil) + } + + @Test("Schema lists collections and properties") + func schemaListsCollections() async throws { + let transport = FakeWeaviateTransport() + transport.respond(method: "GET", path: "/v1/schema", status: 200, json: WeaviateFixtures.schema) + let client = testClient(transport: transport) + + let collections = try await client.schema() + + #expect(collections.map(\.name) == ["Article"]) + #expect(collections.first?.properties.map(\.name) == ["title", "wordCount"]) + #expect(collections.first?.properties.map(\.dataType) == ["text", "int"]) + } + + @Test("Objects include uuid, properties and the vector as display text") + func objectsIncludeUUIDAndVector() async throws { + let transport = FakeWeaviateTransport() + transport.respond(method: "GET", path: "/v1/objects", status: 200, json: WeaviateFixtures.objects) + let client = testClient(transport: transport) + + let objects = try await client.objects(collection: "Article", limit: 25, offset: 0) + let object = try #require(objects.first) + let columns = ["uuid", "title", "wordCount", "vector"] + let row = WeaviateObjectCodec.row(for: object, columns: columns) + + #expect(object.uuid == WeaviateFixtures.articleUUID) + #expect(row[0] == WeaviateFixtures.articleUUID) + #expect(row[1] == "Hello") + #expect(row[2] == "12") + #expect(row[3]?.contains("0.1") == true) + #expect(row[3]?.hasPrefix("[") == true) + #expect(transport.requests.first?.url.query?.contains("class=Article") == true) + #expect(transport.requests.first?.url.query?.contains("include=vector") == true) + } + + @Test("HTTP 401 becomes an authentication error") + func unauthorizedIsAuthentication() async throws { + let transport = FakeWeaviateTransport() + transport.respond( + method: "GET", + path: "/v1/meta", + status: 401, + json: ["error": [["message": "invalid api key"]]] + ) + let client = testClient(transport: transport) + + await #expect(throws: WeaviateError.authentication("invalid api key")) { + try await client.ping() + } + } + + @Test("A server error body is surfaced") + func serverErrorMessage() async throws { + let transport = FakeWeaviateTransport() + transport.respond( + method: "GET", + path: "/v1/schema", + status: 500, + json: ["error": [["message": "store unavailable"]]] + ) + let client = testClient(transport: transport) + + await #expect(throws: WeaviateError.api(status: 500, message: "store unavailable")) { + _ = try await client.schema() + } + } + + @Test("GraphQL Get rows flatten uuid from _additional") + func graphqlGetFlattensUUID() async throws { + let transport = FakeWeaviateTransport() + transport.respond(method: "POST", path: "/v1/graphql", status: 200, json: WeaviateFixtures.graphqlGet) + let client = testClient(transport: transport) + + let response = try await client.graphql("{ Get { Article { title } } }") + let objects = WeaviateObjectCodec.objects(fromGraphQL: response.json as Any) + let object = try #require(objects.first) + + #expect(object.uuid == WeaviateFixtures.articleUUID) + #expect(object.properties["title"] == "Hello") + #expect(object.vectorText?.contains("0.1") == true) + } + + @Test("GraphQL errors in a 200 response still fail") + func graphqlErrorsFail() async throws { + let transport = FakeWeaviateTransport() + transport.respond( + method: "POST", + path: "/v1/graphql", + status: 200, + json: ["errors": [["message": "Cannot query field"]]] + ) + let client = testClient(transport: transport) + + await #expect(throws: WeaviateError.api(status: 200, message: "Cannot query field")) { + _ = try await client.graphql("{ Get { Missing { title } } }") + } + } +} + +@Suite("Weaviate uuid edits") +struct WeaviateUUIDEditTests { + @Test("An update is a PATCH keyed by uuid and does not write uuid or vector") + func updateIsPatchByUUID() async throws { + let batch = WeaviateStatementGenerator.generate( + collection: "Article", + columns: ["uuid", "title", "vector"], + typeNames: ["uuid", "text", "vector"], + changes: [ + WeaviateTrackedChange( + kind: .update, + uuid: WeaviateFixtures.articleUUID, + values: [:], + cellChanges: [ + WeaviateCellChange(column: "title", newText: "Edited"), + WeaviateCellChange(column: "vector", newText: "[9,9]"), + WeaviateCellChange(column: "uuid", newText: "nope") + ] + ) + ] + ) + let request = try #require(batch.requests.first) + #expect(request.method == "PATCH") + #expect(request.path == "/v1/objects/\(WeaviateFixtures.articleUUID)") + #expect(request.query["class"] == "Article") + let body = try #require(request.body) + #expect(body.contains("\"title\":\"Edited\"")) + #expect(!body.contains("vector")) + #expect(!body.contains("nope")) + + let transport = FakeWeaviateTransport() + transport.respond( + method: "PATCH", + path: "/v1/objects/\(WeaviateFixtures.articleUUID)", + status: 200, + json: ["id": WeaviateFixtures.articleUUID] + ) + let client = testClient(transport: transport) + let response = try await client.execute(write: request) + #expect(response.statusCode == 200) + #expect(transport.requests.first?.httpMethodMatchesPatch == true) + } + + @Test("A delete is DELETE /v1/objects/{uuid}") + func deleteUsesUUID() async throws { + let batch = WeaviateStatementGenerator.generate( + collection: "Article", + columns: ["uuid", "title"], + typeNames: ["uuid", "text"], + changes: [ + WeaviateTrackedChange( + kind: .delete, + uuid: WeaviateFixtures.articleUUID, + values: [:], + cellChanges: [] + ) + ] + ) + let request = try #require(batch.requests.first) + #expect(request.method == "DELETE") + #expect(request.path.hasSuffix(WeaviateFixtures.articleUUID)) + + let transport = FakeWeaviateTransport() + transport.respond( + method: "DELETE", + path: "/v1/objects/\(WeaviateFixtures.articleUUID)", + status: 204, + body: "" + ) + let client = testClient(transport: transport) + let response = try await client.execute(write: request) + #expect(response.statusCode == 204) + } + + @Test("An update without a uuid is skipped") + func updateWithoutUUIDIsSkipped() { + let batch = WeaviateStatementGenerator.generate( + collection: "Article", + columns: ["uuid", "title"], + typeNames: ["uuid", "text"], + changes: [ + WeaviateTrackedChange( + kind: .update, + uuid: nil, + values: [:], + cellChanges: [WeaviateCellChange(column: "title", newText: "Edited")] + ) + ] + ) + #expect(batch.requests.isEmpty) + #expect(batch.skipped == [WeaviateSkippedChange(kind: .update, reason: .missingUUID)]) + } + + @Test("Insert posts the collection and properties, and an explicit uuid") + func insertPostsObject() throws { + let batch = WeaviateStatementGenerator.generate( + collection: "Article", + columns: ["uuid", "title", "wordCount"], + typeNames: ["uuid", "text", "int"], + changes: [ + WeaviateTrackedChange( + kind: .insert, + uuid: WeaviateFixtures.articleUUID, + values: [ + "uuid": WeaviateFixtures.articleUUID, + "title": "Hello", + "wordCount": "12" + ], + cellChanges: [] + ) + ] + ) + let request = try #require(batch.requests.first) + #expect(request.method == "POST") + #expect(request.path == "/v1/objects") + let body = try #require(request.body) + #expect(body.contains("\"class\":\"Article\"")) + #expect(body.contains("\"id\":\"\(WeaviateFixtures.articleUUID)\"")) + #expect(body.contains("\"title\":\"Hello\"")) + #expect(body.contains("\"wordCount\":12")) + } +} + +private extension WeaviateHTTPRequest { + var httpMethodMatchesPatch: Bool { method == "PATCH" } +} diff --git a/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateFilterAndPathTests.swift b/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateFilterAndPathTests.swift new file mode 100644 index 0000000000..4b3336785c --- /dev/null +++ b/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateFilterAndPathTests.swift @@ -0,0 +1,565 @@ +import Foundation +import Testing +@testable import TableProWeaviateCore + +private let articleTypes = [ + "title": "text", + "wordCount": "int", + "ratio": "number", + "live": "boolean", + "published": "date", + "tags": "text[]", + "scores": "int[]" +] + +private let articleSchema = articleTypes.mapValues { WeaviateProperty(name: "", dataType: $0) } + +private func operand(_ column: String, _ op: String, _ value: String, second: String? = nil) throws -> String { + try WeaviateFilterBuilder.operand( + for: WeaviateFilterSpec(column: column, op: op, value: value, secondValue: second), + types: articleTypes + ) +} + +@Suite("Weaviate filter value types") +struct WeaviateFilterValueTypeTests { + @Test("Each data type picks the value field Weaviate demands") + func valueFieldFollowsDataType() throws { + #expect(try operand("title", "=", "Hello").contains("valueText: \"Hello\"")) + #expect(try operand("wordCount", "=", "12").contains("valueInt: 12")) + #expect(try operand("ratio", ">", "1.5").contains("valueNumber: 1.5")) + #expect(try operand("live", "=", "true").contains("valueBoolean: true")) + #expect(try operand("published", ">", "2024-01-31T00:00:00Z") + .contains("valueDate: \"2024-01-31T00:00:00Z\"")) + } + + @Test("An array property filters on its element type") + func arrayUsesElementField() throws { + #expect(try operand("tags", "=", "alpha").contains("valueText: \"alpha\"")) + #expect(try operand("scores", "=", "3").contains("valueInt: 3")) + } + + @Test("The uuid column filters as text on the id path") + func uuidFiltersAsText() throws { + let clause = try operand(WeaviateSchema.uuidColumn, "=", "c8f5c3e0-1b2a-4d3e-9f10-111213141516") + #expect(clause.contains("path: [\"id\"]")) + #expect(clause.contains("valueText:")) + } + + @Test("A value that is not a number is refused instead of becoming zero") + func badNumberThrows() { + #expect(throws: WeaviateFilterError.notANumber(column: "wordCount", value: "abc")) { + _ = try operand("wordCount", "=", "abc") + } + #expect(throws: WeaviateFilterError.notANumber(column: "ratio", value: "abc")) { + _ = try operand("ratio", "=", "abc") + } + } + + @Test("A value that is not a boolean is refused instead of becoming false") + func badBooleanThrows() { + #expect(throws: WeaviateFilterError.notABoolean(column: "live", value: "yes")) { + _ = try operand("live", "=", "yes") + } + } + + @Test("A date needs a full RFC 3339 timestamp") + func badDateThrows() { + #expect(throws: WeaviateFilterError.notADate(column: "published", value: "2024-01-31")) { + _ = try operand("published", ">", "2024-01-31") + } + #expect(WeaviateDateLiteral.isRFC3339("2024-01-31T00:00:00+07:00")) + #expect(WeaviateDateLiteral.isRFC3339("2024-01-31T00:00:00.123Z")) + } +} + +@Suite("Weaviate filter operators") +struct WeaviateFilterOperatorTests { + @Test("Substring operators become Like patterns") + func likePatterns() throws { + #expect(try operand("title", "CONTAINS", "ell").contains("operator: Like valueText: \"*ell*\"")) + #expect(try operand("title", "STARTS WITH", "He").contains("valueText: \"He*\"")) + #expect(try operand("title", "ENDS WITH", "lo").contains("valueText: \"*lo\"")) + #expect(try operand("title", "NOT CONTAINS", "ell").hasPrefix("{ operator: Not operands: [")) + } + + @Test("IN and NOT IN become ContainsAny") + func listOperators() throws { + let inClause = try operand("wordCount", "IN", "10, 30") + #expect(inClause.contains("operator: ContainsAny valueInt: [10, 30]")) + let notIn = try operand("title", "NOT IN", "a,b") + #expect(notIn.contains("operator: ContainsNone valueText: [\"a\", \"b\"]")) + } + + @Test("IN with no values is refused") + func emptyListThrows() { + #expect(throws: WeaviateFilterError.emptyList(column: "title")) { + _ = try operand("title", "IN", " , ") + } + } + + @Test("BETWEEN becomes a bounded And and needs its upper bound") + func betweenBounds() throws { + let clause = try operand("wordCount", "BETWEEN", "5", second: "10") + #expect(clause.hasPrefix("{ operator: And operands: [")) + #expect(clause.contains("operator: GreaterThanEqual valueInt: 5")) + #expect(clause.contains("operator: LessThanEqual valueInt: 10")) + + #expect(throws: WeaviateFilterError.missingUpperBound(column: "wordCount")) { + _ = try operand("wordCount", "BETWEEN", "5") + } + } + + @Test("IS NULL maps to IsNull in both directions") + func nullOperators() throws { + #expect(try operand("title", "IS NULL", "").contains("operator: IsNull valueBoolean: true")) + #expect(try operand("title", "IS NOT NULL", "").contains("operator: IsNull valueBoolean: false")) + } + + @Test("An operator Weaviate cannot express is reported, not dropped") + func unsupportedOperatorsThrow() { + #expect(throws: WeaviateFilterError.unsupportedOperator("REGEX")) { + _ = try operand("title", "REGEX", "^a") + } + } + + @Test("Emptiness counts with len(), which is what Weaviate offers") + func emptinessUsesLength() throws { + #expect(try operand("title", "IS EMPTY", "") + .contains("path: [\"len(title)\"] operator: Equal valueInt: 0")) + #expect(try operand("title", "IS NOT EMPTY", "") + .contains("path: [\"len(title)\"] operator: GreaterThan valueInt: 0")) + #expect(throws: WeaviateFilterError.textMatchNeedsText(column: "wordCount", op: "IS EMPTY")) { + _ = try operand("wordCount", "IS EMPTY", "") + } + } + + @Test("Comparing text, and matching a number as text, are both refused") + func mismatchedOperatorsThrow() { + #expect(throws: WeaviateFilterError.comparisonNeedsNumberOrDate(column: "title", op: ">")) { + _ = try operand("title", ">", "Row 1") + } + #expect(throws: WeaviateFilterError.textMatchNeedsText(column: "wordCount", op: "CONTAINS")) { + _ = try operand("wordCount", "CONTAINS", "1") + } + } + + @Test("The vector column cannot be filtered") + func vectorFilterThrows() { + #expect(throws: WeaviateFilterError.vectorNotFilterable(column: "vector")) { + _ = try operand(WeaviateSchema.vectorColumn, "=", "[1,2]") + } + } + + @Test("A quote in a value cannot break out of the GraphQL string") + func valuesAreEscaped() throws { + let clause = try operand("title", "=", "a\"b\\c") + #expect(clause.contains("valueText: \"a\\\"b\\\\c\"")) + } + + @Test("Several filters join under one logic operator") + func logicMode() throws { + let built = try WeaviateFilterBuilder.graphQLWhere( + filters: [ + WeaviateFilterSpec(column: "title", op: "=", value: "a"), + WeaviateFilterSpec(column: "wordCount", op: ">", value: "3") + ], + logicMode: "OR", + types: articleTypes + ) + let clause = try #require(built) + #expect(clause.hasPrefix("{ operator: Or operands: [")) + } +} + +@Suite("Weaviate sorting") +struct WeaviateSortTests { + @Test("A uuid sort reaches GraphQL, which can sort by the object id") + func uuidSortUsesGraphQL() throws { + let encoded = WeaviateBrowseQuery.encode( + collection: "Article", + offset: 0, + limit: 25, + sorts: [WeaviateSortSpec(column: WeaviateSchema.uuidColumn, ascending: false)], + filters: [], + logicMode: "AND", + propertyNames: ["uuid", "title"] + ) + let parsed = try #require(WeaviateBrowseQuery.parse(encoded)) + #expect(parsed.usesGraphQL) + + let query = try WeaviateGraphQL.getQuery( + collection: "Article", + properties: parsed.propertyNames, + limit: parsed.limit, + offset: parsed.offset, + sorts: parsed.sortableSorts, + filters: [], + logicMode: "AND", + schema: articleSchema + ) + #expect(query.contains("sort: [{ path: [\"id\"] order: desc }]")) + } + + @Test("A vector sort is dropped, because Weaviate has no such property") + func vectorSortIsDropped() throws { + let encoded = WeaviateBrowseQuery.encode( + collection: "Article", + offset: 0, + limit: 25, + sorts: [WeaviateSortSpec(column: WeaviateSchema.vectorColumn, ascending: true)], + filters: [], + logicMode: "AND", + propertyNames: ["uuid", "vector"] + ) + let parsed = try #require(WeaviateBrowseQuery.parse(encoded)) + #expect(parsed.sortableSorts.isEmpty) + #expect(!parsed.usesGraphQL) + } + + @Test("BETWEEN survives the browse tag round-trip") + func secondValueRoundTrips() throws { + let encoded = WeaviateBrowseQuery.encode( + collection: "Article", + offset: 0, + limit: 25, + sorts: [], + filters: [WeaviateFilterSpec(column: "wordCount", op: "BETWEEN", value: "5", secondValue: "10")], + logicMode: "AND", + propertyNames: ["uuid"] + ) + let parsed = try #require(WeaviateBrowseQuery.parse(encoded)) + #expect(parsed.filters.first?.secondValue == "10") + } +} + +@Suite("Weaviate request paths") +struct WeaviatePathTests { + private let base = URL(string: "http://localhost:8080")! + + @Test("A console path keeps its query string instead of encoding the question mark") + func consoleQueryStringSurvives() throws { + let url = try #require( + WeaviatePathEncoding.resolve("/v1/objects?class=Article&limit=10", against: base) + ) + #expect(url.path == "/v1/objects") + #expect(!url.absoluteString.contains("%3F")) + #expect(url.query == "class=Article&limit=10") + } + + @Test("A path and passed query items merge") + func mergedQueryItems() throws { + let url = try #require( + WeaviatePathEncoding.resolve("/v1/objects?class=Article", query: ["limit": "5"], against: base) + ) + let items = try #require(URLComponents(url: url, resolvingAgainstBaseURL: false)?.queryItems) + #expect(items.contains(URLQueryItem(name: "class", value: "Article"))) + #expect(items.contains(URLQueryItem(name: "limit", value: "5"))) + } + + @Test("An already-encoded segment is not encoded twice") + func encodedSegmentSurvives() throws { + let uuid = "c8f5c3e0-1b2a-4d3e-9f10-111213141516" + let url = try #require( + WeaviatePathEncoding.resolve("/v1/objects/\(WeaviatePathEncoding.segment(uuid))", against: base) + ) + #expect(url.path == "/v1/objects/\(uuid)") + } + + @Test("An absolute or relative path is refused") + func hostiledPathsRefused() { + #expect(WeaviatePathEncoding.resolve("http://evil.example/v1", against: base) == nil) + #expect(WeaviatePathEncoding.resolve("v1/schema", against: base) == nil) + } +} + +@Suite("Weaviate response decoding") +struct WeaviateResponseDecodingTests { + @Test("A vector component at the edge of Int does not trap") + func hugeVectorComponent() { + let object = WeaviateObject( + uuid: "u1", + className: "Article", + properties: [:], + vector: [9_223_372_036_854_775_808.0, 1.5] + ) + let text = object.vectorText + #expect(text?.hasPrefix("[9.223372036854776e+18") == true) + #expect(text?.hasSuffix("1.5]") == true) + } + + @Test("A Get row keeps the _additional fields a vector search returns") + func additionalFieldsBecomeColumns() { + let objects = WeaviateObjectCodec.objects(fromGraphQL: [ + "data": [ + "Get": [ + "Article": [ + [ + "title": "Hello", + "_additional": ["id": "u1", "distance": 0.42, "score": "0.9"] + ] + ] + ] + ] + ]) + let object = objects.first + #expect(object?.uuid == "u1") + #expect(object?.properties["title"] == "Hello") + #expect(object?.properties["distance"] == "0.42") + #expect(object?.properties["score"] == "0.9") + } + + @Test("A property keeps its name when _additional carries the same one") + func propertyWinsOverAdditional() { + let objects = WeaviateObjectCodec.objects(fromGraphQL: [ + "data": ["Get": ["Article": [["distance": "mine", "_additional": ["id": "u1", "distance": 0.42]]]]] + ]) + #expect(objects.first?.properties["distance"] == "mine") + #expect(objects.first?.properties["_additional.distance"] == "0.42") + } + + @Test("A response body is parsed once and still compares by its bytes") + func responseEquality() { + let body = Data(#"{"a":1}"#.utf8) + let first = WeaviateHTTPResponse(statusCode: 200, body: body) + let second = WeaviateHTTPResponse(statusCode: 200, body: body) + #expect(first == second) + #expect(WeaviateJSON.dictionary(first.json)?["a"] as? Int == 1) + #expect(WeaviateJSON.dictionary(first.json)?["a"] as? Int == 1) + #expect(first != WeaviateHTTPResponse(statusCode: 500, body: body)) + } +} + +@Suite("Weaviate write generation") +struct WeaviateWriteGenerationTests { + @Test("A duplicate column name does not trap the generator") + func duplicateColumnNames() throws { + let batch = WeaviateStatementGenerator.generate( + collection: "Article", + columns: ["uuid", "title", "title"], + typeNames: ["uuid", "text", "int"], + changes: [ + WeaviateTrackedChange( + kind: .update, + uuid: "c8f5c3e0-1b2a-4d3e-9f10-111213141516", + values: [:], + cellChanges: [WeaviateCellChange(column: "title", newText: "Edited")] + ) + ] + ) + let body = try #require(batch.requests.first?.body) + #expect(body.contains("\"title\":\"Edited\"")) + } + + @Test("A delete with no uuid is reported rather than dropped in silence") + func deleteWithoutUUIDIsReported() { + let batch = WeaviateStatementGenerator.generate( + collection: "Article", + columns: ["uuid", "title"], + typeNames: ["uuid", "text"], + changes: [WeaviateTrackedChange(kind: .delete, uuid: nil, values: [:], cellChanges: [])] + ) + #expect(batch.requests.isEmpty) + #expect(batch.skipped == [WeaviateSkippedChange(kind: .delete, reason: .missingUUID)]) + } + + @Test("An update touching only read-only columns is reported") + func updateWithNoEditableColumns() { + let batch = WeaviateStatementGenerator.generate( + collection: "Article", + columns: ["uuid", "vector"], + typeNames: ["uuid", "vector"], + changes: [ + WeaviateTrackedChange( + kind: .update, + uuid: "c8f5c3e0-1b2a-4d3e-9f10-111213141516", + values: [:], + cellChanges: [WeaviateCellChange(column: "vector", newText: "[1,2]")] + ) + ] + ) + #expect(batch.requests.isEmpty) + #expect(batch.skipped == [WeaviateSkippedChange(kind: .update, reason: .noEditableColumns)]) + } +} + + +@Suite("Weaviate selection sets") +struct WeaviateSelectionSetTests { + private func query(_ properties: [WeaviateProperty]) throws -> String { + try WeaviateGraphQL.getQuery( + collection: "Article", + properties: properties.map(\.name), + limit: 5, + offset: 0, + sorts: [], + filters: [], + logicMode: "AND", + schema: Dictionary(properties.map { ($0.name, $0) }, uniquingKeysWith: { first, _ in first }) + ) + } + + @Test("A structured property asks for its own fields instead of failing the query") + func structuredPropertiesGetSubSelections() throws { + let built = try query([ + WeaviateProperty(name: "title", dataType: "text"), + WeaviateProperty(name: "place", dataType: "geoCoordinates"), + WeaviateProperty(name: "phone", dataType: "phoneNumber") + ]) + #expect(built.contains("place { latitude longitude }")) + #expect(built.contains("phone { input internationalFormatted")) + #expect(built.contains(" title ")) + } + + @Test("An object property selects its declared nested properties") + func objectSelectsNested() throws { + let built = try query([ + WeaviateProperty( + name: "meta", + dataTypes: ["object"], + nestedProperties: [ + WeaviateProperty(name: "k", dataType: "text"), + WeaviateProperty(name: "inner", dataTypes: ["object"], nestedProperties: [ + WeaviateProperty(name: "deep", dataType: "int") + ]) + ] + ) + ]) + #expect(built.contains("meta { k inner { deep } }")) + } + + @Test("An object with no declared nested properties is left out") + func emptyObjectIsOmitted() throws { + let built = try query([ + WeaviateProperty(name: "title", dataType: "text"), + WeaviateProperty(name: "meta", dataTypes: ["object"], nestedProperties: []) + ]) + #expect(!built.contains("meta")) + #expect(built.contains("title")) + } + + @Test("A cross-reference asks for the referenced ids") + func crossReferenceUsesFragments() throws { + let built = try query([ + WeaviateProperty(name: "category", dataTypes: ["Category", "Topic"], nestedProperties: []) + ]) + #expect(built.contains("category { ... on Category { _additional { id } } ... on Topic { _additional { id } } }")) + } + + @Test("A nested property list survives schema parsing") + func schemaParsesNestedProperties() throws { + let collections = WeaviateSchema.collections(from: [ + "classes": [[ + "class": "Article", + "properties": [ + ["name": "meta", "dataType": ["object"], "nestedProperties": [["name": "k", "dataType": ["text"]]]], + ["name": "category", "dataType": ["Category"]] + ] + ]] + ]) + let properties = try #require(collections.first?.properties) + #expect(properties.first?.nestedProperties.map(\.name) == ["k"]) + #expect(WeaviatePropertyShape.of(properties[1]) == .crossReference(["Category"])) + } +} + +@Suite("Weaviate value round-trip") +struct WeaviateParsedValueTests { + @Test("Text keeps its own punctuation instead of being parsed as JSON") + func textStaysText() { + #expect(WeaviateJSON.parsedValue("{\"a\":1}", typeName: "text") as? String == "{\"a\":1}") + #expect(WeaviateJSON.parsedValue("[1,2]", typeName: "text") as? String == "[1,2]") + #expect(WeaviateJSON.parsedValue("{\"a\":1}", typeName: "string") as? String == "{\"a\":1}") + #expect(WeaviateJSON.parsedValue("2024-01-31T00:00:00Z", typeName: "date") as? String == "2024-01-31T00:00:00Z") + } + + @Test("A type the grid renders as JSON parses back") + func structuredParsesBack() { + #expect(WeaviateJSON.parsedValue("[\"a\",\"b\"]", typeName: "text[]") as? [String] == ["a", "b"]) + #expect(WeaviateJSON.parsedValue("[1,2]", typeName: "int[]") as? [Int] == [1, 2]) + let object = WeaviateJSON.parsedValue("{\"k\":\"v\"}", typeName: "object") as? [String: Any] + #expect(object?["k"] as? String == "v") + let geo = WeaviateJSON.parsedValue("{\"latitude\":1}", typeName: "geoCoordinates") as? [String: Any] + #expect(geo?["latitude"] as? Int == 1) + } + + @Test("A scalar parses to its own type, and an unparsable one stays text") + func scalarsParse() { + #expect(WeaviateJSON.parsedValue("12", typeName: "int") as? Int == 12) + #expect(WeaviateJSON.parsedValue("1.5", typeName: "number") as? Double == 1.5) + #expect(WeaviateJSON.parsedValue("true", typeName: "boolean") as? Bool == true) + #expect(WeaviateJSON.parsedValue("abc", typeName: "int") as? String == "abc") + #expect(WeaviateJSON.parsedValue("yes", typeName: "boolean") as? String == "yes") + } + + @Test("A property named with a leading underscore is written, and a synthetic column is not") + func underscoreNamedPropertyIsWritten() throws { + let batch = WeaviateStatementGenerator.generate( + collection: "Event", + columns: ["uuid", "_source", "distance"], + typeNames: ["uuid", "text", "number"], + changes: [ + WeaviateTrackedChange( + kind: .update, + uuid: "c8f5c3e0-1b2a-4d3e-9f10-111213141516", + values: [:], + cellChanges: [ + WeaviateCellChange(column: "_source", newText: "manual"), + WeaviateCellChange(column: "_additional.distance", newText: "0.4") + ] + ) + ] + ) + let body = try #require(batch.requests.first?.body) + #expect(body.contains("\"_source\":\"manual\"")) + #expect(!body.contains("_additional")) + } +} + +@Suite("Weaviate console requests") +struct WeaviateConsoleRequestTests { + @Test("A body typed on the request line is kept") + func inlineBodySurvives() throws { + let request = try #require( + WeaviateConsoleParser.parse("POST /v1/objects {\"class\": \"Article\"}") + ) + #expect(request.method == "POST") + #expect(request.path == "/v1/objects") + #expect(request.body == "{\"class\": \"Article\"}") + } + + @Test("A body on the lines below still wins") + func multilineBodyWins() throws { + let request = try #require(WeaviateConsoleParser.parse("POST /v1/graphql\n{ \"query\": \"x\" }")) + #expect(request.body == "{ \"query\": \"x\" }") + } + + @Test("Any path is resolved under /v1") + func everyPathIsPrefixed() throws { + #expect(try #require(WeaviateConsoleParser.parse("GET /nodes")).path == "/v1/nodes") + #expect(try #require(WeaviateConsoleParser.parse("POST /batch/objects")).path == "/v1/batch/objects") + #expect(try #require(WeaviateConsoleParser.parse("GET /v1/schema")).path == "/v1/schema") + #expect(try #require(WeaviateConsoleParser.parse("GET /")).path == "/") + } + + @Test("SQL is still not a console request") + func sqlIsRefused() { + #expect(WeaviateConsoleParser.parse("DELETE FROM Article") == nil) + #expect(WeaviateConsoleParser.parse("UPDATE Article SET title = 'x'") == nil) + } + + @Test("A browse that is not showing the vector does not ask for it") + func vectorIsOptional() throws { + let withVector = try WeaviateGraphQL.getQuery( + collection: "Article", properties: ["uuid", "title"], limit: 5, offset: 0, + sorts: [], filters: [], logicMode: "AND", schema: [:], includeVector: true + ) + let withoutVector = try WeaviateGraphQL.getQuery( + collection: "Article", properties: ["uuid", "title"], limit: 5, offset: 0, + sorts: [], filters: [], logicMode: "AND", schema: [:], includeVector: false + ) + #expect(withVector.contains("_additional { id vector }")) + #expect(withoutVector.contains("_additional { id }")) + #expect(!withoutVector.contains("vector")) + } +} diff --git a/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateTestSupport.swift b/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateTestSupport.swift new file mode 100644 index 0000000000..cd6a8ba115 --- /dev/null +++ b/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateTestSupport.swift @@ -0,0 +1,120 @@ +import Foundation +@testable import TableProWeaviateCore + +final class FakeWeaviateTransport: WeaviateTransport, @unchecked Sendable { + struct Route: Equatable { + let method: String + let path: String + } + + var responses: [String: WeaviateHTTPResponse] = [:] + var requests: [WeaviateHTTPRequest] = [] + var error: WeaviateError? + + func send(_ request: WeaviateHTTPRequest) async throws -> WeaviateHTTPResponse { + if let error { + throw error + } + requests.append(request) + let key = Self.key(method: request.method, url: request.url) + if let response = responses[key] ?? responses[request.url.path] { + return response + } + throw WeaviateError.malformedResponse("No fake response for \(key)") + } + + func cancelAll() {} + + func respond(method: String, path: String, status: Int, json: Any) { + let data = (try? JSONSerialization.data(withJSONObject: json)) ?? Data() + responses[Self.key(method: method, path: path)] = WeaviateHTTPResponse(statusCode: status, body: data) + } + + func respond(method: String, path: String, status: Int, body: String) { + responses[Self.key(method: method, path: path)] = WeaviateHTTPResponse( + statusCode: status, + body: Data(body.utf8) + ) + } + + static func key(method: String, path: String) -> String { + "\(method.uppercased()) \(path)" + } + + static func key(method: String, url: URL) -> String { + key(method: method, path: url.path) + } +} + +func testSettings(auth: WeaviateAuth = WeaviateAuth(method: .none)) -> WeaviateConnectionSettings { + WeaviateConnectionSettings( + host: "localhost", + port: 8_080, + usesTLS: false, + auth: auth, + skipTLSVerify: false + ) +} + +func testClient( + transport: FakeWeaviateTransport, + auth: WeaviateAuth = WeaviateAuth(method: .none) +) -> WeaviateClient { + WeaviateClient(settings: testSettings(auth: auth), transport: transport, timeout: { 30 }) +} + +enum WeaviateFixtures { + static let articleUUID = "c8f5c3e0-1b2a-4d3e-9f10-111213141516" + + static var schema: [String: Any] { + [ + "classes": [ + [ + "class": "Article", + "vectorizer": "none", + "properties": [ + ["name": "title", "dataType": ["text"]], + ["name": "wordCount", "dataType": ["int"]] + ] + ] + ] + ] + } + + static var objects: [String: Any] { + [ + "objects": [ + [ + "id": articleUUID, + "class": "Article", + "properties": ["title": "Hello", "wordCount": 12], + "vector": [0.1, 0.2, 0.3] + ] + ], + "totalResults": 1 + ] + } + + static var graphqlGet: [String: Any] { + [ + "data": [ + "Get": [ + "Article": [ + [ + "title": "Hello", + "wordCount": 12, + "_additional": [ + "id": articleUUID, + "vector": [0.1, 0.2] + ] + ] + ] + ] + ] + ] + } + + static var meta: [String: Any] { + ["version": "1.27.0"] + } +} diff --git a/Plugins/MySQLDriverPlugin/MariaDBFieldMetadata.swift b/Plugins/MySQLDriverPlugin/MariaDBFieldMetadata.swift new file mode 100644 index 0000000000..fe98664288 --- /dev/null +++ b/Plugins/MySQLDriverPlugin/MariaDBFieldMetadata.swift @@ -0,0 +1,74 @@ +// +// MariaDBFieldMetadata.swift +// MySQLDriverPlugin +// + +import Foundation +import TableProPluginKit + +internal let mysqlNotNullFlag: UInt = 0x0001 +internal let mysqlPriKeyFlag: UInt = 0x0002 +internal let mysqlBinaryFlag: UInt = 0x0080 +internal let mysqlEnumFlag: UInt = 0x0100 +internal let mysqlAutoIncrementFlag: UInt = 0x0200 +internal let mysqlSetFlag: UInt = 0x0800 +internal let mysqlBinaryCharset: UInt32 = 63 + +internal func makeColumnMeta(name: String, typeName: String, flags: UInt) -> PluginColumnInfo { + PluginColumnInfo( + name: name, + dataType: typeName, + isNullable: (flags & mysqlNotNullFlag) == 0, + isPrimaryKey: (flags & mysqlPriKeyFlag) != 0, + identityKind: (flags & mysqlAutoIncrementFlag) != 0 ? .byDefault : nil + ) +} + +internal func mariaDBTypeName( + typeRaw: UInt32, + flags: UInt, + charsetnr: UInt32, + length: UInt +) -> String { + let isBinary = (flags & mysqlBinaryFlag) != 0 && charsetnr == mysqlBinaryCharset + + switch typeRaw { + case 0: return "DECIMAL" + case 1: return "TINYINT" + case 2: return "SMALLINT" + case 3: return "INT" + case 4: return "FLOAT" + case 5: return "DOUBLE" + case 6: return "NULL" + case 7: return "TIMESTAMP" + case 8: return "BIGINT" + case 9: return "MEDIUMINT" + case 10: return "DATE" + case 11: return "TIME" + case 12: return "DATETIME" + case 13: return "YEAR" + case 14: return "NEWDATE" + case 15: return "VARCHAR" + case 16: return "BIT" + case 245: return "JSON" + case 246: return "NEWDECIMAL" + case 247: return "ENUM" + case 248: return "SET" + case 249: + return isBinary ? "TINYBLOB" : "TINYTEXT" + case 250: + return isBinary ? "MEDIUMBLOB" : "MEDIUMTEXT" + case 251: + return isBinary ? "LONGBLOB" : "LONGTEXT" + case 252: + if isBinary { + return length > 65_535 ? "LONGBLOB" : "BLOB" + } else { + return length > 65_535 ? "LONGTEXT" : "TEXT" + } + case 253: return isBinary ? "VARBINARY" : "VARCHAR" + case 254: return isBinary ? "BINARY" : "CHAR" + case 255: return "GEOMETRY" + default: return "UNKNOWN" + } +} diff --git a/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift b/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift index 819ef2572e..6e29301542 100644 --- a/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift +++ b/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift @@ -11,27 +11,8 @@ import Foundation import OSLog import TableProPluginKit -// MySQL/MariaDB field flag and charset constants -internal let mysqlNotNullFlag: UInt = 0x0001 -internal let mysqlPriKeyFlag: UInt = 0x0002 -internal let mysqlBinaryFlag: UInt = 0x0080 -internal let mysqlEnumFlag: UInt = 0x0100 -internal let mysqlAutoIncrementFlag: UInt = 0x0200 -internal let mysqlSetFlag: UInt = 0x0800 -internal let mysqlBinaryCharset: UInt32 = 63 - private let logger = Logger(subsystem: "com.TablePro", category: "MariaDBPluginConnection") -internal func makeColumnMeta(name: String, typeName: String, flags: UInt) -> PluginColumnInfo { - PluginColumnInfo( - name: name, - dataType: typeName, - isNullable: (flags & mysqlNotNullFlag) == 0, - isPrimaryKey: (flags & mysqlPriKeyFlag) != 0, - identityKind: (flags & mysqlAutoIncrementFlag) != 0 ? .byDefault : nil - ) -} - // MARK: - Error Types struct MariaDBPluginError: Error { @@ -96,60 +77,6 @@ func mysqlTypeToString(_ fieldPtr: UnsafePointer) -> String { ) } -/// Pure mapping from raw MySQL/MariaDB field type code + flags to TablePro's -/// column-type-name string. Separated from `mysqlTypeToString` so it can be -/// unit-tested without an actual `MYSQL_FIELD` struct. -internal func mariaDBTypeName( - typeRaw: UInt32, - flags: UInt, - charsetnr: UInt32, - length: UInt -) -> String { - // Binary flag alone is insufficient — MariaDB sets it on text columns with - // binary collation (e.g. utf8mb4_bin for JSON). Only charset 63 is truly binary. - let isBinary = (flags & mysqlBinaryFlag) != 0 && charsetnr == mysqlBinaryCharset - - switch typeRaw { - case 0: return "DECIMAL" - case 1: return "TINYINT" - case 2: return "SMALLINT" - case 3: return "INT" - case 4: return "FLOAT" - case 5: return "DOUBLE" - case 6: return "NULL" - case 7: return "TIMESTAMP" - case 8: return "BIGINT" - case 9: return "MEDIUMINT" - case 10: return "DATE" - case 11: return "TIME" - case 12: return "DATETIME" - case 13: return "YEAR" - case 14: return "NEWDATE" - case 15: return "VARCHAR" - case 16: return "BIT" - case 245: return "JSON" - case 246: return "NEWDECIMAL" - case 247: return "ENUM" - case 248: return "SET" - case 249: - return isBinary ? "TINYBLOB" : "TINYTEXT" - case 250: - return isBinary ? "MEDIUMBLOB" : "MEDIUMTEXT" - case 251: - return isBinary ? "LONGBLOB" : "LONGTEXT" - case 252: - if isBinary { - return length > 65_535 ? "LONGBLOB" : "BLOB" - } else { - return length > 65_535 ? "LONGTEXT" : "TEXT" - } - case 253: return isBinary ? "VARBINARY" : "VARCHAR" - case 254: return isBinary ? "BINARY" : "CHAR" - case 255: return "GEOMETRY" - default: return "UNKNOWN" - } -} - // MARK: - Connection Class final class MariaDBPluginConnection: @unchecked Sendable { @@ -168,6 +95,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { private let sslConfig: SSLConfiguration private let enableCleartextPlugin: Bool private let queryTimeoutSeconds: Int + private let connectionEncoding: MySQLConnectionEncoding private let stateLock = NSLock() private let cancellationGate = PluginQueryCancellationGate() @@ -240,7 +168,8 @@ final class MariaDBPluginConnection: @unchecked Sendable { database: String, sslConfig: SSLConfiguration, enableCleartextPlugin: Bool = false, - queryTimeoutSeconds: Int = 0 + queryTimeoutSeconds: Int = 0, + connectionEncoding: MySQLConnectionEncoding = .utf8 ) { self.host = host self.port = UInt32(port) @@ -250,6 +179,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { self.sslConfig = sslConfig self.enableCleartextPlugin = enableCleartextPlugin self.queryTimeoutSeconds = queryTimeoutSeconds + self.connectionEncoding = connectionEncoding } deinit { @@ -393,22 +323,43 @@ final class MariaDBPluginConnection: @unchecked Sendable { mysql_close(mysql) throw error } + do { + try establishSessionCharacterSet(on: mysql) + } catch { + mysql_close(mysql) + throw error + } return mysql } - private func readError(from mysql: UnsafeMutablePointer) -> MariaDBPluginError { - let code = mysql_errno(mysql) - let message: String - if let msgPtr = mysql_error(mysql) { - message = String(cString: msgPtr) - } else { - message = "Unknown error" + private func establishSessionCharacterSet(on mysql: UnsafeMutablePointer) throws { + if mysql_set_character_set(mysql, MySQLConnectionEncoding.sessionCharacterSetName) != 0 { + let refusal = errorMessage(from: mysql) + if runSessionStatement(MySQLConnectionEncoding.sessionFallbackStatement, on: mysql) { + logger.notice("Server refused utf8mb4 (\(refusal, privacy: .public)), so the session uses utf8") + } else { + logger.warning("Server refused a UTF-8 session (\(refusal, privacy: .public)); keeping its own") + } } - var sqlState: String? - if let statePtr = mysql_sqlstate(mysql), statePtr[0] != 0 { - sqlState = String(cString: statePtr) + for statement in connectionEncoding.sessionStatements where !runSessionStatement(statement, on: mysql) { + throw readError(from: mysql) + } + } + + private func runSessionStatement(_ statement: String, on mysql: UnsafeMutablePointer) -> Bool { + let status = statement.withCString { mysql_real_query(mysql, $0, UInt(strlen($0))) } + if let discarded = mysql_store_result(mysql) { + mysql_free_result(discarded) } - return MariaDBPluginError(code: code, message: message, sqlState: sqlState) + return status == 0 + } + + private func readError(from mysql: UnsafeMutablePointer) -> MariaDBPluginError { + MariaDBPluginError( + code: mysql_errno(mysql), + message: mysql_error(mysql).map(decodedMessage) ?? "Unknown error", + sqlState: sqlState(mysql_sqlstate(mysql)) + ) } func disconnect() { @@ -670,45 +621,8 @@ final class MariaDBPluginConnection: @unchecked Sendable { } } - let numFields = Int(mysql_num_fields(resultPtr)) - var columns: [String] = [] - var columnTypes: [UInt32] = [] - var columnTypeNames: [String] = [] - var columnIsBinary: [Bool] = [] - var columnIsBoolean: [Bool] = [] - var columnMeta: [PluginColumnInfo] = [] - columns.reserveCapacity(numFields) - columnTypes.reserveCapacity(numFields) - columnTypeNames.reserveCapacity(numFields) - columnIsBinary.reserveCapacity(numFields) - columnIsBoolean.reserveCapacity(numFields) - columnMeta.reserveCapacity(numFields) let sessionFlavor = flavor - - if let fields = mysql_fetch_fields(resultPtr) { - for i in 0.., metadata: UnsafeMutablePointer, - columns: [String], - columnTypes: [UInt32], - columnTypeNames: [String], - columnIsBinary: [Bool], + columns: MySQLResultColumns, rowCap: Int? = nil, generation: Int, sentAt: Date @@ -950,8 +842,8 @@ final class MariaDBPluginConnection: @unchecked Sendable { break } - // Re-fetch truncated columns with correctly sized buffers if fetchStatus == MYSQL_DATA_TRUNCATED { + var grewBuffer = false for i in 0.. Int(resultBinds[i].buffer_length) { @@ -962,33 +854,22 @@ final class MariaDBPluginConnection: @unchecked Sendable { resultBuffers[i] = newBuffer resultBinds[i].buffer = newBuffer resultBinds[i].buffer_length = UInt(actualLength) + grewBuffer = true if mysql_stmt_fetch_column(stmt, &resultBinds[i], UInt32(i), 0) != 0 { logger.warning("mysql_stmt_fetch_column failed for column \(i)") } } } - } - - var row: [PluginCellValue] = [] - for i in 0..= batchSize { continuation.yield(.rows(batch)) batch.removeAll(keepingCapacity: true) @@ -1296,77 +1093,83 @@ final class MariaDBPluginConnection: @unchecked Sendable { _cachedServerVersion } - private static func cellValue( - _ buffer: UnsafeRawBufferPointer, - typeRaw: UInt32, - isBinary: Bool, - isBoolean: Bool, - flavor: MySQLServerFlavor - ) -> PluginCellValue { - if flavor.isDatabend { - if isBoolean { - return .text(DatabendResultShape.booleanText(fromWireText: String(bytes: buffer, encoding: .utf8) ?? "")) - } - if isBinary { - return .bytes(DatabendResultShape.binaryValue(fromWireText: Data(buffer))) - } - if typeRaw == 255 { - return .text(String(bytes: buffer, encoding: .utf8) ?? "") - } - } - if typeRaw == 255 { - return .text(GeometryWKBParser.parse(buffer)) - } - if MariaDBFieldClassifier.isBit(typeRaw: typeRaw) { - return .text(MariaDBFieldClassifier.bitFieldToString(buffer)) - } - if isBinary { - return .bytes(Data(buffer)) - } - if let text = String(bytes: buffer, encoding: .utf8) { - return .text(text) - } - return .text(String(bytes: buffer, encoding: .isoLatin1) ?? "") - } - // MARK: - Private Helpers private func getError() -> MariaDBPluginError { guard let mysql = mysql else { return MariaDBPluginError.notConnected } + return readError(from: mysql) + } - let code = mysql_errno(mysql) - let message: String - if let msgPtr = mysql_error(mysql) { - message = String(cString: msgPtr) - } else { - message = "Unknown error" - } + private func getStmtError(_ stmt: UnsafeMutablePointer) -> MariaDBPluginError { + MariaDBPluginError( + code: mysql_stmt_errno(stmt), + message: mysql_stmt_error(stmt).map(decodedMessage) ?? "Unknown statement error", + sqlState: sqlState(mysql_stmt_sqlstate(stmt)) + ) + } - var sqlState: String? - if let statePtr = mysql_sqlstate(mysql), statePtr[0] != 0 { - sqlState = String(cString: statePtr) - } + private func decodedMessage(_ message: UnsafePointer) -> String { + mysqlSessionText(cString: message, encoding: connectionEncoding) + } - return MariaDBPluginError(code: code, message: message, sqlState: sqlState) + private func sqlState(_ state: UnsafePointer?) -> String? { + guard let state, state[0] != 0 else { return nil } + return String(cString: state) } - private func getStmtError(_ stmt: UnsafeMutablePointer) -> MariaDBPluginError { - let code = mysql_stmt_errno(stmt) - let message: String - if let msgPtr = mysql_stmt_error(stmt) { - message = String(cString: msgPtr) - } else { - message = "Unknown statement error" + private func describeColumns(_ fields: UnsafeMutablePointer?, count: Int) -> MySQLResultColumns { + var columns = MySQLResultColumns() + guard let fields else { return columns } + let sessionFlavor = flavor + for index in 0.. String { + guard let name = field.name else { return "column_\(index)" } + let bytes = UnsafeRawBufferPointer(start: name, count: strnlen(name, Int(field.name_length))) + return mysqlSessionText(bytes, encoding: connectionEncoding) + } - return MariaDBPluginError(code: code, message: message, sqlState: sqlState) + private static func typeCode(of field: MYSQL_FIELD, flags: UInt) -> UInt32 { + if (flags & mysqlSetFlag) != 0 { return 248 } + if (flags & mysqlEnumFlag) != 0 { return 247 } + return field.type.rawValue + } + + private static func characterSetName(forCollation collation: UInt32) -> String? { + guard let info = mariadb_get_charset_by_nr(collation), let name = info.pointee.csname else { return nil } + return String(cString: name) + } + + private func textProtocolRow( + _ row: MYSQL_ROW, + lengths: UnsafeMutablePointer?, + columns: MySQLResultColumns + ) -> [PluginCellValue] { + columns.row(encoding: connectionEncoding) { index in + guard let value = row[index] else { return nil } + return UnsafeRawBufferPointer(start: value, count: Int(clamping: lengths?[index] ?? 0)) + } } } diff --git a/Plugins/MySQLDriverPlugin/MySQLCharacterSet.swift b/Plugins/MySQLDriverPlugin/MySQLCharacterSet.swift new file mode 100644 index 0000000000..92ffcb0bc4 --- /dev/null +++ b/Plugins/MySQLDriverPlugin/MySQLCharacterSet.swift @@ -0,0 +1,111 @@ +// +// MySQLCharacterSet.swift +// MySQLDriverPlugin +// + +import CoreFoundation +import Foundation + +internal struct MySQLCharacterSet: Hashable, Sendable { + static let utf8mb4 = MySQLCharacterSet(serverName: "utf8mb4") + static let latin1 = MySQLCharacterSet(serverName: "latin1") + + let name: String + private let decoding: Decoding + + init(serverName: String) { + let normalized = serverName.trimmingCharacters(in: .whitespaces).lowercased() + name = normalized == "utf8" ? "utf8mb3" : normalized + decoding = Self.decoding(forName: name) + } + + func decode(_ bytes: UnsafeRawBufferPointer) -> String { + switch decoding { + case .utf8: + return Self.decodeUTF8ReplacingInvalid(bytes) + case .utf8OrMySQLLatin1: + return Self.decodeUTF8OrMySQLLatin1(bytes) + case .foundation(let encoding, let isSingleByte): + return Self.decode(bytes, encoding: encoding, isSingleByte: isSingleByte) + } + } + + static func decodeUTF8OrMySQLLatin1(_ bytes: UnsafeRawBufferPointer) -> String { + if let text = String(bytes: bytes, encoding: .utf8) { + return text + } + return MySQLLatin1.decode(bytes) + } + + static func decodeUTF8ReplacingInvalid(_ bytes: UnsafeRawBufferPointer) -> String { + String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + } + + static var singleByteDecodedNames: [String] { + Array(singleByteEncodings.keys).sorted() + } + + static var multiByteDecodedNames: [String] { + Array(multiByteEncodings.keys).sorted() + } + + private enum Decoding: Hashable, Sendable { + case utf8 + case utf8OrMySQLLatin1 + case foundation(String.Encoding, isSingleByte: Bool) + } + + private static let utf8Names: Set = ["utf8mb4", "utf8mb3", "ascii", "binary"] + + private static func decoding(forName name: String) -> Decoding { + if utf8Names.contains(name) { return .utf8 } + if name == "latin1" { return .utf8OrMySQLLatin1 } + if let encoding = singleByteEncodings[name] { return .foundation(encoding, isSingleByte: true) } + if let encoding = multiByteEncodings[name] { return .foundation(encoding, isSingleByte: false) } + return .utf8 + } + + private static let singleByteEncodings: [String: String.Encoding] = [ + "latin2": .isoLatin2, + "cp1250": .windowsCP1250, + "cp1251": .windowsCP1251, + "cp1256": encoding(.windowsArabic), + "cp1257": encoding(.windowsBalticRim), + "cp850": encoding(.dosLatin1), + "cp852": encoding(.dosLatin2), + "latin5": encoding(.isoLatin5), + "macce": encoding(.macCentralEurRoman), + "macroman": .macOSRoman + ] + + private static let multiByteEncodings: [String: String.Encoding] = [ + "cp932": .shiftJIS, + "gbk": encoding(.GBK_95), + "gb2312": encoding(.EUC_CN), + "gb18030": encoding(.GB_18030_2000), + "ucs2": .utf16BigEndian, + "utf16": .utf16BigEndian, + "utf16le": .utf16LittleEndian, + "utf32": .utf32BigEndian + ] + + private static func encoding(_ encoding: CFStringEncodings) -> String.Encoding { + String.Encoding(rawValue: CFStringConvertEncodingToNSStringEncoding(CFStringEncoding(encoding.rawValue))) + } + + private static func decode( + _ bytes: UnsafeRawBufferPointer, + encoding: String.Encoding, + isSingleByte: Bool + ) -> String { + if let text = String(bytes: bytes, encoding: encoding) { + return text + } + guard isSingleByte else { + return decodeUTF8ReplacingInvalid(bytes) + } + return bytes.reduce(into: "") { text, byte in + text += String(bytes: [byte], encoding: encoding) ?? "\u{FFFD}" + } + } +} diff --git a/Plugins/MySQLDriverPlugin/MySQLColumnDecoding.swift b/Plugins/MySQLDriverPlugin/MySQLColumnDecoding.swift new file mode 100644 index 0000000000..9b7b76d97b --- /dev/null +++ b/Plugins/MySQLDriverPlugin/MySQLColumnDecoding.swift @@ -0,0 +1,102 @@ +// +// MySQLColumnDecoding.swift +// MySQLDriverPlugin +// + +import Foundation +import TableProPluginKit + +internal enum MySQLColumnDecoding: Equatable, Sendable { + case geometry + case bit + case bytes + case text(MySQLCharacterSet) + case databendBoolean + case databendHexBytes + + private static let geometryType: UInt32 = 255 + + init( + typeRaw: UInt32, + length: UInt = 0, + charsetnr: UInt32, + characterSetName: String?, + flavor: MySQLServerFlavor = .mysql + ) { + let isBinary = MariaDBFieldClassifier.isBinary(typeRaw: typeRaw, charset: charsetnr) + if flavor.isDatabend, DatabendResultShape.isBoolean(typeRaw: typeRaw, length: length) { + self = .databendBoolean + } else if flavor.isDatabend, isBinary { + self = .databendHexBytes + } else if typeRaw == Self.geometryType { + self = flavor.isDatabend ? .text(.utf8mb4) : .geometry + } else if MariaDBFieldClassifier.isBit(typeRaw: typeRaw) { + self = .bit + } else if isBinary { + self = .bytes + } else if charsetnr == mysqlBinaryCharset { + self = .text(.utf8mb4) + } else { + self = .text(characterSetName.map(MySQLCharacterSet.init(serverName:)) ?? .utf8mb4) + } + } + + func decode(_ bytes: UnsafeRawBufferPointer, encoding: MySQLConnectionEncoding) -> PluginCellValue { + switch self { + case .geometry: + return .text(GeometryWKBParser.parse(bytes)) + case .bit: + return .text(MariaDBFieldClassifier.bitFieldToString(bytes)) + case .bytes: + return .bytes(Data(bytes)) + case .text(let characterSet): + return .text(encoding.presentedText(characterSet.decode(bytes))) + case .databendBoolean: + return .text(DatabendResultShape.booleanText(fromWireText: MySQLCharacterSet.utf8mb4.decode(bytes))) + case .databendHexBytes: + return .bytes(DatabendResultShape.binaryValue(fromWireText: Data(bytes))) + } + } +} + +internal struct MySQLResultColumns { + private(set) var names: [String] = [] + private(set) var typeCodes: [UInt32] = [] + private(set) var typeNames: [String] = [] + private(set) var decodings: [MySQLColumnDecoding] = [] + private(set) var metadata: [PluginColumnInfo] = [] + + var count: Int { names.count } + + mutating func append( + name: String, + typeCode: UInt32, + typeName: String, + decoding: MySQLColumnDecoding, + flags: UInt + ) { + names.append(name) + typeCodes.append(typeCode) + typeNames.append(typeName) + decodings.append(decoding) + metadata.append(makeColumnMeta(name: name, typeName: typeName, flags: flags)) + } + + func row( + encoding: MySQLConnectionEncoding, + value: (Int) -> UnsafeRawBufferPointer? + ) -> [PluginCellValue] { + decodings.indices.map { index in + guard let bytes = value(index) else { return .null } + return decodings[index].decode(bytes, encoding: encoding) + } + } +} + +internal func mysqlSessionText(_ bytes: UnsafeRawBufferPointer, encoding: MySQLConnectionEncoding) -> String { + encoding.presentedText(MySQLCharacterSet.decodeUTF8OrMySQLLatin1(bytes)) +} + +internal func mysqlSessionText(cString: UnsafePointer, encoding: MySQLConnectionEncoding) -> String { + mysqlSessionText(UnsafeRawBufferPointer(start: cString, count: strlen(cString)), encoding: encoding) +} diff --git a/Plugins/MySQLDriverPlugin/MySQLConnectionEncoding.swift b/Plugins/MySQLDriverPlugin/MySQLConnectionEncoding.swift new file mode 100644 index 0000000000..34457c1a42 --- /dev/null +++ b/Plugins/MySQLDriverPlugin/MySQLConnectionEncoding.swift @@ -0,0 +1,50 @@ +// +// MySQLConnectionEncoding.swift +// MySQLDriverPlugin +// + +import Foundation +import TableProPluginKit + +internal enum MySQLConnectionEncoding: String, CaseIterable, Sendable { + case utf8 = "" + case utf8ViaLatin1 + + static let fieldId = "mysqlConnectionEncoding" + static let sessionCharacterSetName = "utf8mb4" + static let sessionFallbackStatement = "SET NAMES utf8" + + static var connectionField: ConnectionField { + ConnectionField( + id: fieldId, + label: String(localized: "Encoding"), + fieldType: .dropdown(options: allCases.map { .init(value: $0.rawValue, label: $0.displayName) }), + section: .advanced + ) + } + + init(fieldValue: String?) { + self = fieldValue.flatMap(Self.init(rawValue:)) ?? .utf8 + } + + var displayName: String { + switch self { + case .utf8: return "UTF-8" + case .utf8ViaLatin1: return String(localized: "UTF-8 via Latin 1") + } + } + + var sessionStatements: [String] { + switch self { + case .utf8: return [] + case .utf8ViaLatin1: return ["SET character_set_client = latin1"] + } + } + + func presentedText(_ text: String) -> String { + switch self { + case .utf8: return text + case .utf8ViaLatin1: return MySQLLatin1.repairingDoubleEncodedUTF8(text) + } + } +} diff --git a/Plugins/MySQLDriverPlugin/MySQLLatin1.swift b/Plugins/MySQLDriverPlugin/MySQLLatin1.swift new file mode 100644 index 0000000000..5fd582f285 --- /dev/null +++ b/Plugins/MySQLDriverPlugin/MySQLLatin1.swift @@ -0,0 +1,64 @@ +// +// MySQLLatin1.swift +// MySQLDriverPlugin +// + +import Foundation + +internal enum MySQLLatin1 { + private static let windowsRangeScalars: [UInt32] = [ + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, + 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178 + ] + + private static let windowsRange = 0x80...0x9F + + static let scalarsByByte: [Unicode.Scalar] = (0...255).map { byte in + guard windowsRange.contains(byte), + let scalar = Unicode.Scalar(windowsRangeScalars[byte - windowsRange.lowerBound]) else { + return Unicode.Scalar(UInt8(byte)) + } + return scalar + } + + private static let bytesByScalar: [UInt32: UInt8] = Dictionary( + uniqueKeysWithValues: scalarsByByte.enumerated().map { ($0.element.value, UInt8($0.offset)) } + ) + + static func decode(_ bytes: UnsafeRawBufferPointer) -> String { + guard bytes.contains(where: { $0 >= 0x80 }) else { + return MySQLCharacterSet.decodeUTF8ReplacingInvalid(bytes) + } + var scalars = String.UnicodeScalarView() + scalars.reserveCapacity(bytes.count) + for byte in bytes { + scalars.append(scalarsByByte[Int(byte)]) + } + return String(scalars) + } + + static func decode(_ bytes: [UInt8]) -> String { + bytes.withUnsafeBytes { decode($0) } + } + + static func bytes(representing text: String) -> [UInt8]? { + var bytes: [UInt8] = [] + bytes.reserveCapacity(text.utf8.count) + for scalar in text.unicodeScalars { + guard let byte = bytesByScalar[scalar.value] else { return nil } + bytes.append(byte) + } + return bytes + } + + static func repairingDoubleEncodedUTF8(_ text: String) -> String { + guard text.utf8.contains(where: { $0 >= 0x80 }) else { return text } + guard let original = bytes(representing: text), + let repaired = String(bytes: original, encoding: .utf8) else { + return text + } + return repaired + } +} diff --git a/Plugins/MySQLDriverPlugin/MySQLPlugin.swift b/Plugins/MySQLDriverPlugin/MySQLPlugin.swift index 33f95710de..bdba8d1fca 100644 --- a/Plugins/MySQLDriverPlugin/MySQLPlugin.swift +++ b/Plugins/MySQLDriverPlugin/MySQLPlugin.swift @@ -30,7 +30,8 @@ final class MySQLPlugin: NSObject, TableProPlugin, DriverPlugin { defaultValue: "0", fieldType: .stepper(range: ConnectionField.IntRange(0...240)), section: .advanced - ) + ), + MySQLConnectionEncoding.connectionField ] static let additionalDatabaseTypeIds: [String] = ["MariaDB", "TiDB", "Databend"] diff --git a/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift b/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift index 97eaf4ba58..2d15ac1406 100644 --- a/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift +++ b/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift @@ -124,7 +124,10 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { database: _activeDatabase, sslConfig: sslConfig, enableCleartextPlugin: config.additionalFields["enableCleartextPlugin"] == "true", - queryTimeoutSeconds: config.additionalFields["queryTimeoutSeconds"].flatMap { Int($0) } ?? 0 + queryTimeoutSeconds: config.additionalFields["queryTimeoutSeconds"].flatMap { Int($0) } ?? 0, + connectionEncoding: MySQLConnectionEncoding( + fieldValue: config.additionalFields[MySQLConnectionEncoding.fieldId] + ) ) try await conn.connect() diff --git a/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index c979af827a..3326feda18 100644 --- a/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -14,21 +14,6 @@ import TableProPluginKit private let logger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "LibPQPluginConnection") -// MARK: - Error Types - -struct LibPQPluginError: Error { - let message: String - let sqlState: String? - let detail: String? - - static let notConnected = LibPQPluginError( - message: String(localized: "Not connected to database"), sqlState: nil, detail: nil) - static let connectionFailed = LibPQPluginError( - message: String(localized: "Failed to establish connection"), sqlState: nil, detail: nil) - static let connectionTimedOut = LibPQPluginError( - message: String(localized: "Timed out while connecting to the server"), sqlState: nil, detail: nil) -} - // MARK: - Query Result struct LibPQPluginQueryResult { @@ -1243,22 +1228,12 @@ final class LibPQPluginConnection: @unchecked Sendable { private func getResultError(from result: OpaquePointer) -> LibPQPluginError { var message = "Unknown error" - var sqlState: String? - var detail: String? - if let msgPtr = PQresultErrorMessage(result) { message = String(cString: msgPtr).trimmingCharacters(in: .whitespacesAndNewlines) } - - if let statePtr = PQresultErrorField(result, Int32(80)) { - sqlState = String(cString: statePtr) - } - - if let detailPtr = PQresultErrorField(result, Int32(68)) { - detail = String(cString: detailPtr) + return LibPQPluginError(message: message) { field in + PQresultErrorField(result, field).map { String(cString: $0) } } - - return LibPQPluginError(message: message, sqlState: sqlState, detail: detail) } private func getAffectedRows(from result: OpaquePointer) -> Int { @@ -1275,11 +1250,3 @@ final class LibPQPluginConnection: @unchecked Sendable { return nil } } - -// MARK: - PluginDriverError Conformance - -extension LibPQPluginError: PluginDriverError { - var pluginErrorMessage: String { message } - var pluginSqlState: String? { sqlState } - var pluginErrorDetail: String? { detail } -} diff --git a/Plugins/PostgreSQLDriverPlugin/LibPQPluginError.swift b/Plugins/PostgreSQLDriverPlugin/LibPQPluginError.swift new file mode 100644 index 0000000000..e2ad6aa6c1 --- /dev/null +++ b/Plugins/PostgreSQLDriverPlugin/LibPQPluginError.swift @@ -0,0 +1,34 @@ +import Foundation +import TableProPluginKit + +struct LibPQPluginError: Error { + let message: String + let sqlState: String? + let detail: String? + + static let notConnected = LibPQPluginError( + message: String(localized: "Not connected to database"), sqlState: nil, detail: nil) + static let connectionFailed = LibPQPluginError( + message: String(localized: "Failed to establish connection"), sqlState: nil, detail: nil) + static let connectionTimedOut = LibPQPluginError( + message: String(localized: "Timed out while connecting to the server"), sqlState: nil, detail: nil) +} + +internal extension LibPQPluginError { + private static let sqlStateField = Int32(UInt8(ascii: "C")) + private static let detailField = Int32(UInt8(ascii: "D")) + + init(message: String, readingResultField readField: (Int32) -> String?) { + self.init( + message: message, + sqlState: readField(Self.sqlStateField), + detail: readField(Self.detailField) + ) + } +} + +extension LibPQPluginError: PluginDriverError { + var pluginErrorMessage: String { message } + var pluginSqlState: String? { sqlState } + var pluginErrorDetail: String? { detail } +} diff --git a/Plugins/SQLExportPlugin/SQLExportEncodingDeclaration.swift b/Plugins/SQLExportPlugin/SQLExportEncodingDeclaration.swift new file mode 100644 index 0000000000..2f5dc785fa --- /dev/null +++ b/Plugins/SQLExportPlugin/SQLExportEncodingDeclaration.swift @@ -0,0 +1,42 @@ +// +// SQLExportEncodingDeclaration.swift +// SQLExportPlugin +// + +import Foundation +import TableProPluginKit + +internal struct SQLExportEncodingDeclaration: Equatable { + static let empty = SQLExportEncodingDeclaration(prologue: "", epilogue: "") + + let prologue: String + let epilogue: String + + static func forDialect(_ dialect: SqlDialect) -> SQLExportEncodingDeclaration { + switch dialect { + case .mysql: + return mysql + default: + return .empty + } + } + + private static let mysql = SQLExportEncodingDeclaration( + prologue: """ + /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; + /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; + /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; + /*!40101 SET NAMES utf8 */; + /*!50503 SET NAMES utf8mb4 */; + + + """, + epilogue: """ + + /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; + /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; + /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; + + """ + ) +} diff --git a/Plugins/SQLExportPlugin/SQLExportFileWriter.swift b/Plugins/SQLExportPlugin/SQLExportFileWriter.swift index a0673e562a..2afa774923 100644 --- a/Plugins/SQLExportPlugin/SQLExportFileWriter.swift +++ b/Plugins/SQLExportPlugin/SQLExportFileWriter.swift @@ -27,20 +27,33 @@ internal final class SQLExportFileWriter { private let destination: URL private let splitSizeBytes: Int + private let encodingDeclaration: SQLExportEncodingDeclaration private var handle: FileHandle private var tempURL: URL private var bytesInCurrentPart = 0 + private var currentPartHasStatements = false private var partIndex = 1 private var pending: [(temp: URL, final: URL)] = [] private var isCommitted = false - internal init(destination: URL, splitSizeMegabytes: Int) throws { + internal init( + destination: URL, + splitSizeMegabytes: Int, + encodingDeclaration: SQLExportEncodingDeclaration = .empty + ) throws { self.destination = destination self.splitSizeBytes = max(0, splitSizeMegabytes) * 1_024 * 1_024 + self.encodingDeclaration = encodingDeclaration let (handle, tempURL) = try PluginExportUtilities.beginAtomicWrite(for: destination) self.handle = handle self.tempURL = tempURL + do { + try writeRaw(encodingDeclaration.prologue) + } catch { + rollback() + throw error + } } /// True once a second part exists, so the caller can report the split rather than leaving the @@ -51,17 +64,20 @@ internal final class SQLExportFileWriter { internal func write(_ text: String) throws { let data = try text.toUTF8Data() - if splitSizeBytes > 0, bytesInCurrentPart > 0, bytesInCurrentPart + data.count > splitSizeBytes { + let partSize = bytesInCurrentPart + data.count + encodingDeclaration.epilogue.utf8.count + if splitSizeBytes > 0, currentPartHasStatements, partSize > splitSizeBytes { try rotate() } try handle.write(contentsOf: data) bytesInCurrentPart += data.count + currentPartHasStatements = true } /// Publishes every part and returns where they landed. An unsplit export keeps the name the /// user chose; a split one numbers all of its parts, so no part silently claims that name. @discardableResult internal func commit() throws -> [URL] { + try writeRaw(encodingDeclaration.epilogue) try handle.close() let finalURL = didSplit ? Self.partURL(for: destination, part: partIndex) : destination pending.append((tempURL, finalURL)) @@ -88,6 +104,7 @@ internal final class SQLExportFileWriter { internal var currentFileURL: URL { tempURL } private func rotate() throws { + try writeRaw(encodingDeclaration.epilogue) try handle.close() pending.append((tempURL, Self.partURL(for: destination, part: partIndex))) partIndex += 1 @@ -95,5 +112,14 @@ internal final class SQLExportFileWriter { handle = nextHandle tempURL = nextTemp bytesInCurrentPart = 0 + currentPartHasStatements = false + try writeRaw(encodingDeclaration.prologue) + } + + private func writeRaw(_ text: String) throws { + guard !text.isEmpty else { return } + let data = try text.toUTF8Data() + try handle.write(contentsOf: data) + bytesInCurrentPart += data.count } } diff --git a/Plugins/SQLExportPlugin/SQLExportPlugin.swift b/Plugins/SQLExportPlugin/SQLExportPlugin.swift index 9feba46ef5..ab5b969155 100644 --- a/Plugins/SQLExportPlugin/SQLExportPlugin.swift +++ b/Plugins/SQLExportPlugin/SQLExportPlugin.swift @@ -139,7 +139,11 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send metadataWarnings.append(String(localized: "A compressed export is written as one file, so the split size was not applied.")) } - let writer = try SQLExportFileWriter(destination: actualDestination, splitSizeMegabytes: splitSize) + let writer = try SQLExportFileWriter( + destination: actualDestination, + splitSizeMegabytes: splitSize, + encodingDeclaration: .forDialect(SqlDialect.from(databaseTypeId: dataSource.databaseTypeId)) + ) var committed = false defer { if !committed { writer.rollback() } @@ -478,10 +482,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw SQLExportObjectError.emptyDefinition } - try writer.write(ddl) - if !ddl.hasSuffix(";") { - try writer.write(";") - } + try writer.write(ddl.hasSuffix(";") ? ddl : ddl + ";") try writer.write("\n\n") } catch { ddlFailures.append(sanitizedName) @@ -524,10 +525,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw SQLExportObjectError.emptyDefinition } - try writer.write(ddl) - if !ddl.hasSuffix(";") { - try writer.write(";") - } + try writer.write(ddl.hasSuffix(";") ? ddl : ddl + ";") try writer.write("\n\n") } catch { ddlFailures.append(sanitizedName) diff --git a/Plugins/WeaviateDriverPlugin/Info.plist b/Plugins/WeaviateDriverPlugin/Info.plist new file mode 100644 index 0000000000..48893d9d17 --- /dev/null +++ b/Plugins/WeaviateDriverPlugin/Info.plist @@ -0,0 +1,14 @@ + + + + + TableProMinAppVersion + 0.73.0 + TableProPluginKitVersion + 25 + TableProProvidesDatabaseTypeIds + + Weaviate + + + diff --git a/Plugins/WeaviateDriverPlugin/WeaviatePlugin.swift b/Plugins/WeaviateDriverPlugin/WeaviatePlugin.swift new file mode 100644 index 0000000000..d533f77b68 --- /dev/null +++ b/Plugins/WeaviateDriverPlugin/WeaviatePlugin.swift @@ -0,0 +1,146 @@ +import Foundation +import TableProPluginKit +import TableProWeaviateCore + +final class WeaviatePlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Weaviate Driver" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Weaviate support over the REST API with a GraphQL console" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "Weaviate" + static let databaseDisplayName = "Weaviate" + static let iconName = "weaviate-icon" + static let defaultPort = WeaviateConnectionSettings.defaultPort + static let isDownloadable = true + + static let navigationModel: NavigationModel = .standard + static let pathFieldRole: PathFieldRole = .database + static let requiresAuthentication = false + static let brandColorHex = "#01B0D3" + static let queryLanguageName = "GraphQL" + static let editorLanguage: EditorLanguage = .javascript + static let supportsForeignKeys = false + static let supportsSchemaEditing = false + static let supportsDatabaseSwitching = false + static let supportsImport = false + static let supportsExport = true + static let supportsSSH = false + static let supportsSSL = true + static let supportsReadOnlyMode = true + static let supportsForeignKeyDisable = false + static let supportsAddColumn = false + static let supportsModifyColumn = false + static let supportsDropColumn = false + static let supportsAddIndex = false + static let supportsDropIndex = false + static let supportsModifyPrimaryKey = false + static let databaseGroupingStrategy: GroupingStrategy = .flat + static let defaultGroupName = "default" + static let tableEntityName = "Collections" + static let containerEntityName = "Cluster" + static let immutableColumns: [String] = WeaviateSchema.immutableColumns + static let defaultPrimaryKeyColumn: String? = WeaviateSchema.uuidColumn + static let structureColumnFields: [StructureColumnField] = [.name, .type, .nullable] + static let sqlDialect: SQLDialectDescriptor? = nil + + static let columnTypesByCategory: [String: [String]] = weaviateColumnTypes + + static let additionalConnectionFields: [ConnectionField] = weaviateConnectionFields() + + static var statementCompletions: [CompletionEntry] { weaviateCompletions } + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + WeaviatePluginDriver(config: config) + } +} + +func weaviateConnectionFields() -> [ConnectionField] { + [ + ConnectionField( + id: WeaviateFieldID.authMethod, + label: String(localized: "Auth Method"), + defaultValue: WeaviateAuthMethod.none.rawValue, + fieldType: .dropdown(options: [ + .init(value: WeaviateAuthMethod.none.rawValue, label: "None"), + .init(value: WeaviateAuthMethod.apiKey.rawValue, label: "API Key") + ]), + section: .authentication + ), + ConnectionField( + id: WeaviateFieldID.apiKey, + label: String(localized: "API Key"), + placeholder: "Weaviate API key", + fieldType: .secure, + section: .authentication, + hidesPassword: true + ).withHidesUsername(true), + ConnectionField( + id: WeaviateFieldID.skipTLSVerify, + label: String(localized: "Skip TLS Verification"), + defaultValue: "false", + fieldType: .toggle, + section: .advanced + ) + ] +} + +let weaviateCompletions: [CompletionEntry] = [ + CompletionEntry( + label: "Get", + insertText: """ + { + Get { + Article(limit: 10) { + title + _additional { id distance } + } + } + } + """ + ), + CompletionEntry( + label: "Near text", + insertText: """ + { + Get { + Article( + nearText: { concepts: ["search term"] } + limit: 10 + ) { + title + _additional { id distance } + } + } + } + """ + ), + CompletionEntry( + label: "Hybrid", + insertText: """ + { + Get { + Article( + hybrid: { query: "search term", alpha: 0.5 } + limit: 10 + ) { + title + _additional { id score } + } + } + } + """ + ), + CompletionEntry(label: "GET /v1/schema", insertText: "GET /v1/schema"), + CompletionEntry(label: "GET /v1/meta", insertText: "GET /v1/meta"), + CompletionEntry(label: "GET /v1/objects", insertText: "GET /v1/objects?class=Article&limit=10") +] + +let weaviateColumnTypes: [String: [String]] = [ + "Text": ["text", "text[]", "string", "string[]", "uuid", "uuid[]"], + "Numeric": ["int", "int[]", "number", "number[]"], + "Boolean": ["boolean", "boolean[]"], + "Date": ["date", "date[]"], + "Structured": ["object", "object[]", "geoCoordinates", "phoneNumber"], + "Vector": ["vector"] +] diff --git a/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift b/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift new file mode 100644 index 0000000000..55632941d2 --- /dev/null +++ b/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift @@ -0,0 +1,202 @@ +import Foundation +import TableProPluginKit +import TableProWeaviateCore + +extension WeaviatePluginDriver { + func execute(query: String) async throws -> PluginQueryResult { + let started = Date() + let client = try requireClient() + let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + + if trimmed.lowercased() == "select 1" { + try await client.ping() + return PluginQueryResult( + columns: ["ok"], + columnTypeNames: ["int"], + rows: [[.text("1")]], + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } + + if WeaviateBrowseQuery.isTagged(trimmed) { + return try await executeSearch(trimmed, client: client, started: started) + } + if WeaviateWriteCodec.isTagged(trimmed) { + return try await executeWrite(trimmed, client: client, started: started) + } + if let console = WeaviateConsoleParser.parse(trimmed) { + return try await executeConsole(console, client: client, started: started) + } + if WeaviateGraphQL.looksLikeGraphQL(trimmed) { + return try await executeGraphQL(trimmed, client: client, started: started) + } + + throw WeaviateError.malformedResponse( + String(localized: "Enter a GraphQL query, or a request like GET /v1/schema.") + ) + } + + private func executeSearch( + _ query: String, + client: WeaviateClient, + started: Date + ) async throws -> PluginQueryResult { + guard let parsed = WeaviateBrowseQuery.parse(query) else { + throw WeaviateError.malformedResponse(String(localized: "Invalid browse request.")) + } + let collection = try await cachedCollection(parsed.collection) + let wantsVector = parsed.propertyNames.isEmpty + || parsed.propertyNames.contains(WeaviateSchema.vectorColumn) + let objects: [WeaviateObject] + if parsed.usesGraphQL { + let graphql = try WeaviateGraphQL.getQuery( + collection: parsed.collection, + properties: parsed.propertyNames, + limit: parsed.limit, + offset: parsed.offset, + sorts: parsed.sortableSorts, + filters: parsed.filters, + logicMode: parsed.logicMode, + schema: propertySchema(of: collection), + includeVector: wantsVector + ) + let response = try await client.graphql(graphql) + objects = WeaviateObjectCodec.objects(fromGraphQL: response.json as Any) + } else { + objects = try await client.objects( + collection: parsed.collection, + limit: parsed.limit, + offset: parsed.offset, + includeVector: wantsVector + ) + } + return render(objects: objects, collection: collection, columns: parsed.propertyNames, started: started) + } + + private func executeWrite( + _ statement: String, + client: WeaviateClient, + started: Date + ) async throws -> PluginQueryResult { + guard let request = WeaviateWriteCodec.decode(statement) else { + throw WeaviateError.malformedResponse(String(localized: "Invalid write request.")) + } + let response = try await client.execute(write: request) + let outcome: String + if let json = WeaviateJSON.dictionary(response.json), let id = json["id"] as? String { + outcome = id + } else if response.statusCode == 204 { + outcome = "deleted" + } else { + outcome = "ok" + } + return PluginQueryResult( + columns: ["result"], + columnTypeNames: ["text"], + rows: [[.text(outcome)]], + rowsAffected: 1, + executionTime: Date().timeIntervalSince(started) + ) + } + + private func executeConsole( + _ request: WeaviateConsoleRequest, + client: WeaviateClient, + started: Date + ) async throws -> PluginQueryResult { + if request.method == "POST", request.path.hasPrefix("/v1/graphql"), let body = request.body { + return try await executeGraphQL(body, client: client, started: started) + } + let response = try await client.execute(console: request) + if request.path.hasPrefix("/v1/objects"), let json = response.json { + let objects = WeaviateObject.parseList(json) + if !objects.isEmpty { + return await renderReturnedColumns(objects, started: started) + } + } + return renderJSON(response, started: started) + } + + private func executeGraphQL( + _ query: String, + client: WeaviateClient, + started: Date + ) async throws -> PluginQueryResult { + let response = try await client.graphql(query) + let objects = WeaviateObjectCodec.objects(fromGraphQL: response.json as Any) + if !objects.isEmpty { + return await renderReturnedColumns(objects, started: started) + } + return renderJSON(response, started: started) + } + + /// A console query selects its own fields, so the result shows what came back rather than every + /// column the collection has. That is also what carries `_additional { distance }` into the grid. + private func renderReturnedColumns(_ objects: [WeaviateObject], started: Date) async -> PluginQueryResult { + let collectionName = objects.first?.className ?? "" + let collection = (try? await cachedCollection(collectionName)) + ?? WeaviateCollection(name: collectionName, properties: []) + return render( + objects: objects, + collection: collection, + columns: returnedColumns(of: objects, collection: collection), + started: started + ) + } + + private func returnedColumns(of objects: [WeaviateObject], collection: WeaviateCollection) -> [String] { + let declared = collection.properties.map(\.name) + let returned = Set(objects.flatMap { $0.properties.keys }) + var columns: [String] = [] + if objects.contains(where: { !$0.uuid.isEmpty }) { + columns.append(WeaviateSchema.uuidColumn) + } + columns += declared.filter { returned.contains($0) } + columns += returned.subtracting(declared).sorted() + if objects.contains(where: { $0.vector != nil }) { + columns.append(WeaviateSchema.vectorColumn) + } + return columns.isEmpty ? [WeaviateSchema.uuidColumn] : columns + } + + private func render( + objects: [WeaviateObject], + collection: WeaviateCollection, + columns: [String], + started: Date + ) -> PluginQueryResult { + let resolved = columns.isEmpty + ? WeaviateSchema.columns(for: collection).map(\.name) + : columns + let rows = objects.map { object in + WeaviateObjectCodec.row(for: object, columns: resolved).map { value in + value.map(PluginCellValue.text) ?? .null + } + } + return PluginQueryResult( + columns: resolved, + columnTypeNames: resolved.map { typeName(for: $0, collection: collection) }, + rows: rows, + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } + + private func renderJSON(_ response: WeaviateHTTPResponse, started: Date) -> PluginQueryResult { + let pretty: String + if let json = response.json, JSONSerialization.isValidJSONObject(json), + let text = try? WeaviateJSON.text(json, pretty: true) { + pretty = text + } else { + pretty = response.text + } + return PluginQueryResult( + columns: ["response"], + columnTypeNames: ["json"], + rows: [[.text(pretty)]], + rowsAffected: 0, + executionTime: Date().timeIntervalSince(started) + ) + } +} diff --git a/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Metadata.swift b/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Metadata.swift new file mode 100644 index 0000000000..eb3f150c54 --- /dev/null +++ b/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Metadata.swift @@ -0,0 +1,215 @@ +import Foundation +import TableProPluginKit +import TableProWeaviateCore + +extension WeaviatePluginDriver { + func fetchDatabases() async throws -> [String] { + [WeaviatePlugin.defaultGroupName] + } + + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + let collections = try await requireClient().schema() + remember(collections) + return collections.map { PluginTableInfo(name: $0.name, type: "TABLE") } + } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + let collection = try await cachedCollection(table) + return WeaviateSchema.columns(for: collection).map { column in + PluginColumnInfo( + name: column.name, + dataType: column.type, + isNullable: !column.isPrimaryKey, + isPrimaryKey: column.isPrimaryKey + ) + } + } + + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { + [ + PluginIndexInfo( + name: WeaviateSchema.uuidColumn, + columns: [WeaviateSchema.uuidColumn], + isUnique: true, + isPrimary: true, + type: "PRIMARY KEY" + ) + ] + } + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + [] + } + + func fetchTableDDL(table: String, schema: String?) async throws -> String { + let collections = try await requireClient().schema() + guard let collection = collections.first(where: { $0.name == table }) else { + return "{}" + } + var payload: [String: Any] = [ + "class": collection.name, + "properties": collection.properties.map { ["name": $0.name, "dataType": [$0.dataType]] } + ] + if let vectorizer = collection.vectorizer { + payload["vectorizer"] = vectorizer + } + return (try? WeaviateJSON.text(payload, pretty: true)) ?? "{}" + } + + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + throw WeaviateError.configuration(String(localized: "Weaviate does not support views.")) + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table, engine: "Weaviate") + } + + func buildBrowseQuery( + table: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int + ) -> String? { + buildFilteredQuery( + table: table, + schema: nil, + queryFilters: [], + logicMode: "AND", + sortColumns: sortColumns, + columns: columns, + limit: limit, + offset: offset, + columnKinds: [:] + ) + } + + func buildFilteredQuery( + table: String, + schema: String?, + queryFilters filters: [PluginQueryFilter], + logicMode: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int, + columnKinds: [String: PluginColumnKind] + ) -> String? { + let sorts = sortColumns.compactMap { sort -> WeaviateSortSpec? in + guard sort.columnIndex >= 0, sort.columnIndex < columns.count else { return nil } + let column = columns[sort.columnIndex] + guard column != WeaviateSchema.vectorColumn else { return nil } + return WeaviateSortSpec(column: column, ascending: sort.ascending) + } + let specs = filters.map { filter in + WeaviateFilterSpec( + column: filter.column, + op: filter.op, + value: filter.value, + secondValue: filter.secondValue + ) + } + return WeaviateBrowseQuery.encode( + collection: table, + offset: offset, + limit: limit, + sorts: sorts, + filters: specs, + logicMode: logicMode, + propertyNames: columns + ) + } + + func generateStatements( + table: String, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) -> [(statement: String, parameters: [PluginCellValue])]? { + let collection = rememberedCollection(table) + let typeNames = columns.map { column in + typeName(for: column, collection: collection ?? WeaviateCollection(name: table, properties: [])) + } + let tracked = changes.compactMap { change -> WeaviateTrackedChange? in + mappedChange( + change, + columns: columns, + insertedRowData: insertedRowData, + deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices + ) + } + let batch = WeaviateStatementGenerator.generate( + collection: table, + columns: columns, + typeNames: typeNames, + changes: tracked + ) + for skipped in batch.skipped { + WeaviatePluginDriver.logger.warning( + "Skipped a \(skipped.kind.rawValue, privacy: .public) on \(table, privacy: .private): \(skipped.reason.rawValue, privacy: .public)" + ) + } + return batch.requests.map { (WeaviateWriteCodec.encode($0), []) } + } + + private func mappedChange( + _ change: PluginRowChange, + columns: [String], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set + ) -> WeaviateTrackedChange? { + switch change.type { + case .insert: + guard insertedRowIndices.contains(change.rowIndex) else { return nil } + var values: [String: String?] = [:] + if let row = insertedRowData[change.rowIndex] { + for (index, column) in columns.enumerated() where index < row.count { + guard let text = row[index].asText else { continue } + values[column] = text + } + } else { + for cell in change.cellChanges { + guard let text = cell.newValue.asText else { continue } + values[cell.columnName] = text + } + } + return WeaviateTrackedChange( + kind: .insert, + uuid: values[WeaviateSchema.uuidColumn] ?? nil, + values: values, + cellChanges: [] + ) + case .update: + let uuid = uuid(from: change, columns: columns) + let cells = change.cellChanges.map { + WeaviateCellChange(column: $0.columnName, newText: $0.newValue.asText) + } + return WeaviateTrackedChange(kind: .update, uuid: uuid, values: [:], cellChanges: cells) + case .delete: + guard deletedRowIndices.contains(change.rowIndex) else { return nil } + return WeaviateTrackedChange( + kind: .delete, + uuid: uuid(from: change, columns: columns), + values: [:], + cellChanges: [] + ) + } + } + + private func uuid(from change: PluginRowChange, columns: [String]) -> String? { + guard let original = change.originalRow, + let index = columns.firstIndex(of: WeaviateSchema.uuidColumn), + index < original.count + else { return nil } + return original[index].asText + } +} diff --git a/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver.swift b/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver.swift new file mode 100644 index 0000000000..7f869e31f1 --- /dev/null +++ b/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver.swift @@ -0,0 +1,122 @@ +import Foundation +import os +import TableProPluginKit +import TableProWeaviateCore + +internal final class WeaviatePluginDriver: PluginDatabaseDriver, @unchecked Sendable { + static let logger = Logger(subsystem: "com.TablePro", category: "WeaviatePluginDriver") + + private let config: DriverConnectionConfig + private let lock = NSLock() + private var client: WeaviateClient? + private var cachedCollections: [String: WeaviateCollection] = [:] + let queryTimeout = HttpQueryTimeoutBox() + + init(config: DriverConnectionConfig) { + self.config = config + } + + var serverVersion: String? { lock.withLock { client?.serverVersion } } + + var supportsTransactions: Bool { false } + + var capabilities: PluginCapabilities { [.cancelQuery] } + + var parameterStyle: ParameterStyle { .questionMark } + + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + + func connect() async throws { + let settings = try WeaviateConnectionSettings.parse( + host: config.host, + port: config.port, + usesTLS: config.ssl.isEnabled, + fields: config.additionalFields + ) + let skipTLS = settings.skipTLSVerify + || (config.ssl.isEnabled && !config.ssl.verifiesCertificate) + let timeout = queryTimeout + let transport = URLSessionWeaviateTransport( + resourceTimeout: HttpQueryTimeout.sessionResourceTimeout, + skipTLSVerify: skipTLS + ) + let client = WeaviateClient( + settings: settings, + transport: transport, + timeout: { timeout.requestTimeoutInterval } + ) + try await client.connect() + lock.withLock { + self.client = client + cachedCollections.removeAll() + } + } + + func disconnect() { + lock.withLock { + client?.cancelAll() + client = nil + cachedCollections.removeAll() + } + } + + func ping() async throws { + try await requireClient().ping() + } + + func cancelQuery() throws { + lock.withLock { client?.cancelAll() } + } + + func applyQueryTimeout(_ seconds: Int) async throws { + queryTimeout.set(serverTimeoutSeconds: seconds) + } + + func requireClient() throws -> WeaviateClient { + guard let client = lock.withLock({ client }) else { + throw WeaviateError.notConnected + } + return client + } + + func remember(_ collections: [WeaviateCollection]) { + lock.withLock { + for collection in collections { + cachedCollections[collection.name] = collection + } + } + } + + func rememberedCollection(_ name: String) -> WeaviateCollection? { + lock.withLock { cachedCollections[name] } + } + + func cachedCollection(_ name: String) async throws -> WeaviateCollection { + if let cached = rememberedCollection(name) { + return cached + } + let collections = try await requireClient().schema() + remember(collections) + return rememberedCollection(name) ?? WeaviateCollection(name: name, properties: []) + } + + func propertySchema(of collection: WeaviateCollection) -> [String: WeaviateProperty] { + Dictionary( + collection.properties.map { ($0.name, $0) }, + uniquingKeysWith: { first, _ in first } + ) + } + + func typeName(for column: String, collection: WeaviateCollection) -> String { + switch column { + case WeaviateSchema.uuidColumn: + return "uuid" + case WeaviateSchema.vectorColumn: + return "vector" + default: + return collection.properties.first { $0.name == column }?.dataType ?? "text" + } + } +} diff --git a/TablePro/Assets.xcassets/weaviate-icon.imageset/Contents.json b/TablePro/Assets.xcassets/weaviate-icon.imageset/Contents.json new file mode 100644 index 0000000000..fcfa2ae760 --- /dev/null +++ b/TablePro/Assets.xcassets/weaviate-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "weaviate.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/TablePro/Assets.xcassets/weaviate-icon.imageset/weaviate.svg b/TablePro/Assets.xcassets/weaviate-icon.imageset/weaviate.svg new file mode 100644 index 0000000000..fafccd810f --- /dev/null +++ b/TablePro/Assets.xcassets/weaviate-icon.imageset/weaviate.svg @@ -0,0 +1 @@ +Weaviate diff --git a/TablePro/Core/DataGrid/DataGridDisplayState.swift b/TablePro/Core/DataGrid/DataGridDisplayState.swift index ddb718ca36..b4f23d2920 100644 --- a/TablePro/Core/DataGrid/DataGridDisplayState.swift +++ b/TablePro/Core/DataGrid/DataGridDisplayState.swift @@ -50,4 +50,5 @@ final class DataGridDisplayState { /// reports a schema and a format change and clears the text it was just handed. var identitySchema: ColumnIdentitySchema? var displayFormats: [ValueDisplayFormat?]? + var highlightRuleSetKey: HighlightRuleSet.Key? } diff --git a/TablePro/Core/DataGrid/RowDisplayBox.swift b/TablePro/Core/DataGrid/RowDisplayBox.swift index b4f475587c..99f39e1bd2 100644 --- a/TablePro/Core/DataGrid/RowDisplayBox.swift +++ b/TablePro/Core/DataGrid/RowDisplayBox.swift @@ -25,6 +25,7 @@ final class RowDisplayCache { } private var storage: [RowID: Entry] = [:] + private var highlights: [RowID: RowHighlight] = [:] private var insertionOrder: [RowID] = [] private var insertionHead: Int = 0 private var totalCost: Int = 0 @@ -52,8 +53,28 @@ final class RowDisplayCache { evictIfNeeded() } + func highlight(forID id: RowID) -> RowHighlight? { + highlights[id] + } + + func setHighlight(_ highlight: RowHighlight, forID id: RowID) { + if highlights.count >= countLimit { + highlights.removeAll(keepingCapacity: true) + } + highlights[id] = highlight + } + + func clearHighlight(forID id: RowID) { + highlights.removeValue(forKey: id) + } + + func clearHighlights() { + highlights.removeAll(keepingCapacity: true) + } + func removeAll() { storage.removeAll(keepingCapacity: true) + highlights.removeAll(keepingCapacity: true) insertionOrder.removeAll(keepingCapacity: true) insertionHead = 0 totalCost = 0 @@ -64,6 +85,7 @@ final class RowDisplayCache { /// whose content changed in place keeps its id and would otherwise be served /// its pre-edit text. func clearValues(forID id: RowID) { + highlights.removeValue(forKey: id) guard let existing = storage[id] else { return } totalCost -= existing.cost for index in existing.box.values.indices { diff --git a/TablePro/Core/Menu/ViewMenuBuilder.swift b/TablePro/Core/Menu/ViewMenuBuilder.swift index b6545cdf8c..16a2d8d11a 100644 --- a/TablePro/Core/Menu/ViewMenuBuilder.swift +++ b/TablePro/Core/Menu/ViewMenuBuilder.swift @@ -71,6 +71,10 @@ enum ViewMenuBuilder { shortcut: .toggleFilters, keyboard: keyboard ), + MenuItemFactory.item( + String(localized: "Highlight Rules…"), + action: #selector(MainSplitViewController.showHighlightRules(_:)) + ), MenuItemFactory.item( String(localized: "Show Query History"), action: #selector(MainSplitViewController.toggleQueryHistory(_:)), diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift index 2373cc1bbd..558d6ea777 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift @@ -223,6 +223,16 @@ extension PluginMetadataRegistry { section: .advanced ) + let mysqlEncodingField = ConnectionField( + id: "mysqlConnectionEncoding", + label: String(localized: "Encoding"), + fieldType: .dropdown(options: [ + .init(value: "", label: "UTF-8"), + .init(value: "utf8ViaLatin1", label: String(localized: "UTF-8 via Latin 1")) + ]), + section: .advanced + ) + let defaults: [(typeId: String, snapshot: PluginMetadataSnapshot)] = [ ("MySQL", PluginMetadataSnapshot( displayName: "MySQL", iconName: "mysql-icon", defaultPort: 3_306, @@ -282,7 +292,7 @@ extension PluginMetadataRegistry { columnTypesByCategory: mysqlColumnTypes ), connection: PluginMetadataSnapshot.ConnectionConfig( - additionalConnectionFields: awsIAMFields + [mysqlIdleReleaseField], + additionalConnectionFields: awsIAMFields + [mysqlIdleReleaseField, mysqlEncodingField], category: .relational, tagline: String(localized: "Most popular open-source SQL database"), defaultUnixSocketPath: "/var/run/mysqld/mysqld.sock" @@ -346,7 +356,7 @@ extension PluginMetadataRegistry { columnTypesByCategory: mysqlColumnTypes ), connection: PluginMetadataSnapshot.ConnectionConfig( - additionalConnectionFields: awsIAMFields + [mysqlIdleReleaseField], + additionalConnectionFields: awsIAMFields + [mysqlIdleReleaseField, mysqlEncodingField], category: .relational, tagline: String(localized: "Open-source fork of MySQL"), defaultUnixSocketPath: "/var/run/mysqld/mysqld.sock" diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index d8f4b855e9..f5b3ce0476 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -1141,6 +1141,7 @@ extension PluginMetadataRegistry { + duckdbPluginDefaults(dialect: duckdbDialect, columnTypes: duckdbColumnTypes) + cloudPluginDefaults() + elasticsearchPluginDefaults() + surrealDBPluginDefaults() + kafkaPluginDefaults() + typesensePluginDefaults() + r2SQLPluginDefaults() + + weaviatePluginDefaults() } // swiftlint:enable function_body_length } diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+WeaviateDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+WeaviateDefaults.swift new file mode 100644 index 0000000000..4ca4b1c04c --- /dev/null +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+WeaviateDefaults.swift @@ -0,0 +1,156 @@ +import Foundation +import TableProPluginKit + +extension PluginMetadataRegistry { + func weaviatePluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { + [ + ("Weaviate", PluginMetadataSnapshot( + displayName: "Weaviate", iconName: "weaviate-icon", defaultPort: 8_080, + requiresAuthentication: false, supportsForeignKeys: false, supportsSchemaEditing: false, + isDownloadable: true, primaryUrlScheme: "", parameterStyle: .questionMark, + navigationModel: .standard, explainVariants: [], pathFieldRole: .database, + supportsHealthMonitor: true, urlSchemes: [], postConnectActions: [], + brandColorHex: "#01B0D3", + queryLanguageName: "GraphQL", editorLanguage: .javascript, + connectionMode: .network, supportsDatabaseSwitching: false, + capabilities: PluginMetadataSnapshot.CapabilityFlags( + supportsSchemaSwitching: false, + supportsImport: false, + supportsExport: true, + supportsSSH: false, + supportsSSL: true, + supportsCascadeDrop: false, + supportsForeignKeyDisable: false, + supportsReadOnlyMode: true, + supportsQueryProgress: false, + requiresReconnectForDatabaseSwitch: false, + supportsDropDatabase: false, + supportsAddColumn: false, + supportsModifyColumn: false, + supportsDropColumn: false, + supportsAddIndex: false, + supportsDropIndex: false, + supportsModifyPrimaryKey: false, + supportsOpportunisticTLS: false, + supportsCloudflareTunnel: false, + supportsPrincipalConnectionLimit: false + ), + schema: PluginMetadataSnapshot.SchemaInfo( + defaultSchemaName: "", + defaultGroupName: "default", + tableEntityName: "Collections", + containerEntityName: "Cluster", + defaultPrimaryKeyColumn: "uuid", + immutableColumns: ["uuid", "vector"], + systemDatabaseNames: [], + systemSchemaNames: [], + fileExtensions: [], + databaseGroupingStrategy: .flat, + structureColumnFields: [.name, .type, .nullable] + ), + editor: PluginMetadataSnapshot.EditorConfig( + sqlDialect: nil, + statementCompletions: weaviateCompletions, + columnTypesByCategory: weaviateColumnTypes + ), + connection: PluginMetadataSnapshot.ConnectionConfig( + additionalConnectionFields: weaviateConnectionFields(), + category: .document, + tagline: String(localized: "Open-source vector database"), + hidesBuiltInPassword: true, + hidesBuiltInDatabase: true + ) + )) + ] + } +} + +private let weaviateCompletions: [CompletionEntry] = [ + CompletionEntry( + label: "Get", + insertText: """ + { + Get { + Article(limit: 10) { + title + _additional { id distance } + } + } + } + """ + ), + CompletionEntry( + label: "Near text", + insertText: """ + { + Get { + Article( + nearText: { concepts: ["search term"] } + limit: 10 + ) { + title + _additional { id distance } + } + } + } + """ + ), + CompletionEntry( + label: "Hybrid", + insertText: """ + { + Get { + Article( + hybrid: { query: "search term", alpha: 0.5 } + limit: 10 + ) { + title + _additional { id score } + } + } + } + """ + ), + CompletionEntry(label: "GET /v1/schema", insertText: "GET /v1/schema"), + CompletionEntry(label: "GET /v1/meta", insertText: "GET /v1/meta"), + CompletionEntry(label: "GET /v1/objects", insertText: "GET /v1/objects?class=Article&limit=10") +] + +private let weaviateColumnTypes: [String: [String]] = [ + "Text": ["text", "text[]", "string", "string[]", "uuid", "uuid[]"], + "Numeric": ["int", "int[]", "number", "number[]"], + "Boolean": ["boolean", "boolean[]"], + "Date": ["date", "date[]"], + "Structured": ["object", "object[]", "geoCoordinates", "phoneNumber"], + "Vector": ["vector"] +] + +func weaviateConnectionFields() -> [ConnectionField] { + [ + ConnectionField( + id: "wvAuthMethod", + label: String(localized: "Auth Method"), + defaultValue: "none", + fieldType: .dropdown(options: [ + .init(value: "none", label: "None"), + .init(value: "apiKey", label: "API Key") + ]), + section: .authentication + ), + ConnectionField( + id: "wvApiKey", + label: String(localized: "API Key"), + placeholder: "Weaviate API key", + fieldType: .secure, + section: .authentication, + hidesPassword: true + ).withHidesUsername(true), + ConnectionField( + id: "wvSkipTLSVerify", + label: String(localized: "Skip TLS Verification"), + defaultValue: "false", + fieldType: .toggle, + section: .advanced + ) + ] +} diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/TablePro/Core/Plugins/PluginMetadataRegistry.swift index d6c246abaa..d11638b29c 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -686,7 +686,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { return .analytical case "Spanner": return .relational - case "MongoDB", "Elasticsearch", "SurrealDB", "Typesense": + case "MongoDB", "Elasticsearch", "SurrealDB", "Typesense", "Weaviate": return .document case "Redis": return .keyValue @@ -728,6 +728,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { case "SurrealDB": return String(localized: "Multi-model database with SurrealQL") case "Kafka": return String(localized: "Event streaming platform") case "Typesense": return String(localized: "Typo-tolerant open-source search engine") + case "Weaviate": return String(localized: "Open-source vector database") default: return "" } } diff --git a/TablePro/Core/Services/Highlight/HighlightCondition.swift b/TablePro/Core/Services/Highlight/HighlightCondition.swift new file mode 100644 index 0000000000..2f16f6e85c --- /dev/null +++ b/TablePro/Core/Services/Highlight/HighlightCondition.swift @@ -0,0 +1,275 @@ +// +// HighlightCondition.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +struct HighlightCondition { + static let searchLimit = 10_000 + + private enum ValueKind { + case numeric + case boolean + case text + } + + private struct Operand { + let text: String + let number: Decimal? + let boolean: Bool? + let isNullLiteral: Bool + + init(_ raw: String, allowsNullLiteral: Bool) { + let trimmed = raw.trimmingCharacters(in: .whitespaces) + text = raw + number = HighlightCondition.number(from: trimmed) + boolean = HighlightCondition.boolean(from: trimmed) + isNullLiteral = allowsNullLiteral && HighlightCondition.isNullKeyword(trimmed) + } + } + + private let filterOperator: FilterOperator + private let valueKind: ValueKind + private let comparesCaseInsensitively: Bool + private let supportsEmptyString: Bool + private let operand: Operand + private let secondOperand: Operand + private let listOperands: [Operand] + private let regex: NSRegularExpression? + + init(rule: HighlightRule, columnType: ColumnType?) { + filterOperator = rule.filterOperator + valueKind = Self.valueKind(for: columnType) + comparesCaseInsensitively = rule.filterOperator.supportsCaseSensitivity && !rule.isCaseSensitive + supportsEmptyString = ColumnTypeSQLQuoting.supportsEmptyStringComparison(columnType) + + let allowsNullLiteral = Self.allowsNullLiteral(for: columnType) + operand = Operand(rule.value, allowsNullLiteral: allowsNullLiteral) + secondOperand = Operand(rule.secondValue ?? "", allowsNullLiteral: allowsNullLiteral) + listOperands = rule.filterOperator == .inList || rule.filterOperator == .notInList + ? Self.listItems(rule.value).map { Operand($0, allowsNullLiteral: allowsNullLiteral) } + : [] + regex = rule.filterOperator == .regex + ? Self.regularExpression(rule.value, ignoresCase: comparesCaseInsensitively) + : nil + } + + func matches(_ value: PluginCellValue) -> Bool { + switch value { + case .null: + return matchesNull() + case .bytes: + return matchesBinary() + case .text(let text): + return matches(text: text) + } + } + + private func matchesNull() -> Bool { + switch filterOperator { + case .isNull, .isEmpty: + return true + case .equal: + return operand.isNullLiteral + case .inList: + return listOperands.contains { $0.isNullLiteral } + case .notEqual, .contains, .notContains, .startsWith, .endsWith, .greaterThan, .greaterOrEqual, + .lessThan, .lessOrEqual, .isNotNull, .isNotEmpty, .notInList, .between, .regex: + return false + } + } + + private func matchesBinary() -> Bool { + switch filterOperator { + case .isNotNull, .isNotEmpty: + return true + case .isNull, .isEmpty, .equal, .notEqual, .contains, .notContains, .startsWith, .endsWith, + .greaterThan, .greaterOrEqual, .lessThan, .lessOrEqual, .inList, .notInList, .between, .regex: + return false + } + } + + private func matches(text: String) -> Bool { + switch filterOperator { + case .equal: + return !operand.isNullLiteral && order(text, against: operand) == .orderedSame + case .notEqual: + return operand.isNullLiteral || order(text, against: operand) != .orderedSame + case .contains: + return contains(text) + case .notContains: + return !contains(text) + case .startsWith: + return hasAffix(text, anchoredAtEnd: false) + case .endsWith: + return hasAffix(text, anchoredAtEnd: true) + case .greaterThan: + return !operand.isNullLiteral && order(text, against: operand) == .orderedDescending + case .greaterOrEqual: + return !operand.isNullLiteral && order(text, against: operand) != .orderedAscending + case .lessThan: + return !operand.isNullLiteral && order(text, against: operand) == .orderedAscending + case .lessOrEqual: + return !operand.isNullLiteral && order(text, against: operand) != .orderedDescending + case .isNull: + return false + case .isNotNull: + return true + case .isEmpty: + return supportsEmptyString && text.isEmpty + case .isNotEmpty: + return !supportsEmptyString || !text.isEmpty + case .inList: + return listOperands.contains { !$0.isNullLiteral && order(text, against: $0) == .orderedSame } + case .notInList: + let values = listOperands.filter { !$0.isNullLiteral } + return !values.isEmpty && !values.contains { order(text, against: $0) == .orderedSame } + case .between: + return order(text, against: operand) != .orderedAscending + && order(text, against: secondOperand) != .orderedDescending + case .regex: + return matchesRegex(text) + } + } + + private func order(_ text: String, against operand: Operand) -> ComparisonResult { + if prefersNumbers, let lhs = Self.number(from: text), let rhs = operand.number { + return Self.compare(lhs, rhs) + } + if prefersBooleans, let lhs = Self.boolean(from: text), let rhs = operand.boolean { + return Self.compare(lhs ? 1 : 0, rhs ? 1 : 0) + } + return text.compare(operand.text, options: comparesCaseInsensitively ? [.caseInsensitive] : [.literal]) + } + + private var prefersNumbers: Bool { + switch valueKind { + case .numeric: + return true + case .boolean: + return false + case .text: + return isOrderingOperator + } + } + + private var prefersBooleans: Bool { + switch valueKind { + case .numeric, .boolean: + return true + case .text: + return false + } + } + + private var isOrderingOperator: Bool { + switch filterOperator { + case .greaterThan, .greaterOrEqual, .lessThan, .lessOrEqual, .between: + return true + case .equal, .notEqual, .contains, .notContains, .startsWith, .endsWith, .isNull, .isNotNull, + .isEmpty, .isNotEmpty, .inList, .notInList, .regex: + return false + } + } + + private var searchOptions: String.CompareOptions { + comparesCaseInsensitively ? [.caseInsensitive] : [.literal] + } + + private func contains(_ text: String) -> Bool { + guard !operand.text.isEmpty else { return true } + return Self.searchable(text).range(of: operand.text, options: searchOptions) != nil + } + + private func hasAffix(_ text: String, anchoredAtEnd: Bool) -> Bool { + guard !operand.text.isEmpty else { return true } + let options = searchOptions.union(anchoredAtEnd ? [.anchored, .backwards] : [.anchored]) + return text.range(of: operand.text, options: options) != nil + } + + private func matchesRegex(_ text: String) -> Bool { + guard let regex else { return false } + let searchable = Self.searchable(text) as NSString + return regex.firstMatch( + in: searchable as String, + options: [], + range: NSRange(location: 0, length: searchable.length) + ) != nil + } + + private static func searchable(_ text: String) -> String { + let source = text as NSString + guard source.length > searchLimit else { return text } + let cut = source.rangeOfComposedCharacterSequence(at: searchLimit).location + return source.substring(to: cut) + } + + private static func regularExpression(_ pattern: String, ignoresCase: Bool) -> NSRegularExpression? { + guard !pattern.isEmpty, (pattern as NSString).length <= searchLimit else { return nil } + return try? NSRegularExpression(pattern: pattern, options: ignoresCase ? [.caseInsensitive] : []) + } + + private static func valueKind(for columnType: ColumnType?) -> ValueKind { + switch columnType { + case .integer, .decimal: + return .numeric + case .boolean: + return .boolean + case .text, .date, .timestamp, .datetime, .blob, .json, .enumType, .set, .spatial, .array, .none: + return .text + } + } + + private static func listItems(_ input: String) -> [String] { + input.split(separator: ",", omittingEmptySubsequences: true).compactMap { + let trimmed = $0.trimmingCharacters(in: .whitespaces) + return trimmed.isEmpty ? nil : trimmed + } + } + + static func readsAsNullLiteral(_ text: String, columnType: ColumnType?) -> Bool { + allowsNullLiteral(for: columnType) && isNullKeyword(text.trimmingCharacters(in: .whitespaces)) + } + + private static func allowsNullLiteral(for columnType: ColumnType?) -> Bool { + !ColumnTypeSQLQuoting.isKnownTextLike(columnType) + } + + private static func isNullKeyword(_ text: String) -> Bool { + text.caseInsensitiveCompare("NULL") == .orderedSame + } + + static func number(from text: String) -> Decimal? { + let trimmed = text.trimmingCharacters(in: .whitespaces) + guard PluginNumericLiteral.isValid(trimmed) else { return nil } + return Decimal(string: trimmed, locale: Locale(identifier: "en_US_POSIX")) + } + + static func boolean(from text: String) -> Bool? { + let trimmed = text.trimmingCharacters(in: .whitespaces) + switch PluginSQLLiteral.booleanSynonym(for: trimmed) { + case .isTrue: + return true + case .isFalse: + return false + default: + break + } + switch trimmed.lowercased() { + case "t": + return true + case "f": + return false + default: + return nil + } + } + + private static func compare(_ lhs: Value, _ rhs: Value) -> ComparisonResult { + if lhs < rhs { return .orderedAscending } + if lhs > rhs { return .orderedDescending } + return .orderedSame + } +} diff --git a/TablePro/Core/Services/Highlight/HighlightRuleSet.swift b/TablePro/Core/Services/Highlight/HighlightRuleSet.swift new file mode 100644 index 0000000000..15cb302910 --- /dev/null +++ b/TablePro/Core/Services/Highlight/HighlightRuleSet.swift @@ -0,0 +1,93 @@ +// +// HighlightRuleSet.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +struct HighlightRuleSet { + struct Key: Equatable { + let rules: [HighlightRule] + let columns: [String] + let columnTypes: [ColumnType] + } + + private struct CompiledRule { + let rule: HighlightRule + let column: Int + let condition: HighlightCondition + } + + let key: Key + let unresolvedRuleIDs: Set + private let rowRules: [CompiledRule] + private let cellRules: [CompiledRule] + + static let empty = HighlightRuleSet(rules: [], columns: [], columnTypes: []) + + init(rules: [HighlightRule], columns: [String], columnTypes: [ColumnType]) { + key = Key(rules: rules, columns: columns, columnTypes: columnTypes) + + var rowRules: [CompiledRule] = [] + var cellRules: [CompiledRule] = [] + var unresolved = Set() + for rule in rules where rule.isEnabled && rule.isValid { + guard let column = Self.columnIndex( + named: rule.columnName, + occurrence: rule.columnOccurrence, + in: columns + ) else { + unresolved.insert(rule.id) + continue + } + let columnType = column < columnTypes.count ? columnTypes[column] : nil + let compiled = CompiledRule( + rule: rule, + column: column, + condition: HighlightCondition(rule: rule, columnType: columnType) + ) + switch rule.target { + case .row: + rowRules.append(compiled) + case .cell: + cellRules.append(compiled) + } + } + self.rowRules = rowRules + self.cellRules = cellRules + self.unresolvedRuleIDs = unresolved + } + + var isEmpty: Bool { rowRules.isEmpty && cellRules.isEmpty } + + func highlight(for values: ContiguousArray) -> RowHighlight { + guard !isEmpty else { return .none } + let rowRule = rowRules.first { Self.matches($0, in: values) }?.rule + var matchedCells: [Int: HighlightRule] = [:] + for compiled in cellRules where matchedCells[compiled.column] == nil && Self.matches(compiled, in: values) { + matchedCells[compiled.column] = compiled.rule + } + return RowHighlight(rowRule: rowRule, cellRules: matchedCells) + } + + static func columnIndex(named name: String, occurrence: Int, in columns: [String]) -> Int? { + var seen = 0 + for (index, column) in columns.enumerated() where column == name { + if seen == occurrence { return index } + seen += 1 + } + return nil + } + + static func occurrence(ofColumnAt index: Int, in columns: [String]) -> Int { + guard index >= 0, index < columns.count else { return 0 } + let name = columns[index] + return columns[..) -> Bool { + guard compiled.column < values.count else { return false } + return compiled.condition.matches(values[compiled.column]) + } +} diff --git a/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift b/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift index 2a11ff404c..997bf4727f 100644 --- a/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift +++ b/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift @@ -21,6 +21,7 @@ struct MenuValidationContext: Equatable { var canUseGridFindCommands = false /// Jump to Column reads the mounted data grid, so it needs one on screen with columns to list. var canJumpToColumn = false + var canPresentHighlightRules = false /// Save As writes the selected tab's SQL, so it needs a query tab and not merely a connection. var isQueryTab = false /// Export Results exports the selected tab's rows, so an empty grid has nothing to offer. @@ -261,6 +262,8 @@ extension MainSplitViewController: NSMenuItemValidation { case #selector(toggleFilterBar(_:)): return context.isConnected && context.canUseTableResultCommands + case #selector(showHighlightRules(_:)): + return context.isConnected && context.canPresentHighlightRules case #selector(pinResult(_:)): return context.canPinResultTab case #selector(navigateBack(_:)): @@ -298,6 +301,7 @@ extension MainSplitViewController: NSMenuItemValidation { canUseTableResultCommands: actions.canUseTableResultCommands, canUseGridFindCommands: actions.canUseGridFindCommands, canJumpToColumn: actions.canJumpToColumn, + canPresentHighlightRules: actions.canPresentHighlightRules, isQueryTab: actions.isQueryTab, hasResultRows: actions.hasResultRows, isCurrentTabEditable: actions.isCurrentTabEditable, diff --git a/TablePro/Core/Services/Infrastructure/MainSplitViewController+ViewMenuActions.swift b/TablePro/Core/Services/Infrastructure/MainSplitViewController+ViewMenuActions.swift index 39f1496233..882d26cfbb 100644 --- a/TablePro/Core/Services/Infrastructure/MainSplitViewController+ViewMenuActions.swift +++ b/TablePro/Core/Services/Infrastructure/MainSplitViewController+ViewMenuActions.swift @@ -63,6 +63,10 @@ extension MainSplitViewController { commandActions?.toggleFilterPanel() } + @objc func showHighlightRules(_ sender: Any?) { + commandActions?.showHighlightRules() + } + @objc func toggleQueryHistory(_ sender: Any?) { commandActions?.toggleHistoryPanel() } diff --git a/TablePro/Core/Storage/ConnectionLocalState.swift b/TablePro/Core/Storage/ConnectionLocalState.swift index 35a481283c..0891c8ab73 100644 --- a/TablePro/Core/Storage/ConnectionLocalState.swift +++ b/TablePro/Core/Storage/ConnectionLocalState.swift @@ -38,6 +38,7 @@ internal enum ConnectionLocalState { } FilterSettingsStorage.shared.removeFilters(for: connectionIds) + HighlightRuleStorage.shared.removeRules(for: connectionIds) DatabaseTreeFilterStorage.shared.removeFilters(for: connectionIds) RecentlyClosedTabStore.shared.removeEntries(for: connectionIds) WorkspaceRailOrderStore.shared.removeEntries(for: connectionIds) diff --git a/TablePro/Core/Storage/HighlightRuleStorage.swift b/TablePro/Core/Storage/HighlightRuleStorage.swift new file mode 100644 index 0000000000..ae2ba5bad7 --- /dev/null +++ b/TablePro/Core/Storage/HighlightRuleStorage.swift @@ -0,0 +1,179 @@ +// +// HighlightRuleStorage.swift +// TablePro +// + +import Foundation +import Observation +import os + +@MainActor +@Observable +final class HighlightRuleStorage { + static let shared = HighlightRuleStorage() + + nonisolated private static let logger = Logger( + subsystem: "com.TablePro", + category: "HighlightRuleStorage" + ) + + private(set) var revision = 0 + + @ObservationIgnored private let storageDirectory: URL + @ObservationIgnored private var cache: [UUID: [String: [HighlightRule]]] = [:] + @ObservationIgnored private let encoder: JSONEncoder = { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return encoder + }() + @ObservationIgnored private let decoder = JSONDecoder() + + init(storageDirectory: URL? = nil) { + self.storageDirectory = storageDirectory ?? Self.resolvedStorageDirectory() + do { + try FileManager.default.createDirectory(at: self.storageDirectory, withIntermediateDirectories: true) + } catch { + Self.logger.error("Failed to create storage directory: \(error.localizedDescription)") + } + } + + func rules(for scope: TableScope) -> [HighlightRule] { + _ = revision + return loadEntries(for: scope.connectionId)[scope.storageComponent] ?? [] + } + + func setRules(_ rules: [HighlightRule], for scope: TableScope) { + var entries = loadEntries(for: scope.connectionId) + guard entries[scope.storageComponent, default: []] != rules else { return } + if rules.isEmpty { + entries.removeValue(forKey: scope.storageComponent) + } else { + entries[scope.storageComponent] = rules + } + commit(entries, for: scope.connectionId) + } + + func rename(from oldScope: TableScope, to newScope: TableScope) { + guard oldScope.storageComponent != newScope.storageComponent else { return } + var entries = loadEntries(for: oldScope.connectionId) + guard let moving = entries.removeValue(forKey: oldScope.storageComponent) else { return } + entries[newScope.storageComponent] = moving + commit(entries, for: oldScope.connectionId) + } + + func renameScope( + connectionId: UUID, + fromDatabase: String, + fromSchema: String?, + toDatabase: String, + toSchema: String? + ) { + let oldPrefix = TableScope.storagePrefix(connectionId: connectionId, database: fromDatabase, schema: fromSchema) + let newPrefix = TableScope.storagePrefix(connectionId: connectionId, database: toDatabase, schema: toSchema) + guard oldPrefix != newPrefix else { return } + + var entries = loadEntries(for: connectionId) + let moving = entries.keys.filter { $0.hasPrefix(oldPrefix) } + guard !moving.isEmpty else { return } + for key in moving { + entries[newPrefix + key.dropFirst(oldPrefix.count)] = entries.removeValue(forKey: key) + } + commit(entries, for: connectionId) + } + + func removeRules(for connectionIds: Set) { + guard !connectionIds.isEmpty else { return } + for connectionId in connectionIds { + cache[connectionId] = [:] + removeFile(at: fileURL(for: connectionId)) + } + revision &+= 1 + } + + private func commit(_ entries: [String: [HighlightRule]], for connectionId: UUID) { + cache[connectionId] = entries + if entries.isEmpty { + removeFile(at: fileURL(for: connectionId)) + } else { + write(entries, for: connectionId) + } + revision &+= 1 + } + + private func loadEntries(for connectionId: UUID) -> [String: [HighlightRule]] { + if let cached = cache[connectionId] { return cached } + + let url = fileURL(for: connectionId) + guard FileManager.default.fileExists(atPath: url.path) else { + cache[connectionId] = [:] + return [:] + } + + do { + let data = try Data(contentsOf: url) + let decoded = try decoder.decode([String: [LossyHighlightRule]].self, from: data) + let entries = decoded.compactMapValues { lossy -> [HighlightRule]? in + let rules = lossy.compactMap(\.rule) + return rules.isEmpty ? nil : rules + } + cache[connectionId] = entries + return entries + } catch { + Self.logger.error( + "Unreadable highlight rules for \(connectionId, privacy: .public): \(error.localizedDescription, privacy: .public)" + ) + preserveUnreadableFile(at: url) + cache[connectionId] = [:] + return [:] + } + } + + private func write(_ entries: [String: [HighlightRule]], for connectionId: UUID) { + do { + let data = try encoder.encode(entries) + try data.write(to: fileURL(for: connectionId), options: .atomic) + } catch { + Self.logger.error( + "Failed to write highlight rules for \(connectionId, privacy: .public): \(error.localizedDescription, privacy: .public)" + ) + } + } + + private func preserveUnreadableFile(at url: URL) { + let preserved = url.deletingPathExtension().appendingPathExtension("unreadable.json") + let fileManager = FileManager.default + try? fileManager.removeItem(at: preserved) + do { + try fileManager.moveItem(at: url, to: preserved) + } catch { + Self.logger.error("Failed to set aside unreadable highlight rules: \(error.localizedDescription, privacy: .public)") + } + } + + private func removeFile(at url: URL) { + guard FileManager.default.fileExists(atPath: url.path) else { return } + do { + try FileManager.default.removeItem(at: url) + } catch { + Self.logger.error("Failed to remove highlight rules file: \(error.localizedDescription, privacy: .public)") + } + } + + private func fileURL(for connectionId: UUID) -> URL { + storageDirectory.appendingPathComponent("\(connectionId.uuidString).json") + } + + private static func resolvedStorageDirectory() -> URL { + AppStorageEnvironment.shared.applicationSupportRoot + .appendingPathComponent("TablePro", isDirectory: true) + .appendingPathComponent("HighlightRules", isDirectory: true) + } +} + +private struct LossyHighlightRule: Decodable { + let rule: HighlightRule? + + init(from decoder: Decoder) throws { + rule = try? HighlightRule(from: decoder) + } +} diff --git a/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/TablePro/Core/Utilities/SQL/QueryClassifier.swift index 0b06ebbc99..76d634cd3b 100644 --- a/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -581,6 +581,8 @@ private extension QueryClassifier { return elasticsearchClassification(trimmed) case .typesense: return typesenseClassification(trimmed) + case .weaviate: + return weaviateClassification(trimmed) default: return nil } @@ -749,4 +751,69 @@ private extension QueryClassifier { } return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: touchesUnsafeSurface) } + + /// A bare operation, a `{"query": ...}` envelope and a console body are the same request, and + /// the driver forwards the envelope verbatim, so the read-only gate has to read all three. + static func weaviateDeclaresMutation(_ body: String) -> Bool { + let trimmed = body.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.lowercased().hasPrefix("mutation") { + return true + } + guard let data = trimmed.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let query = object["query"] as? String + else { return false } + return query.trimmingCharacters(in: .whitespacesAndNewlines).lowercased().hasPrefix("mutation") + } + + /// The driver takes a body from the rest of the request line as well as from the lines below + /// it, so the gate has to read the same two places. + static func weaviateConsoleBody(_ trimmed: String) -> String { + let lines = trimmed.split(separator: "\n", maxSplits: 1, omittingEmptySubsequences: false) + let header = lines.first.map(String.init) ?? "" + let following = lines.count > 1 ? String(lines[1]) : "" + let parts = header.split(maxSplits: 2, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) + let inline = parts.count > 2 ? String(parts[2]) : "" + return inline.isEmpty ? following : inline + } + + static func weaviateClassification(_ trimmed: String) -> QueryClassification { + if trimmed.hasPrefix("WEAVIATE_SEARCH:") { + return .safe + } + if trimmed.hasPrefix("WEAVIATE_WRITE:") { + let encoded = String(trimmed.dropFirst("WEAVIATE_WRITE:".count)) + if let data = Data(base64Encoded: encoded), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + (json["method"] as? String)?.uppercased() == "DELETE" { + return QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) + } + return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + } + let lowered = trimmed.lowercased() + if lowered.hasPrefix("mutation") { + return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + } + if trimmed.hasPrefix("{") || lowered.hasPrefix("query") || lowered.hasPrefix("fragment") { + return weaviateDeclaresMutation(trimmed) + ? QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + : .safe + } + let (verb, path) = typesenseRequestLine(trimmed) + if verb == "GET" || verb == "HEAD" { + return .safe + } + if verb == "POST", path == "/V1/GRAPHQL" { + return weaviateDeclaresMutation(weaviateConsoleBody(trimmed)) + ? QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + : .safe + } + if verb == "DELETE" { + return QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) + } + if verb.isEmpty { + return .safe + } + return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + } } diff --git a/TablePro/Models/Connection/DatabaseType.swift b/TablePro/Models/Connection/DatabaseType.swift index db70ba1768..713c7e0331 100644 --- a/TablePro/Models/Connection/DatabaseType.swift +++ b/TablePro/Models/Connection/DatabaseType.swift @@ -49,6 +49,7 @@ extension DatabaseType { static let typesense = DatabaseType(rawValue: "Typesense") static let teradata = DatabaseType(rawValue: "Teradata") static let trino = DatabaseType(rawValue: "Trino") + static let weaviate = DatabaseType(rawValue: "Weaviate") } extension DatabaseType: Codable { diff --git a/TablePro/Models/Highlight/HighlightRule.swift b/TablePro/Models/Highlight/HighlightRule.swift new file mode 100644 index 0000000000..9dcec164c7 --- /dev/null +++ b/TablePro/Models/Highlight/HighlightRule.swift @@ -0,0 +1,139 @@ +// +// HighlightRule.swift +// TablePro +// + +import Foundation + +enum HighlightColor: String, CaseIterable, Identifiable, Codable, Sendable { + case red + case orange + case yellow + case green + case blue + case purple + case gray + + var id: String { rawValue } + + var displayName: String { + switch self { + case .red: return String(localized: "Red") + case .orange: return String(localized: "Orange") + case .yellow: return String(localized: "Yellow") + case .green: return String(localized: "Green") + case .blue: return String(localized: "Blue") + case .purple: return String(localized: "Purple") + case .gray: return String(localized: "Gray") + } + } +} + +enum HighlightTarget: String, CaseIterable, Identifiable, Codable, Sendable { + case row + case cell + + var id: String { rawValue } + + var displayName: String { + switch self { + case .row: return String(localized: "Row") + case .cell: return String(localized: "Cell") + } + } +} + +struct HighlightRule: Identifiable, Equatable, Hashable, Codable, Sendable { + let id: UUID + var isEnabled: Bool + var columnName: String + var columnOccurrence: Int + var filterOperator: FilterOperator + var value: String + var secondValue: String? + var isCaseSensitive: Bool + var color: HighlightColor + var target: HighlightTarget + + init( + id: UUID = UUID(), + isEnabled: Bool = true, + columnName: String, + columnOccurrence: Int = 0, + filterOperator: FilterOperator = .equal, + value: String = "", + secondValue: String? = nil, + isCaseSensitive: Bool? = nil, + color: HighlightColor = .yellow, + target: HighlightTarget = .row + ) { + self.id = id + self.isEnabled = isEnabled + self.columnName = columnName + self.columnOccurrence = max(0, columnOccurrence) + self.filterOperator = filterOperator + self.value = value + self.secondValue = secondValue + self.isCaseSensitive = isCaseSensitive ?? filterOperator.defaultIsCaseSensitive + self.color = color + self.target = target + } + + private enum CodingKeys: String, CodingKey { + case id, isEnabled, columnName, columnOccurrence, filterOperator, value, secondValue + case isCaseSensitive, color, target + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let decodedOperator = try container.decode(FilterOperator.self, forKey: .filterOperator) + self.id = try container.decodeIfPresent(UUID.self, forKey: .id) ?? UUID() + self.isEnabled = try container.decodeIfPresent(Bool.self, forKey: .isEnabled) ?? true + self.columnName = try container.decode(String.self, forKey: .columnName) + self.columnOccurrence = max(0, try container.decodeIfPresent(Int.self, forKey: .columnOccurrence) ?? 0) + self.filterOperator = decodedOperator + self.value = try container.decodeIfPresent(String.self, forKey: .value) ?? "" + self.secondValue = try container.decodeIfPresent(String.self, forKey: .secondValue) + self.isCaseSensitive = try container.decodeIfPresent(Bool.self, forKey: .isCaseSensitive) + ?? decodedOperator.defaultIsCaseSensitive + self.color = try container.decode(HighlightColor.self, forKey: .color) + self.target = try container.decodeIfPresent(HighlightTarget.self, forKey: .target) ?? .row + } + + var isValid: Bool { + guard !columnName.isEmpty else { return false } + guard filterOperator.requiresValue else { return true } + guard !value.isEmpty else { return false } + guard filterOperator.requiresSecondValue else { return true } + return !(secondValue?.isEmpty ?? true) + } + + func hasSameCondition(as other: HighlightRule) -> Bool { + columnName == other.columnName + && columnOccurrence == other.columnOccurrence + && filterOperator == other.filterOperator + && value == other.value + && (filterOperator.requiresSecondValue ? secondValue == other.secondValue : true) + && isCaseSensitive == other.isCaseSensitive + && target == other.target + } +} + +struct RowHighlight: Equatable, Sendable { + let rowRule: HighlightRule? + let cellRules: [Int: HighlightRule] + + static let none = RowHighlight(rowRule: nil, cellRules: [:]) + + var isEmpty: Bool { rowRule == nil && cellRules.isEmpty } + + var rowColor: HighlightColor? { rowRule?.color } + + func cellRule(forColumn column: Int) -> HighlightRule? { + cellRules[column] + } + + func describingRule(forColumn column: Int) -> HighlightRule? { + cellRules[column] ?? rowRule + } +} diff --git a/TablePro/Models/Highlight/HighlightRuleDescription.swift b/TablePro/Models/Highlight/HighlightRuleDescription.swift new file mode 100644 index 0000000000..18c1ab6a20 --- /dev/null +++ b/TablePro/Models/Highlight/HighlightRuleDescription.swift @@ -0,0 +1,67 @@ +// +// HighlightRuleDescription.swift +// TablePro +// + +import Foundation + +enum HighlightRuleDescription { + static let menuValueLimit = 32 + + static func condition(of rule: HighlightRule, valueLimit: Int? = nil) -> String { + condition( + columnName: rule.columnName, + filterOperator: rule.filterOperator, + value: rule.value, + secondValue: rule.secondValue, + valueLimit: valueLimit + ) + } + + static func condition( + columnName: String, + filterOperator: FilterOperator, + value: String, + secondValue: String?, + valueLimit: Int? = nil + ) -> String { + guard filterOperator.requiresValue else { + return String(format: String(localized: "%1$@ %2$@"), columnName, filterOperator.displayName) + } + + let first = truncated(value, to: valueLimit) + if filterOperator.requiresSecondValue { + return String( + format: String(localized: "%1$@ between “%2$@” and “%3$@”"), + columnName, + first, + truncated(secondValue ?? "", to: valueLimit) + ) + } + + return String( + format: String(localized: "%1$@ %2$@ “%3$@”"), + columnName, + operatorText(filterOperator), + first + ) + } + + static func truncated(_ value: String, to limit: Int?) -> String { + guard let limit, limit > 0 else { return value } + let source = value as NSString + guard source.length > limit else { return value } + let cut = source.rangeOfComposedCharacterSequence(at: limit).location + return source.substring(to: cut) + "\u{2026}" + } + + private static func operatorText(_ filterOperator: FilterOperator) -> String { + switch filterOperator { + case .equal, .notEqual, .greaterThan, .greaterOrEqual, .lessThan, .lessOrEqual: + return filterOperator.symbol + case .contains, .notContains, .startsWith, .endsWith, .isNull, .isNotNull, .isEmpty, + .isNotEmpty, .inList, .notInList, .between, .regex: + return filterOperator.displayName + } + } +} diff --git a/TablePro/Models/Query/QueryTab.swift b/TablePro/Models/Query/QueryTab.swift index ece3ac2985..e94279226c 100644 --- a/TablePro/Models/Query/QueryTab.swift +++ b/TablePro/Models/Query/QueryTab.swift @@ -74,6 +74,7 @@ struct QueryTab: Identifiable, Equatable { /// run with no `DataGridView` in the view tree. Living on the grid's SwiftUI coordinator meant /// switching result mode did not hide the order, it deleted it. (#2251) var valueFilter: GridValueFilterState + var sessionHighlightRules: [HighlightRule] = [] var pagination: PaginationState var chartConfiguration: ResultChartConfiguration var hasUserInteraction: Bool @@ -407,6 +408,7 @@ struct QueryTab: Identifiable, Equatable { && lhs.pagination == rhs.pagination && lhs.sortState == rhs.sortState && lhs.valueFilter == rhs.valueFilter + && lhs.sessionHighlightRules == rhs.sessionHighlightRules && lhs.chartConfiguration == rhs.chartConfiguration && lhs.display == rhs.display && lhs.tableContext.isEditable == rhs.tableContext.isEditable diff --git a/TablePro/Models/Query/QueryTabState.swift b/TablePro/Models/Query/QueryTabState.swift index e4480deabf..48e920e70f 100644 --- a/TablePro/Models/Query/QueryTabState.swift +++ b/TablePro/Models/Query/QueryTabState.swift @@ -657,6 +657,7 @@ struct TabDisplayState: Equatable { var isResultsCollapsed: Bool = false var resultSets: [ResultSet] = [] var activeResultSetId: UUID? + var highlightRulesPresentationRequest: Int = 0 var activeResultSet: ResultSet? { guard let id = activeResultSetId else { return resultSets.last } @@ -698,5 +699,6 @@ struct TabDisplayState: Equatable { && lhs.isResultsCollapsed == rhs.isResultsCollapsed && lhs.resultSets.map(\.id) == rhs.resultSets.map(\.id) && lhs.activeResultSetId == rhs.activeResultSetId + && lhs.highlightRulesPresentationRequest == rhs.highlightRulesPresentationRequest } } diff --git a/TablePro/Models/Query/ResultStatusModel.swift b/TablePro/Models/Query/ResultStatusModel.swift index f58e2c1ad3..ccb0032825 100644 --- a/TablePro/Models/Query/ResultStatusModel.swift +++ b/TablePro/Models/Query/ResultStatusModel.swift @@ -41,6 +41,7 @@ struct ResultStatusControls: Equatable { var showsCountInProgress = false var showsFetchAll = false var showsColumns = false + var showsHighlightRules = false var showsFilters = false var showsPagination = false /// First, Previous, Next, Last and the page number, which an engine that cannot skip rows has @@ -115,6 +116,7 @@ struct ResultStatusModel: Equatable { && !pagination.isLoadingMore controls.showsColumns = viewMode.showsColumnControls && describesAResult + controls.showsHighlightRules = viewMode == .data && describesAResult controls.showsFilters = viewMode.showsRowFilters && isTable && snapshot.hasTableName controls.showsPagination = viewMode.showsResultScope && isTable && snapshot.hasTableName controls.showsPageNavigation = controls.showsPagination && snapshot.paginationCapability.allowsSeeking diff --git a/TablePro/Views/Components/ClosureMenuTarget.swift b/TablePro/Views/Components/ClosureMenuTarget.swift new file mode 100644 index 0000000000..0e85c2f969 --- /dev/null +++ b/TablePro/Views/Components/ClosureMenuTarget.swift @@ -0,0 +1,31 @@ +// +// ClosureMenuTarget.swift +// TablePro +// + +import AppKit + +/// `NSMenuItem` holds its target weakly, so the closure needs an owner that outlives the menu. +/// `representedObject` is that owner: it is strong, it belongs to the item, and it goes when the +/// item does. +@MainActor +final class ClosureMenuTarget: NSObject { + private let action: () -> Void + + init(action: @escaping () -> Void) { + self.action = action + } + + @objc func fire() { + action() + } + + static func item(title: String, isEnabled: Bool = true, action: @escaping () -> Void) -> NSMenuItem { + let item = NSMenuItem(title: title, action: #selector(fire), keyEquivalent: "") + let target = ClosureMenuTarget(action: action) + item.target = target + item.representedObject = target + item.isEnabled = isEnabled + return item + } +} diff --git a/TablePro/Views/Highlight/HighlightColor+AppKit.swift b/TablePro/Views/Highlight/HighlightColor+AppKit.swift new file mode 100644 index 0000000000..fb3bc5b7d6 --- /dev/null +++ b/TablePro/Views/Highlight/HighlightColor+AppKit.swift @@ -0,0 +1,42 @@ +// +// HighlightColor+AppKit.swift +// TablePro +// + +import AppKit + +extension HighlightColor { + static let washAlpha: CGFloat = 0.2 + + var systemColor: NSColor { + switch self { + case .red: return .systemRed + case .orange: return .systemOrange + case .yellow: return .systemYellow + case .green: return .systemGreen + case .blue: return .systemBlue + case .purple: return .systemPurple + case .gray: return .systemGray + } + } + + var washColor: NSColor { + systemColor.withAlphaComponent(Self.washAlpha) + } + + func swatchImage(diameter: CGFloat = 12) -> NSImage { + let color = systemColor + let image = NSImage(size: NSSize(width: diameter, height: diameter), flipped: false) { rect in + color.setFill() + NSBezierPath(ovalIn: rect.insetBy(dx: 0.5, dy: 0.5)).fill() + NSColor.separatorColor.setStroke() + let outline = NSBezierPath(ovalIn: rect.insetBy(dx: 0.5, dy: 0.5)) + outline.lineWidth = 0.5 + outline.stroke() + return true + } + image.isTemplate = false + image.accessibilityDescription = displayName + return image + } +} diff --git a/TablePro/Views/Highlight/HighlightColumnOption.swift b/TablePro/Views/Highlight/HighlightColumnOption.swift new file mode 100644 index 0000000000..945e8fb854 --- /dev/null +++ b/TablePro/Views/Highlight/HighlightColumnOption.swift @@ -0,0 +1,31 @@ +// +// HighlightColumnOption.swift +// TablePro +// + +import Foundation + +struct HighlightColumnOption: Identifiable, Hashable { + let name: String + let occurrence: Int + let label: String + + var id: String { Self.identifier(name: name, occurrence: occurrence) } + + static func identifier(name: String, occurrence: Int) -> String { + "\(occurrence)#\(name)" + } + + static func options(for columns: [String]) -> [HighlightColumnOption] { + var seen: [String: Int] = [:] + let totals = columns.reduce(into: [String: Int]()) { $0[$1, default: 0] += 1 } + return columns.map { name in + let occurrence = seen[name, default: 0] + seen[name] = occurrence + 1 + let label = totals[name, default: 0] > 1 + ? String(format: String(localized: "%1$@ (%2$d)"), name, occurrence + 1) + : name + return HighlightColumnOption(name: name, occurrence: occurrence, label: label) + } + } +} diff --git a/TablePro/Views/Highlight/HighlightMenuBuilder.swift b/TablePro/Views/Highlight/HighlightMenuBuilder.swift new file mode 100644 index 0000000000..537505cbba --- /dev/null +++ b/TablePro/Views/Highlight/HighlightMenuBuilder.swift @@ -0,0 +1,144 @@ +// +// HighlightMenuBuilder.swift +// TablePro +// + +import AppKit +import TableProPluginKit + +@MainActor +enum HighlightMenuBuilder { + struct CellContext { + let columnName: String + let columnOccurrence: Int + let columnType: ColumnType? + let value: PluginCellValue + let existingRules: [HighlightRule] + } + + struct Actions { + let apply: (HighlightRule) -> Void + let remove: (HighlightRule) -> Void + let showRules: () -> Void + } + + static func quickRule( + columnName: String, + columnOccurrence: Int, + columnType: ColumnType?, + value: PluginCellValue, + target: HighlightTarget, + color: HighlightColor + ) -> HighlightRule? { + switch value { + case .null: + return HighlightRule( + columnName: columnName, + columnOccurrence: columnOccurrence, + filterOperator: .isNull, + color: color, + target: target + ) + case .text(let text) where text.isEmpty: + return HighlightRule( + columnName: columnName, + columnOccurrence: columnOccurrence, + filterOperator: .isEmpty, + color: color, + target: target + ) + case .text(let text) where HighlightCondition.readsAsNullLiteral(text, columnType: columnType): + return nil + case .text(let text): + return HighlightRule( + columnName: columnName, + columnOccurrence: columnOccurrence, + filterOperator: .equal, + value: text, + color: color, + target: target + ) + case .bytes: + return nil + } + } + + static func sectionTitle(for rule: HighlightRule) -> String { + let condition = HighlightRuleDescription.condition( + of: rule, + valueLimit: HighlightRuleDescription.menuValueLimit + ) + switch rule.target { + case .row: + return String(format: String(localized: "Rows Where %@"), condition) + case .cell: + return String(format: String(localized: "Cells Where %@"), condition) + } + } + + static func menuItem(for context: CellContext, actions: Actions) -> NSMenuItem? { + let templates = HighlightTarget.allCases.compactMap { target in + quickRule( + columnName: context.columnName, + columnOccurrence: context.columnOccurrence, + columnType: context.columnType, + value: context.value, + target: target, + color: .yellow + ) + } + guard !templates.isEmpty else { return nil } + + let submenu = NSMenu() + var existingMatches: [HighlightRule] = [] + for template in templates { + let existing = context.existingRules.first { $0.hasSameCondition(as: template) } + if let existing { existingMatches.append(existing) } + submenu.addItem(.sectionHeader(title: sectionTitle(for: template))) + submenu.addItem(paletteItem(for: template, existing: existing, actions: actions)) + } + + submenu.addItem(.separator()) + if !existingMatches.isEmpty { + submenu.addItem(ClosureMenuTarget.item(title: String(localized: "Remove Highlight")) { + existingMatches.forEach(actions.remove) + }) + } + submenu.addItem(ClosureMenuTarget.item(title: String(localized: "Highlight Rules…"), action: actions.showRules)) + + let item = NSMenuItem(title: String(localized: "Highlight"), action: nil, keyEquivalent: "") + item.image = NSImage(systemSymbolName: "highlighter", accessibilityDescription: nil) + item.submenu = submenu + return item + } + + private static func paletteItem( + for template: HighlightRule, + existing: HighlightRule?, + actions: Actions + ) -> NSMenuItem { + let colors = HighlightColor.allCases + let palette = NSMenu.palette( + colors: colors.map(\.systemColor), + titles: colors.map(\.displayName) + ) { menu in + let selected = menu.selectedItems.compactMap { menu.items.firstIndex(of: $0) } + guard let index = selected.first, colors.indices.contains(index) else { + if let existing { actions.remove(existing) } + return + } + var rule = existing ?? template + rule.color = colors[index] + rule.isEnabled = true + actions.apply(rule) + } + palette.selectionMode = .selectOne + if let existing, let index = colors.firstIndex(of: existing.color), index < palette.items.count { + palette.selectedItems = [palette.items[index]] + } + + let item = NSMenuItem(title: sectionTitle(for: template), action: nil, keyEquivalent: "") + item.submenu = palette + return item + } +} diff --git a/TablePro/Views/Highlight/HighlightRuleRow.swift b/TablePro/Views/Highlight/HighlightRuleRow.swift new file mode 100644 index 0000000000..7e07625e35 --- /dev/null +++ b/TablePro/Views/Highlight/HighlightRuleRow.swift @@ -0,0 +1,224 @@ +// +// HighlightRuleRow.swift +// TablePro +// + +import SwiftUI + +struct HighlightRuleRow: View { + @Binding var rule: HighlightRule + let columnOptions: [HighlightColumnOption] + @Binding var focusedRuleID: UUID? + let canMoveUp: Bool + let canMoveDown: Bool + let onMoveUp: () -> Void + let onMoveDown: () -> Void + let onRemove: () -> Void + let onCancel: () -> Void + + private var isColumnMissing: Bool { + !columnOptions.contains { $0.name == rule.columnName && $0.occurrence == rule.columnOccurrence } + } + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 6) { + Toggle("", isOn: $rule.isEnabled) + .toggleStyle(.checkbox) + .labelsHidden() + .accessibilityLabel(String(localized: "Enable rule")) + .accessibilityIdentifier("highlight-rule-enabled") + .help(String(localized: "Apply this rule")) + conditionEditor + .opacity(rule.isEnabled ? 1 : 0.5) + } + HStack(spacing: 8) { + colorPicker + targetPicker + if isColumnMissing { + Label(String(localized: "Not in this result"), systemImage: "exclamationmark.triangle") + .font(.caption) + .foregroundStyle(.secondary) + .help(String(format: String(localized: "This result has no column named %@"), rule.columnName)) + } + Spacer(minLength: 0) + removeButton + } + .padding(.leading, 22) + .opacity(rule.isEnabled ? 1 : 0.5) + } + .padding(.vertical, 4) + .accessibilityElement(children: .contain) + .accessibilityLabel(HighlightRuleDescription.condition(of: rule)) + .accessibilityActions { + if canMoveUp { + Button(String(localized: "Move Rule Up"), action: onMoveUp) + } + if canMoveDown { + Button(String(localized: "Move Rule Down"), action: onMoveDown) + } + } + } + + private var conditionEditor: some View { + HStack(spacing: 6) { + columnPicker + operatorMenu + valueFields + } + } + + private var columnSelection: Binding { + Binding( + get: { HighlightColumnOption.identifier(name: rule.columnName, occurrence: rule.columnOccurrence) }, + set: { identifier in + guard let option = columnOptions.first(where: { $0.id == identifier }) else { return } + rule.columnName = option.name + rule.columnOccurrence = option.occurrence + } + ) + } + + private var columnPicker: some View { + Picker("", selection: columnSelection) { + ForEach(columnOptions) { option in + Text(option.label).tag(option.id) + } + if isColumnMissing { + Divider() + Text(rule.columnName) + .tag(HighlightColumnOption.identifier(name: rule.columnName, occurrence: rule.columnOccurrence)) + } + } + .pickerStyle(.menu) + .controlSize(.small) + .fixedSize() + .labelsHidden() + .accessibilityLabel(String(localized: "Rule column")) + .accessibilityValue(rule.columnName) + } + + private var operatorSelection: Binding { + Binding( + get: { rule.filterOperator }, + set: { newOperator in + guard newOperator != rule.filterOperator else { return } + rule.filterOperator = newOperator + rule.isCaseSensitive = newOperator.defaultIsCaseSensitive + } + ) + } + + private var operatorMenu: some View { + Menu { + Picker("", selection: operatorSelection) { + ForEach(FilterOperator.allCases) { filterOperator in + Text(Self.operatorLabel(filterOperator)) + .accessibilityLabel(filterOperator.displayName) + .tag(filterOperator) + } + } + .pickerStyle(.inline) + .labelsHidden() + + if rule.filterOperator.supportsCaseSensitivity { + Divider() + Toggle(String(localized: "Match Case"), isOn: $rule.isCaseSensitive) + } + } label: { + HStack(spacing: 3) { + Text(Self.operatorLabel(rule.filterOperator)) + if rule.filterOperator.supportsCaseSensitivity, + rule.isCaseSensitive != rule.filterOperator.defaultIsCaseSensitive { + Image(systemName: "textformat") + .imageScale(.small) + .foregroundStyle(.secondary) + } + } + } + .menuStyle(.button) + .controlSize(.small) + .fixedSize() + .accessibilityLabel(String(localized: "Rule operator")) + .accessibilityValue(rule.filterOperator.displayName) + } + + @ViewBuilder + private var valueFields: some View { + if rule.filterOperator.requiresValue { + FilterValueTextField( + text: $rule.value, + focusedId: $focusedRuleID, + identity: rule.id, + placeholder: String(localized: "Value"), + onCancel: onCancel + ) + .frame(minWidth: 90) + .accessibilityLabel(String(localized: "Rule value")) + + if rule.filterOperator.requiresSecondValue { + Text("and") + .font(.subheadline) + .foregroundStyle(.secondary) + TextField("Value", text: Binding( + get: { rule.secondValue ?? "" }, + set: { rule.secondValue = $0 } + )) + .textFieldStyle(.roundedBorder) + .controlSize(.small) + .autocorrectionDisabled(true) + .frame(minWidth: 70) + .accessibilityLabel(String(localized: "Second rule value")) + } + } else { + Spacer(minLength: 0) + } + } + + private var colorPicker: some View { + Picker("", selection: $rule.color) { + ForEach(HighlightColor.allCases) { color in + Label { + Text(color.displayName) + } icon: { + Image(nsImage: color.swatchImage()) + } + .tag(color) + } + } + .pickerStyle(.menu) + .controlSize(.small) + .fixedSize() + .labelsHidden() + .accessibilityLabel(String(localized: "Highlight color")) + .accessibilityValue(rule.color.displayName) + } + + private var targetPicker: some View { + Picker("", selection: $rule.target) { + ForEach(HighlightTarget.allCases) { target in + Text(target.displayName).tag(target) + } + } + .pickerStyle(.segmented) + .controlSize(.small) + .fixedSize() + .labelsHidden() + .accessibilityLabel(String(localized: "Apply To")) + .help(String(localized: "Color the whole row, or only the matching cell")) + } + + private var removeButton: some View { + Button(String(localized: "Remove Rule"), systemImage: "minus", action: onRemove) + .labelStyle(.iconOnly) + .buttonStyle(.bordered) + .controlSize(.small) + .help(String(localized: "Remove this rule")) + } + + private static func operatorLabel(_ filterOperator: FilterOperator) -> String { + filterOperator.symbol.isEmpty + ? filterOperator.displayName + : "\(filterOperator.symbol) \(filterOperator.displayName)" + } +} diff --git a/TablePro/Views/Highlight/HighlightRulesPopover.swift b/TablePro/Views/Highlight/HighlightRulesPopover.swift new file mode 100644 index 0000000000..642004f9c0 --- /dev/null +++ b/TablePro/Views/Highlight/HighlightRulesPopover.swift @@ -0,0 +1,144 @@ +// +// HighlightRulesPopover.swift +// TablePro +// + +import SwiftUI + +struct HighlightRulesPopover: View { + let columns: [String] + let rules: [HighlightRule] + let isPersisted: Bool + let onChange: ([HighlightRule]) -> Void + + @State private var focusedRuleID: UUID? + @Environment(\.dismiss) private var dismiss + + private static let rowHeight: CGFloat = 64 + private static let maximumListHeight: CGFloat = 420 + + private var columnOptions: [HighlightColumnOption] { + HighlightColumnOption.options(for: columns) + } + + var body: some View { + VStack(alignment: .leading, spacing: 0) { + header + Divider() + if rules.isEmpty { + emptyState + } else { + ruleList + } + Divider() + footer + } + .frame(width: 540) + } + + private var header: some View { + VStack(alignment: .leading, spacing: 2) { + Text("Highlight Rules") + .font(.headline) + Text("Rules are checked in order. The first match sets the color.") + .font(.caption) + .foregroundStyle(.secondary) + } + .padding(.horizontal, 12) + .padding(.vertical, 10) + } + + private var emptyState: some View { + ContentUnavailableView { + Label(String(localized: "No Highlight Rules"), systemImage: "highlighter") + } description: { + Text("Right-click a cell and choose Highlight to color rows by value.") + } + .frame(maxWidth: .infinity) + .padding(.vertical, 8) + } + + private var ruleList: some View { + List { + ForEach(rules) { rule in + HighlightRuleRow( + rule: binding(for: rule), + columnOptions: columnOptions, + focusedRuleID: $focusedRuleID, + canMoveUp: rules.first?.id != rule.id, + canMoveDown: rules.last?.id != rule.id, + onMoveUp: { move(rule, by: -1) }, + onMoveDown: { move(rule, by: 1) }, + onRemove: { remove(rule) }, + onCancel: close + ) + } + .onMove(perform: move) + } + .listStyle(.plain) + .scrollContentBackground(.hidden) + .frame(height: min(CGFloat(rules.count) * Self.rowHeight + 8, Self.maximumListHeight)) + } + + private var footer: some View { + HStack(spacing: 8) { + Button(String(localized: "Add Rule"), systemImage: "plus", action: addRule) + .controlSize(.small) + .disabled(columns.isEmpty) + .accessibilityIdentifier("highlight-rules-add") + + Spacer(minLength: 8) + + if !isPersisted { + Text("Rules for this query result are not saved.") + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + } + } + .padding(.horizontal, 12) + .padding(.vertical, 8) + } + + private func binding(for rule: HighlightRule) -> Binding { + Binding( + get: { rules.first { $0.id == rule.id } ?? rule }, + set: { updated in + var next = rules + guard let index = next.firstIndex(where: { $0.id == updated.id }) else { return } + next[index] = updated + onChange(next) + } + ) + } + + private func addRule() { + guard let first = columnOptions.first else { return } + let rule = HighlightRule(columnName: first.name, columnOccurrence: first.occurrence) + onChange(rules + [rule]) + focusedRuleID = rule.id + } + + private func close() { + dismiss() + } + + private func remove(_ rule: HighlightRule) { + onChange(rules.filter { $0.id != rule.id }) + } + + private func move(from source: IndexSet, to destination: Int) { + var next = rules + next.move(fromOffsets: source, toOffset: destination) + onChange(next) + } + + private func move(_ rule: HighlightRule, by offset: Int) { + guard let index = rules.firstIndex(where: { $0.id == rule.id }) else { return } + let target = index + offset + guard rules.indices.contains(target) else { return } + var next = rules + next.swapAt(index, target) + onChange(next) + } +} diff --git a/TablePro/Views/Main/Child/DataTabGridDelegate.swift b/TablePro/Views/Main/Child/DataTabGridDelegate.swift index 4de563328c..2de98910ef 100644 --- a/TablePro/Views/Main/Child/DataTabGridDelegate.swift +++ b/TablePro/Views/Main/Child/DataTabGridDelegate.swift @@ -134,6 +134,48 @@ final class DataTabGridDelegate: DataGridViewDelegate { return menu } + func dataGridHighlightMenuItem(forRow displayRow: Int, dataColumn: Int) -> NSMenuItem? { + guard let coordinator, + let grid = tableViewCoordinator, + let tab = coordinator.tabManager.selectedTab, + let row = grid.displayRow(at: displayRow) else { return nil } + let tableRows = grid.tableRowsProvider() + let columns = tableRows.columns + guard columns.indices.contains(dataColumn), dataColumn < row.values.count else { return nil } + + let tabId = tab.id + let context = HighlightMenuBuilder.CellContext( + columnName: columns[dataColumn], + columnOccurrence: HighlightRuleSet.occurrence(ofColumnAt: dataColumn, in: columns), + columnType: dataColumn < tableRows.columnTypes.count ? tableRows.columnTypes[dataColumn] : nil, + value: row.values[dataColumn], + existingRules: coordinator.highlightRules(for: tab) + ) + let actions = HighlightMenuBuilder.Actions( + apply: { [weak coordinator] rule in + coordinator?.applyQuickHighlight(rule, forTab: tabId) + }, + remove: { [weak coordinator] rule in + coordinator?.removeHighlightRules(sharingConditionWith: rule, forTab: tabId) + }, + showRules: { [weak coordinator] in + coordinator?.presentHighlightRules() + } + ) + return HighlightMenuBuilder.menuItem(for: context, actions: actions) + } + + func dataGridHighlightValuesMenuItem(forColumn dataColumnIndex: Int) -> NSMenuItem? { + guard coordinator != nil, let grid = tableViewCoordinator else { return nil } + let columns = grid.tableRowsProvider().columns + guard columns.indices.contains(dataColumnIndex) else { return nil } + let columnName = columns[dataColumnIndex] + let occurrence = HighlightRuleSet.occurrence(ofColumnAt: dataColumnIndex, in: columns) + return ClosureMenuTarget.item(title: String(localized: "Highlight Values…")) { [weak coordinator] in + coordinator?.presentHighlightRules(addingRuleForColumn: columnName, occurrence: occurrence) + } + } + weak var tableViewCoordinator: TableViewCoordinator? func dataGridAttach(tableViewCoordinator: TableViewCoordinator) { diff --git a/TablePro/Views/Main/Child/MainEditorContentView.swift b/TablePro/Views/Main/Child/MainEditorContentView.swift index bfe980b42f..07b6b67c9c 100644 --- a/TablePro/Views/Main/Child/MainEditorContentView.swift +++ b/TablePro/Views/Main/Child/MainEditorContentView.swift @@ -916,6 +916,7 @@ struct MainEditorContentView: View { editRefusalMessage: refusal?.message ), displayFormats: coordinator.displayFormats(for: tab), + highlightRules: coordinator.highlightRules(for: tab), delegate: dataTabDelegate, selectedRowIndices: Binding( get: { selectionState.indices }, @@ -1019,6 +1020,18 @@ struct MainEditorContentView: View { ? { coordinator.showColumnJump(seededWith: $0) } : nil ), + highlightState: StatusBarHighlightState( + rules: coordinator.highlightRules(for: tab), + columns: resolvedRows.columns, + isPersisted: coordinator.highlightRuleScope(for: tab) != nil, + presentationRequest: tab.display.highlightRulesPresentationRequest, + onChange: { [coordinator, tabId = tab.id] rules in + coordinator.setHighlightRules(rules, forTab: tabId) + }, + onDismiss: { [coordinator, tabId = tab.id] in + coordinator.discardIncompleteHighlightRules(forTab: tabId) + } + ), paginationCallbacks: PaginationCallbacks( onFirst: onFirstPage, onPrevious: onPreviousPage, diff --git a/TablePro/Views/Main/EditorTabContextMenuBuilder.swift b/TablePro/Views/Main/EditorTabContextMenuBuilder.swift index f06ad43bb6..214b893374 100644 --- a/TablePro/Views/Main/EditorTabContextMenuBuilder.swift +++ b/TablePro/Views/Main/EditorTabContextMenuBuilder.swift @@ -58,27 +58,6 @@ internal enum EditorTabContextMenuBuilder { isEnabled: Bool = true, action: @escaping () -> Void ) { - let item = NSMenuItem(title: title, action: #selector(ClosureMenuTarget.fire), keyEquivalent: "") - let target = ClosureMenuTarget(action: action) - item.target = target - item.representedObject = target - item.isEnabled = isEnabled - menu.addItem(item) - } -} - -/// `NSMenuItem` holds its target weakly, so the closure needs an owner that outlives the menu. -/// `representedObject` is that owner: it is strong, it belongs to the item, and it goes when the -/// item does. -@MainActor -private final class ClosureMenuTarget: NSObject { - private let action: () -> Void - - init(action: @escaping () -> Void) { - self.action = action - } - - @objc func fire() { - action() + menu.addItem(ClosureMenuTarget.item(title: title, isEnabled: isEnabled, action: action)) } } diff --git a/TablePro/Views/Main/Extensions/MainContentCoordinator+HighlightRules.swift b/TablePro/Views/Main/Extensions/MainContentCoordinator+HighlightRules.swift new file mode 100644 index 0000000000..ab684bfd84 --- /dev/null +++ b/TablePro/Views/Main/Extensions/MainContentCoordinator+HighlightRules.swift @@ -0,0 +1,66 @@ +// +// MainContentCoordinator+HighlightRules.swift +// TablePro +// + +import Foundation + +extension MainContentCoordinator { + func highlightRuleScope(for tab: QueryTab) -> TableScope? { + tab.tableContext.scope(connectionId: connectionId) + } + + func highlightRules(for tab: QueryTab) -> [HighlightRule] { + guard let scope = highlightRuleScope(for: tab) else { return tab.sessionHighlightRules } + return HighlightRuleStorage.shared.rules(for: scope) + } + + func setHighlightRules(_ rules: [HighlightRule], forTab tabId: UUID) { + guard let index = tabManager.tabs.firstIndex(where: { $0.id == tabId }) else { return } + if let scope = highlightRuleScope(for: tabManager.tabs[index]) { + HighlightRuleStorage.shared.setRules(rules, for: scope) + return + } + guard tabManager.tabs[index].sessionHighlightRules != rules else { return } + tabManager.mutate(at: index) { $0.sessionHighlightRules = rules } + } + + func applyQuickHighlight(_ rule: HighlightRule, forTab tabId: UUID) { + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }) else { return } + var rules = highlightRules(for: tab) + rules.removeAll { $0.hasSameCondition(as: rule) } + rules.insert(rule, at: 0) + setHighlightRules(rules, forTab: tabId) + } + + func removeHighlightRules(sharingConditionWith rule: HighlightRule, forTab tabId: UUID) { + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }) else { return } + let rules = highlightRules(for: tab).filter { !$0.hasSameCondition(as: rule) } + setHighlightRules(rules, forTab: tabId) + } + + func discardIncompleteHighlightRules(forTab tabId: UUID) { + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }) else { return } + let rules = highlightRules(for: tab) + let complete = rules.filter(\.isValid) + guard complete.count != rules.count else { return } + setHighlightRules(complete, forTab: tabId) + } + + func presentHighlightRules(addingRuleForColumn columnName: String? = nil, occurrence: Int = 0) { + guard let index = tabManager.selectedTabIndex else { return } + let tabId = tabManager.tabs[index].id + if let columnName { + let newRule = HighlightRule(columnName: columnName, columnOccurrence: occurrence) + setHighlightRules(highlightRules(for: tabManager.tabs[index]) + [newRule], forTab: tabId) + } + tabManager.mutate(at: index) { $0.display.highlightRulesPresentationRequest &+= 1 } + } + + var canPresentHighlightRules: Bool { + guard hasMountedDataGrid, + let tab = tabManager.selectedTab, + tab.display.resultsViewMode == .data else { return false } + return !(tabSessionRegistry.existingTableRows(for: tab.id)?.columns.isEmpty ?? true) + } +} diff --git a/TablePro/Views/Main/Extensions/MainContentCoordinator+RenameAdoption.swift b/TablePro/Views/Main/Extensions/MainContentCoordinator+RenameAdoption.swift index 3bcfabbfb6..dd32e6b913 100644 --- a/TablePro/Views/Main/Extensions/MainContentCoordinator+RenameAdoption.swift +++ b/TablePro/Views/Main/Extensions/MainContentCoordinator+RenameAdoption.swift @@ -98,6 +98,11 @@ extension MainContentCoordinator { connectionId: connectionId, databaseName: database, schemaName: schema, tableName: newName ) ) + let scopedDatabase = database.isEmpty ? nil : database + HighlightRuleStorage.shared.rename( + from: TableScope(connectionId: connectionId, database: scopedDatabase, schema: schema, table: oldName), + to: TableScope(connectionId: connectionId, database: scopedDatabase, schema: schema, table: newName) + ) } private func moveFavorite(_ ref: DatabaseTreeTableRef, to newName: String, database: String?) { @@ -139,6 +144,10 @@ extension MainContentCoordinator { connectionId: connectionId, fromDatabase: database, fromSchema: schema, toDatabase: toDatabase, toSchema: toSchema ) + HighlightRuleStorage.shared.renameScope( + connectionId: connectionId, fromDatabase: database, fromSchema: schema, + toDatabase: toDatabase, toSchema: toSchema + ) retargetFavoriteTables( database: database, schema: schema, toDatabase: toDatabase, toSchema: toSchema ) diff --git a/TablePro/Views/Main/MainContentCommandActions.swift b/TablePro/Views/Main/MainContentCommandActions.swift index 8ff2fda18c..b2b3ae6211 100644 --- a/TablePro/Views/Main/MainContentCommandActions.swift +++ b/TablePro/Views/Main/MainContentCommandActions.swift @@ -936,6 +936,15 @@ final class MainContentCommandActions { coordinator.toggleFilterPanel() } + var canPresentHighlightRules: Bool { + coordinator?.canPresentHighlightRules ?? false + } + + func showHighlightRules() { + guard canPresentHighlightRules, let coordinator else { return } + coordinator.presentHighlightRules() + } + func showFindBar() { guard canUseGridFindCommands, let coordinator else { return } coordinator.findCoordinator.show() diff --git a/TablePro/Views/Results/Cells/DataGridCellAccessibilityView.swift b/TablePro/Views/Results/Cells/DataGridCellAccessibilityView.swift index 3978dc9a0c..1d5ba6d736 100644 --- a/TablePro/Views/Results/Cells/DataGridCellAccessibilityView.swift +++ b/TablePro/Views/Results/Cells/DataGridCellAccessibilityView.swift @@ -88,11 +88,20 @@ internal final class DataGridCellAccessibilityView: NSView { override internal func accessibilityValue() -> Any? { text } override internal func accessibilityLabel() -> String? { - String( - format: String(localized: "Row %d, column %d: %@"), + guard let highlight = coordinator?.highlightDescription(row: row, columnIndex: dataColumn) else { + return String( + format: String(localized: "Row %d, column %d: %@"), + row + 1, + dataColumn + 1, + text + ) + } + return String( + format: String(localized: "Row %d, column %d: %@, highlighted where %@"), row + 1, dataColumn + 1, - text + text, + highlight ) } diff --git a/TablePro/Views/Results/Cells/DataGridCellAppearance.swift b/TablePro/Views/Results/Cells/DataGridCellAppearance.swift index 9a0d23f2d9..01b03621fe 100644 --- a/TablePro/Views/Results/Cells/DataGridCellAppearance.swift +++ b/TablePro/Views/Results/Cells/DataGridCellAppearance.swift @@ -15,7 +15,7 @@ struct DataGridCellAppearance: Equatable { let text: String let font: NSFont let textColor: NSColor - /// Painted behind the text, for a find match or a modified value. + /// Painted behind the text, for a find match, a modified value or a highlight rule. let backgroundTint: NSColor? let accessory: DataGridCellAccessory /// Which symbol the accessory draws, resolved here because it follows the row's state rather @@ -52,18 +52,19 @@ struct DataGridCellAppearance: Equatable { } let findTint: NSColor? = state.isCurrentFindMatch ? palette.findMatchTint : nil - let modifiedTint: NSColor? + let highlightColor = state.visualState.cellHighlightColor(forColumn: state.columnIndex) + let stateTint: NSColor? if state.visualState.isDeleted || state.visualState.isInserted { - modifiedTint = nil + stateTint = nil } else if state.visualState.isModified(columnIndex: state.columnIndex) { - modifiedTint = palette.modifiedColumnTint + stateTint = palette.modifiedColumnTint } else { - modifiedTint = nil + stateTint = highlightColor?.washColor } // A find match keeps its own highlight whatever else is true, and the text turns black // against it. Otherwise a selected row's text takes the selection's own colour, and the - // modified tint stands down so the selection fill is not painted over. + // modified or highlight tint stands down so the selection fill is not painted over. let backgroundTint: NSColor? let textColor: NSColor if let findTint { @@ -73,7 +74,7 @@ struct DataGridCellAppearance: Equatable { backgroundTint = nil textColor = .alternateSelectedControlTextColor } else { - backgroundTint = modifiedTint + backgroundTint = stateTint textColor = baseColor } diff --git a/TablePro/Views/Results/DataGridCoordinator.swift b/TablePro/Views/Results/DataGridCoordinator.swift index 88ca3cb1d4..28e2f55309 100644 --- a/TablePro/Views/Results/DataGridCoordinator.swift +++ b/TablePro/Views/Results/DataGridCoordinator.swift @@ -550,6 +550,7 @@ final class TableViewCoordinator: NSObject, NSTableViewDelegate, NSTableViewData static let rowViewIdentifier = NSUserInterfaceItemIdentifier("TableRowView") let visualIndex = RowVisualIndex() + var highlightRuleSet: HighlightRuleSet = .empty private let largeDatasetThreshold = 5_000 var isLargeDataset: Bool { cachedRowCount > largeDatasetThreshold } @@ -693,6 +694,7 @@ final class TableViewCoordinator: NSObject, NSTableViewDelegate, NSTableViewData systemTimeZoneCancellable = nil detachAccessibilityActivationObserver() visualIndex.clear() + highlightRuleSet = .empty displayCache.removeAll() columnDisplayFormats = [] cachedRowCount = 0 @@ -747,6 +749,7 @@ final class TableViewCoordinator: NSObject, NSTableViewDelegate, NSTableViewData visualIndex.rebuild(from: changeManager, displayIDs: displayIDs) updateCache() tableView.insertRows(at: indices, withAnimation: Self.rowAnimation(.slideDown)) + repaintVisibleRowDecorations() } /// Accessibility > Display > Reduce Motion asks for no sliding rows, and the app @@ -764,6 +767,7 @@ final class TableViewCoordinator: NSObject, NSTableViewDelegate, NSTableViewData visualIndex.rebuild(from: changeManager, displayIDs: displayIDs) updateCache() tableView.removeRows(at: indices, withAnimation: Self.rowAnimation(.slideUp)) + repaintVisibleRowDecorations() } private func bumpDisplayRevision() { @@ -1046,6 +1050,7 @@ final class TableViewCoordinator: NSObject, NSTableViewDelegate, NSTableViewData private func invalidateDisplayCache(forDisplayRow displayIndex: Int, column: Int) { guard let row = displayRow(at: displayIndex) else { return } + displayCache.clearHighlight(forID: row.id) guard let box = displayCache.box(forID: row.id), column >= 0, column < box.values.count else { return } box.values[column] = nil @@ -1062,6 +1067,7 @@ final class TableViewCoordinator: NSObject, NSTableViewDelegate, NSTableViewData invalidateDisplayCache(forDisplayRow: row, column: column) visualIndex.updateRow(row, from: changeManager, displayIDs: displayIDs) redrawCells(rows: IndexSet(integer: row), tableColumnIndexes: IndexSet(integer: tableColumn)) + invalidateRowDecoration(displayRow: row) case .cellsChanged(let positions): guard !positions.isEmpty, let tableView else { return } var rowSet = IndexSet() @@ -1080,6 +1086,9 @@ final class TableViewCoordinator: NSObject, NSTableViewDelegate, NSTableViewData visualIndex.updateRow(row, from: changeManager, displayIDs: displayIDs) } redrawCells(rows: rowSet, tableColumnIndexes: colSet) + for row in rowSet { + invalidateRowDecoration(displayRow: row) + } case .rowsInserted(let indices): guard !indices.isEmpty else { return } overlayEditor?.dismiss(commit: false) @@ -1130,21 +1139,6 @@ final class TableViewCoordinator: NSObject, NSTableViewDelegate, NSTableViewData refreshRowVisualState(at: row) } - func refreshVisibleRowVisualStates() { - guard let tableView else { return } - tableView.enumerateAvailableRowViews { [weak self] rowView, row in - guard let self, let dataRowView = rowView as? DataGridRowView else { return } - dataRowView.applyVisualState(self.visualState(for: row)) - } - } - - func refreshRowVisualState(at row: Int) { - guard let tableView, - let dataRowView = tableView.rowView(atRow: row, makeIfNecessary: false) as? DataGridRowView - else { return } - dataRowView.applyVisualState(visualState(for: row)) - } - func commitActiveCellEdit() { overlayEditor?.dismiss(commit: true) overlayViewer?.dismiss() @@ -1419,7 +1413,7 @@ final class TableViewCoordinator: NSObject, NSTableViewDelegate, NSTableViewData if let delegateState = delegate?.dataGridVisualState(forRow: row) { return delegateState } - return visualIndex.visualState(for: row) + return visualIndex.visualState(for: row).highlighted(highlight(forDisplayRow: row)) } // MARK: - NSTableViewDataSource diff --git a/TablePro/Views/Results/DataGridRowView.swift b/TablePro/Views/Results/DataGridRowView.swift index df68344e02..53faf4f8f1 100644 --- a/TablePro/Views/Results/DataGridRowView.swift +++ b/TablePro/Views/Results/DataGridRowView.swift @@ -37,8 +37,9 @@ class DataGridRowView: NSTableRowView { private var seededRowIndex: Int = 0 - private(set) var visualState: RowVisualState = .empty - private var rowTint: NSColor? + var visualState: RowVisualState { + coordinator?.visualState(for: rowIndex) ?? .empty + } /// Draws the row's data cells. /// @@ -201,14 +202,7 @@ class DataGridRowView: NSTableRowView { "hidden": NSNull(), ] - /// The tint derives from the row state and the active theme, so it is recomputed on every call - /// and the colour comparison below decides whether anything needs redrawing. Returning early on - /// an unchanged state would ignore the theme, which is the input a theme change moves. - func applyVisualState(_ state: RowVisualState) { - visualState = state - let nextTint = state.tint - guard !colorsEqual(rowTint, nextTint) else { return } - rowTint = nextTint + func invalidateVisualState() { needsDisplay = true } @@ -236,8 +230,8 @@ class DataGridRowView: NSTableRowView { override func drawBackground(in dirtyRect: NSRect) { super.drawBackground(in: dirtyRect) - if let rowTint, !isSelected { - rowTint.setFill() + if !isSelected, let tint = visualState.tint { + tint.setFill() bounds.fill() } drawCellSelectionFill(in: dirtyRect) @@ -277,14 +271,6 @@ class DataGridRowView: NSTableRowView { private static let emphasizedCellSelectionAlpha: CGFloat = 0.28 - private func colorsEqual(_ lhs: NSColor?, _ rhs: NSColor?) -> Bool { - switch (lhs, rhs) { - case (nil, nil): return true - case let (l?, r?): return l == r - default: return false - } - } - private func addForeignKeyMenuItems(to menu: NSMenu, dataColumnIndex: Int, tableRows: TableRows) { guard let coordinator, dataColumnIndex >= 0, dataColumnIndex < tableRows.columns.count else { return } let columnName = tableRows.columns[dataColumnIndex] @@ -510,6 +496,14 @@ class DataGridRowView: NSTableRowView { jsonViewItem.target = self menu.addItem(jsonViewItem) + if dataColumnIndex >= 0, + let highlightItem = coordinator.delegate?.dataGridHighlightMenuItem( + forRow: rowIndex, + dataColumn: dataColumnIndex + ) { + menu.addItem(highlightItem) + } + let tableRows = coordinator.tableRowsProvider() addForeignKeyMenuItems(to: menu, dataColumnIndex: dataColumnIndex, tableRows: tableRows) diff --git a/TablePro/Views/Results/DataGridUpdateSnapshot.swift b/TablePro/Views/Results/DataGridUpdateSnapshot.swift index 51c3667cfa..c5a1359b4c 100644 --- a/TablePro/Views/Results/DataGridUpdateSnapshot.swift +++ b/TablePro/Views/Results/DataGridUpdateSnapshot.swift @@ -13,6 +13,7 @@ struct DataGridUpdateSnapshot: Equatable { let columns: [String] let valueFilteredIDsCount: Int? let displayFormats: [ValueDisplayFormat?] + let highlightRules: [HighlightRule] let configuration: DataGridConfiguration let isEditable: Bool let rowReorder: DataGridRowReorder diff --git a/TablePro/Views/Results/DataGridView.swift b/TablePro/Views/Results/DataGridView.swift index a01aebcd71..baabbbbcca 100644 --- a/TablePro/Views/Results/DataGridView.swift +++ b/TablePro/Views/Results/DataGridView.swift @@ -19,11 +19,28 @@ struct RowVisualState: Equatable { let isDeleted: Bool let isInserted: Bool let modifiedColumns: Set + let highlight: RowHighlight + + init(isDeleted: Bool, isInserted: Bool, modifiedColumns: Set, highlight: RowHighlight = .none) { + self.isDeleted = isDeleted + self.isInserted = isInserted + self.modifiedColumns = modifiedColumns + self.highlight = highlight + } func isModified(columnIndex: Int) -> Bool { modifiedColumns.contains(columnIndex) } + func highlighted(_ highlight: RowHighlight) -> RowVisualState { + RowVisualState( + isDeleted: isDeleted, + isInserted: isInserted, + modifiedColumns: modifiedColumns, + highlight: highlight + ) + } + static let empty = RowVisualState(isDeleted: false, isInserted: false, modifiedColumns: []) } @@ -33,7 +50,20 @@ extension RowVisualState { @MainActor var tint: NSColor? { if isDeleted { return ThemeEngine.shared.colors.dataGrid.deleted } if isInserted { return ThemeEngine.shared.colors.dataGrid.inserted } - return nil + return highlight.rowColor?.washColor + } + + func cellHighlightColor(forColumn column: Int) -> HighlightColor? { + guard !isDeleted, !isInserted else { return nil } + return highlight.cellRule(forColumn: column)?.color + } + + func drawnHighlightRule(forColumn column: Int) -> HighlightRule? { + guard !isDeleted, !isInserted else { return nil } + if !isModified(columnIndex: column), let cellRule = highlight.cellRule(forColumn: column) { + return cellRule + } + return highlight.rowRule } } @@ -45,6 +75,7 @@ struct DataGridView: NSViewRepresentable { let isEditable: Bool var configuration: DataGridConfiguration = .init() var displayFormats: [ValueDisplayFormat?] = [] + var highlightRules: [HighlightRule] = [] var delegate: (any DataGridViewDelegate)? var layoutPersister: (any ColumnLayoutPersisting)? /// Whether a row may be dragged to a new position, and why not when it may not. @@ -145,6 +176,7 @@ struct DataGridView: NSViewRepresentable { let initialRows = tableRowsProvider() coordinator.rebuildColumnMetadataCache(from: initialRows) + coordinator.syncHighlightRules(highlightRules, tableRows: initialRows) coordinator.isRebuildingColumns = true let storedInitialLayout = coordinator.layoutDiscardingUnownedWidths( @@ -223,6 +255,7 @@ struct DataGridView: NSViewRepresentable { columns: latestRows.columns, valueFilteredIDsCount: coordinator.valueFilteredIDs?.count, displayFormats: displayFormats, + highlightRules: highlightRules, configuration: configuration, isEditable: isEditable, rowReorder: rowReorder, @@ -320,6 +353,7 @@ struct DataGridView: NSViewRepresentable { let liveColumnWidths = latestRows.columns.isEmpty ? [:] : coordinator.currentColumnWidths() coordinator.apply(configuration: configuration, isEditable: isEditable) let schemaChanged = coordinator.rebuildColumnMetadataCache(from: latestRows) + let highlightsChanged = coordinator.syncHighlightRules(highlightRules, tableRows: latestRows) let presentationChanges = coordinator.updateColumnPresentations(from: latestRows) let needsFullReload = structureChanged || schemaChanged @@ -385,6 +419,8 @@ struct DataGridView: NSViewRepresentable { coordinator.startBackgroundPrewarm() } else if displayFormatsChanged { coordinator.reloadAfterDisplayFormatChange() + } else if highlightsChanged { + coordinator.repaintVisibleRowDecorations() } } diff --git a/TablePro/Views/Results/DataGridViewDelegate.swift b/TablePro/Views/Results/DataGridViewDelegate.swift index f21ea8e748..68b40dc12f 100644 --- a/TablePro/Views/Results/DataGridViewDelegate.swift +++ b/TablePro/Views/Results/DataGridViewDelegate.swift @@ -38,6 +38,8 @@ protocol DataGridViewDelegate: AnyObject { func dataGridShowAllColumns() func dataGridColumnStructureMenuItems(forColumn dataColumnIndex: Int) -> [NSMenuItem] func dataGridRowStructureMenuItems(forRow displayRow: Int) -> [NSMenuItem] + func dataGridHighlightMenuItem(forRow displayRow: Int, dataColumn: Int) -> NSMenuItem? + func dataGridHighlightValuesMenuItem(forColumn dataColumnIndex: Int) -> NSMenuItem? func dataGridVisualState(forRow row: Int) -> RowVisualState? func dataGridRowView(for tableView: NSTableView, row: Int, coordinator: TableViewCoordinator) -> NSTableRowView? func dataGridEmptySpaceMenu() -> NSMenu? @@ -83,6 +85,8 @@ extension DataGridViewDelegate { func dataGridShowAllColumns() {} func dataGridColumnStructureMenuItems(forColumn dataColumnIndex: Int) -> [NSMenuItem] { [] } func dataGridRowStructureMenuItems(forRow displayRow: Int) -> [NSMenuItem] { [] } + func dataGridHighlightMenuItem(forRow displayRow: Int, dataColumn: Int) -> NSMenuItem? { nil } + func dataGridHighlightValuesMenuItem(forColumn dataColumnIndex: Int) -> NSMenuItem? { nil } func dataGridVisualState(forRow row: Int) -> RowVisualState? { nil } func dataGridRowView(for tableView: NSTableView, row: Int, coordinator: TableViewCoordinator) -> NSTableRowView? { nil } func dataGridEmptySpaceMenu() -> NSMenu? { nil } diff --git a/TablePro/Views/Results/Extensions/DataGridView+CellCommit.swift b/TablePro/Views/Results/Extensions/DataGridView+CellCommit.swift index ddb83f651c..285570a131 100644 --- a/TablePro/Views/Results/Extensions/DataGridView+CellCommit.swift +++ b/TablePro/Views/Results/Extensions/DataGridView+CellCommit.swift @@ -20,6 +20,7 @@ extension TableViewCoordinator { invalidateDisplayCache() visualIndex.updateRow(row, from: changeManager, displayIDs: displayIDs) + invalidateRowDecoration(displayRow: row) guard let tableColumnIndex = tableColumnIndex(for: columnIndex) else { return } redrawCells(rows: IndexSet(integer: row), tableColumnIndexes: IndexSet(integer: tableColumnIndex)) } diff --git a/TablePro/Views/Results/Extensions/DataGridView+Columns.swift b/TablePro/Views/Results/Extensions/DataGridView+Columns.swift index d454f915cd..a21e7c2109 100644 --- a/TablePro/Views/Results/Extensions/DataGridView+Columns.swift +++ b/TablePro/Views/Results/Extensions/DataGridView+Columns.swift @@ -142,13 +142,7 @@ extension TableViewCoordinator { func tableView(_ tableView: NSTableView, rowViewForRow row: Int) -> NSTableRowView? { if let delegateRowView = delegate?.dataGridRowView(for: tableView, row: row, coordinator: self) { - // Delegate-provided row views (e.g. StructureRowViewWithMenu) must still - // pick up the deleted/inserted/modified tint. Apply the visual state if - // the row view subclasses DataGridRowView; otherwise the delegate is - // responsible for its own visual state. - if let dataGridRow = delegateRowView as? DataGridRowView { - dataGridRow.applyVisualState(visualState(for: row)) - } + (delegateRowView as? DataGridRowView)?.invalidateVisualState() return delegateRowView } let rowView = (tableView.makeView(withIdentifier: Self.rowViewIdentifier, owner: nil) as? DataGridRowView) @@ -156,7 +150,7 @@ extension TableViewCoordinator { rowView.identifier = Self.rowViewIdentifier rowView.coordinator = self rowView.rowIndex = row - rowView.applyVisualState(visualState(for: row)) + rowView.invalidateVisualState() return rowView } } diff --git a/TablePro/Views/Results/Extensions/DataGridView+RowDecoration.swift b/TablePro/Views/Results/Extensions/DataGridView+RowDecoration.swift new file mode 100644 index 0000000000..b3c8e9538d --- /dev/null +++ b/TablePro/Views/Results/Extensions/DataGridView+RowDecoration.swift @@ -0,0 +1,77 @@ +// +// DataGridView+RowDecoration.swift +// TablePro +// + +import AppKit + +extension TableViewCoordinator { + func refreshVisibleRowVisualStates() { + guard let tableView else { return } + tableView.enumerateAvailableRowViews { rowView, _ in + (rowView as? DataGridRowView)?.invalidateVisualState() + } + } + + func refreshRowVisualState(at row: Int) { + guard let tableView, + let dataRowView = tableView.rowView(atRow: row, makeIfNecessary: false) as? DataGridRowView + else { return } + dataRowView.invalidateVisualState() + } + + @discardableResult + func syncHighlightRules(_ rules: [HighlightRule], tableRows: TableRows) -> Bool { + let key = HighlightRuleSet.Key(rules: rules, columns: tableRows.columns, columnTypes: tableRows.columnTypes) + let compiledChanged = highlightRuleSet.key != key + if compiledChanged { + highlightRuleSet = HighlightRuleSet( + rules: rules, + columns: tableRows.columns, + columnTypes: tableRows.columnTypes + ) + } + guard displayState.highlightRuleSetKey != key else { return compiledChanged } + displayState.highlightRuleSetKey = key + displayCache.clearHighlights() + return true + } + + func highlight(forDisplayRow displayIndex: Int) -> RowHighlight { + guard !highlightRuleSet.isEmpty, let row = displayRow(at: displayIndex) else { return .none } + if let cached = displayCache.highlight(forID: row.id) { return cached } + let resolved = highlightRuleSet.highlight(for: row.values) + displayCache.setHighlight(resolved, forID: row.id) + return resolved + } + + func highlightDescription(row: Int, columnIndex: Int) -> String? { + guard let rule = visualState(for: row).drawnHighlightRule(forColumn: columnIndex) else { return nil } + return HighlightRuleDescription.condition(of: rule, valueLimit: HighlightRuleDescription.menuValueLimit) + } + + func invalidateRowDecoration(displayRow row: Int) { + guard let tableView, row >= 0, row < tableView.numberOfRows else { return } + if let rowView = tableView.rowView(atRow: row, makeIfNecessary: false) as? DataGridRowView { + rowView.invalidateVisualState() + rowView.redrawCells() + } + repaintRowGutter(forRow: row) + } + + func repaintVisibleRowDecorations() { + guard let tableView else { return } + tableView.enumerateAvailableRowViews { rowView, _ in + guard let dataRowView = rowView as? DataGridRowView else { return } + dataRowView.invalidateVisualState() + dataRowView.redrawCells() + } + repaintRowGutter() + } + + private func repaintRowGutter(forRow row: Int) { + guard let rowGutter, let tableView else { return } + let band = rowGutter.convert(tableView.rect(ofRow: row), from: tableView) + rowGutter.setNeedsDisplay(NSRect(x: 0, y: band.minY, width: rowGutter.bounds.width, height: band.height)) + } +} diff --git a/TablePro/Views/Results/Extensions/DataGridView+Sort.swift b/TablePro/Views/Results/Extensions/DataGridView+Sort.swift index 2fd249928c..e0b9ea45f2 100644 --- a/TablePro/Views/Results/Extensions/DataGridView+Sort.swift +++ b/TablePro/Views/Results/Extensions/DataGridView+Sort.swift @@ -170,6 +170,11 @@ extension TableViewCoordinator { menu.addItem(clearAllItem) } + if let dataColumnIndex = dataColumnIndex(from: column.identifier), + let highlightItem = delegate?.dataGridHighlightValuesMenuItem(forColumn: dataColumnIndex) { + menu.addItem(highlightItem) + } + if let dataColumnIndex = dataColumnIndex(from: column.identifier) { addDisplayFormatMenu(to: menu, dataColumnIndex: dataColumnIndex, tableRows: tableRows) } diff --git a/TablePro/Views/Results/ResultStatusBar.swift b/TablePro/Views/Results/ResultStatusBar.swift index 35d02b1ad1..c2fa3a6478 100644 --- a/TablePro/Views/Results/ResultStatusBar.swift +++ b/TablePro/Views/Results/ResultStatusBar.swift @@ -24,6 +24,7 @@ struct ResultStatusBar: View { let snapshot: StatusBarSnapshot let filterState: TabFilterState let columnState: StatusBarColumnState + let highlightState: StatusBarHighlightState let paginationCallbacks: PaginationCallbacks let structureFooter: StructureFooterCapability let execution: ExecutionReadout @@ -37,6 +38,7 @@ struct ResultStatusBar: View { let onStructureRemove: () -> Void @State private var showColumnPopover = false + @State private var showHighlightPopover = false var body: some View { HStack(spacing: StatusBarChrome.clusterSpacing) { @@ -50,6 +52,15 @@ struct ResultStatusBar: View { .statusBarChrome() .onChange(of: snapshot.tabId) { _, _ in showColumnPopover = false + showHighlightPopover = false + } + .onChange(of: showHighlightPopover) { _, isShown in + guard !isShown else { return } + highlightState.onDismiss() + } + .onChange(of: highlightPresentation) { previous, current in + guard previous.tabId == current.tabId, model.controls.showsHighlightRules else { return } + showHighlightPopover = true } } @@ -159,6 +170,9 @@ struct ResultStatusBar: View { if model.controls.showsColumns { columnsButton } + if model.controls.showsHighlightRules { + highlightButton + } if model.controls.showsFilters { filtersToggle } @@ -240,6 +254,43 @@ struct ResultStatusBar: View { } } + private var highlightButton: some View { + Button { + showHighlightPopover.toggle() + } label: { + Label { + Text("Highlight Rules") + } icon: { + Image(systemName: "highlighter") + } + } + .labelStyle(.iconOnly) + .controlSize(.small) + .disabled(highlightState.columns.isEmpty) + .help(String(localized: "Highlight Rules")) + .accessibilityLabel(String(localized: "Highlight Rules")) + .accessibilityValue(highlightAccessibilityValue) + .accessibilityIdentifier("result-status-highlight") + .popover(isPresented: $showHighlightPopover, arrowEdge: .top) { + HighlightRulesPopover( + columns: highlightState.columns, + rules: highlightState.rules, + isPersisted: highlightState.isPersisted, + onChange: highlightState.onChange + ) + } + } + + private var highlightPresentation: HighlightPresentationRequest { + HighlightPresentationRequest(tabId: snapshot.tabId, count: highlightState.presentationRequest) + } + + private var highlightAccessibilityValue: String { + let count = highlightState.activeRuleCount + guard count > 0 else { return String(localized: "No highlight rules") } + return String(format: String(localized: "%d rules"), count) + } + private var filtersToggle: some View { Toggle(isOn: Binding(get: { filterState.isVisible }, set: { _ in onToggleFilters() })) { Label { diff --git a/TablePro/Views/Results/ResultStatusInputs.swift b/TablePro/Views/Results/ResultStatusInputs.swift index 5473a9d500..3c060b58d1 100644 --- a/TablePro/Views/Results/ResultStatusInputs.swift +++ b/TablePro/Views/Results/ResultStatusInputs.swift @@ -16,6 +16,24 @@ struct PaginationCallbacks { let onRequestExactCount: () -> Void } +struct HighlightPresentationRequest: Equatable { + let tabId: UUID? + let count: Int +} + +struct StatusBarHighlightState { + let rules: [HighlightRule] + let columns: [String] + let isPersisted: Bool + let presentationRequest: Int + let onChange: ([HighlightRule]) -> Void + let onDismiss: () -> Void + + var activeRuleCount: Int { + rules.filter { $0.isEnabled && $0.isValid }.count + } +} + struct StatusBarColumnState { let hidden: Set let columns: [GridColumnEntry] diff --git a/TablePro/Views/Structure/StructureGridDelegate.swift b/TablePro/Views/Structure/StructureGridDelegate.swift index abc9f5bd9f..319a9d5f5c 100644 --- a/TablePro/Views/Structure/StructureGridDelegate.swift +++ b/TablePro/Views/Structure/StructureGridDelegate.swift @@ -474,13 +474,6 @@ final class StructureGridDelegate: DataGridViewDelegate { rowView.isStructureEditable = connection.type.supportsSchemaEditing let src = sourceRow(for: row) - // Don't set `isDeleted` / visual state here. `DataGridView+Columns` - // calls `applyVisualState(visualState(for: row))` on every row view it - // returns from `tableView(_:rowViewForRow:)`. Setting it twice is a - // smell that previously hid the bug: when `applyVisualState` was a - // tint-only setter, this line was the only place the menu's - // `isDeleted` flag was assigned, and it was assigned only on row-view - // creation. Single source of truth now is `DataGridRowView.visualState`. if selectedTab == .foreignKeys, src < structureChangeManager.workingForeignKeys.count { rowView.referencedTableName = structureChangeManager.workingForeignKeys[src].referencedTable diff --git a/TablePro/Views/Structure/StructureRowViewWithMenu.swift b/TablePro/Views/Structure/StructureRowViewWithMenu.swift index 7cabb81da2..6cd092ac5c 100644 --- a/TablePro/Views/Structure/StructureRowViewWithMenu.swift +++ b/TablePro/Views/Structure/StructureRowViewWithMenu.swift @@ -11,8 +11,8 @@ import AppKit /// Row view providing a context menu tailored to the Structure tab. Inherits /// selection/emphasis cell invalidation, deleted/inserted-row tint, and the /// `RowVisualState` source-of-truth from `DataGridRowView`. The context menu -/// reads `visualState.isDeleted` directly, so a single `applyVisualState` call -/// updates both the tint and the menu without a shadow flag to keep in sync. +/// reads the same live `visualState` the tint is drawn from, so the two cannot +/// disagree. final class StructureRowViewWithMenu: DataGridRowView { var structureTab: StructureTab = .columns var isStructureEditable: Bool = true diff --git a/TableProMobile/TableProMobile/Assets.xcassets/weaviate-icon.imageset/Contents.json b/TableProMobile/TableProMobile/Assets.xcassets/weaviate-icon.imageset/Contents.json new file mode 100644 index 0000000000..fcfa2ae760 --- /dev/null +++ b/TableProMobile/TableProMobile/Assets.xcassets/weaviate-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "weaviate.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/TableProMobile/TableProMobile/Assets.xcassets/weaviate-icon.imageset/weaviate.svg b/TableProMobile/TableProMobile/Assets.xcassets/weaviate-icon.imageset/weaviate.svg new file mode 100644 index 0000000000..fafccd810f --- /dev/null +++ b/TableProMobile/TableProMobile/Assets.xcassets/weaviate-icon.imageset/weaviate.svg @@ -0,0 +1 @@ +Weaviate diff --git a/TableProMobile/TableProWidget/Assets.xcassets/weaviate-icon.imageset/Contents.json b/TableProMobile/TableProWidget/Assets.xcassets/weaviate-icon.imageset/Contents.json new file mode 100644 index 0000000000..fcfa2ae760 --- /dev/null +++ b/TableProMobile/TableProWidget/Assets.xcassets/weaviate-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images": [ + { + "filename": "weaviate.svg", + "idiom": "universal" + } + ], + "info": { + "author": "xcode", + "version": 1 + }, + "properties": { + "preserves-vector-representation": true, + "template-rendering-intent": "template" + } +} diff --git a/TableProMobile/TableProWidget/Assets.xcassets/weaviate-icon.imageset/weaviate.svg b/TableProMobile/TableProWidget/Assets.xcassets/weaviate-icon.imageset/weaviate.svg new file mode 100644 index 0000000000..fafccd810f --- /dev/null +++ b/TableProMobile/TableProWidget/Assets.xcassets/weaviate-icon.imageset/weaviate.svg @@ -0,0 +1 @@ +Weaviate diff --git a/TableProMobile/TableProWidget/Helpers/DatabaseTypeStyle.swift b/TableProMobile/TableProWidget/Helpers/DatabaseTypeStyle.swift index c5e91110f7..3c35e2db30 100644 --- a/TableProMobile/TableProWidget/Helpers/DatabaseTypeStyle.swift +++ b/TableProMobile/TableProWidget/Helpers/DatabaseTypeStyle.swift @@ -22,6 +22,7 @@ enum DatabaseTypeStyle { case "DynamoDB": return "dynamodb-icon" case "BigQuery": return "bigquery-icon" case "Spanner": return "spanner-icon" + case "Weaviate": return "weaviate-icon" default: return "externaldrive" } } diff --git a/TableProTests/Core/DataGrid/RowDisplayCacheTests.swift b/TableProTests/Core/DataGrid/RowDisplayCacheTests.swift index fdf38700ae..ae2b3a0286 100644 --- a/TableProTests/Core/DataGrid/RowDisplayCacheTests.swift +++ b/TableProTests/Core/DataGrid/RowDisplayCacheTests.swift @@ -191,6 +191,27 @@ struct RowDisplayCacheTests { #expect(cache.box(forID: .existing(1)) == nil) } + @Test("A row's highlight lives and dies with its formatted text") + func highlightSharesTheTextLifetime() { + let cache = RowDisplayCache() + let highlight = RowHighlight(rowRule: HighlightRule(columnName: "c", value: "x"), cellRules: [:]) + cache.setBox(makeBox(["x"]), forID: .existing(1)) + cache.setHighlight(highlight, forID: .existing(1)) + cache.setHighlight(highlight, forID: .existing(2)) + + cache.clearValues(forID: .existing(1)) + #expect(cache.highlight(forID: .existing(1)) == nil) + #expect(cache.highlight(forID: .existing(2)) == highlight) + + cache.clearHighlights() + #expect(cache.highlight(forID: .existing(2)) == nil) + #expect(cache.box(forID: .existing(1)) != nil) + + cache.setHighlight(highlight, forID: .existing(3)) + cache.removeAll() + #expect(cache.highlight(forID: .existing(3)) == nil) + } + @Test("Inserted row IDs of both kinds round-trip") func mixedRowIDKinds() { let cache = RowDisplayCache() diff --git a/TableProTests/Core/Database/GeometryWKBParserTests.swift b/TableProTests/Core/Database/GeometryWKBParserTests.swift index 42061f9650..ba80e74719 100644 --- a/TableProTests/Core/Database/GeometryWKBParserTests.swift +++ b/TableProTests/Core/Database/GeometryWKBParserTests.swift @@ -77,7 +77,6 @@ private func wkbPolygon(_ rings: [[(Double, Double)]]) -> [UInt8] { @Suite("GeometryWKBParser") struct GeometryWKBParserTests { - @Test("Point: little-endian binary produces WKT") func testPoint() { let data = mysqlGeometry(wkb: wkbPoint(1.0, 2.0)) @@ -177,246 +176,3 @@ struct GeometryWKBParserTests { #expect(result == "POINT(100.0 200.0)") } } - -// MARK: - Local Copy of GeometryWKBParser - -// Copied from Plugins/MySQLDriverPlugin/GeometryWKBParser.swift -// because the plugin is a bundle target and cannot be imported with @testable import. - -private enum GeometryWKBParser { - static func parse(_ data: Data) -> String { - guard data.count >= 9 else { - return hexString(data) - } - - let wkbData = data.dropFirst(4) - var offset = wkbData.startIndex - return parseWKBGeometry(wkbData, offset: &offset) ?? hexString(data) - } - - static func parse(_ buffer: UnsafeRawBufferPointer) -> String { - let data = Data(buffer) - return parse(data) - } - - private static func parseWKBGeometry(_ data: Data.SubSequence, offset: inout Data.Index) -> String? { - guard offset < data.endIndex else { return nil } - - let byteOrder = data[offset] - let littleEndian = byteOrder == 0x01 - offset = data.index(after: offset) - - guard let typeCode = readUInt32(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - - switch typeCode { - case 1: - return parsePoint(data, offset: &offset, littleEndian: littleEndian) - case 2: - return parseLineString(data, offset: &offset, littleEndian: littleEndian) - case 3: - return parsePolygon(data, offset: &offset, littleEndian: littleEndian) - case 4: - return parseMultiPoint(data, offset: &offset, littleEndian: littleEndian) - case 5: - return parseMultiLineString(data, offset: &offset, littleEndian: littleEndian) - case 6: - return parseMultiPolygon(data, offset: &offset, littleEndian: littleEndian) - case 7: - return parseGeometryCollection(data, offset: &offset, littleEndian: littleEndian) - default: - return nil - } - } - - private static func parsePoint( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> String? { - guard let x = readFloat64(data, offset: &offset, littleEndian: littleEndian), - let y = readFloat64(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - return "POINT(\(formatCoord(x)) \(formatCoord(y)))" - } - - private static func parseLineString( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> String? { - guard let points = readPointList(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - return "LINESTRING(\(points))" - } - - private static func parsePolygon( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> String? { - guard let numRings = readUInt32(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - var rings: [String] = [] - for _ in 0 ..< numRings { - guard let points = readPointList(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - rings.append("(\(points))") - } - return "POLYGON(\(rings.joined(separator: ", ")))" - } - - private static func parseMultiPoint( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> String? { - guard let numGeoms = readUInt32(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - var points: [String] = [] - for _ in 0 ..< numGeoms { - guard let geom = parseWKBGeometry(data, offset: &offset) else { return nil } - if geom.hasPrefix("POINT("), geom.hasSuffix(")") { - let ns = geom as NSString - points.append(ns.substring(with: NSRange(location: 6, length: ns.length - 7))) - } else { - points.append(geom) - } - } - return "MULTIPOINT(\(points.joined(separator: ", ")))" - } - - private static func parseMultiLineString( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> String? { - guard let numGeoms = readUInt32(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - var lineStrings: [String] = [] - for _ in 0 ..< numGeoms { - guard let geom = parseWKBGeometry(data, offset: &offset) else { return nil } - if geom.hasPrefix("LINESTRING("), geom.hasSuffix(")") { - let ns = geom as NSString - lineStrings.append("(\(ns.substring(with: NSRange(location: 11, length: ns.length - 12))))") - } else { - lineStrings.append(geom) - } - } - return "MULTILINESTRING(\(lineStrings.joined(separator: ", ")))" - } - - private static func parseMultiPolygon( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> String? { - guard let numGeoms = readUInt32(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - var polygons: [String] = [] - for _ in 0 ..< numGeoms { - guard let geom = parseWKBGeometry(data, offset: &offset) else { return nil } - if geom.hasPrefix("POLYGON("), geom.hasSuffix(")") { - let ns = geom as NSString - polygons.append("(\(ns.substring(with: NSRange(location: 8, length: ns.length - 9))))") - } else { - polygons.append(geom) - } - } - return "MULTIPOLYGON(\(polygons.joined(separator: ", ")))" - } - - private static func parseGeometryCollection( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> String? { - guard let numGeoms = readUInt32(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - var geoms: [String] = [] - for _ in 0 ..< numGeoms { - guard let geom = parseWKBGeometry(data, offset: &offset) else { return nil } - geoms.append(geom) - } - return "GEOMETRYCOLLECTION(\(geoms.joined(separator: ", ")))" - } - - private static func readUInt32( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> UInt32? { - let endOffset = data.index(offset, offsetBy: 4, limitedBy: data.endIndex) ?? data.endIndex - guard data.distance(from: offset, to: endOffset) == 4 else { return nil } - - let bytes = data[offset ..< endOffset] - offset = endOffset - - if littleEndian { - return bytes.withUnsafeBytes { $0.loadUnaligned(as: UInt32.self).littleEndian } - } else { - return bytes.withUnsafeBytes { $0.loadUnaligned(as: UInt32.self).bigEndian } - } - } - - private static func readFloat64( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> Double? { - let endOffset = data.index(offset, offsetBy: 8, limitedBy: data.endIndex) ?? data.endIndex - guard data.distance(from: offset, to: endOffset) == 8 else { return nil } - - let bytes = data[offset ..< endOffset] - offset = endOffset - - let bits: UInt64 - if littleEndian { - bits = bytes.withUnsafeBytes { $0.loadUnaligned(as: UInt64.self).littleEndian } - } else { - bits = bytes.withUnsafeBytes { $0.loadUnaligned(as: UInt64.self).bigEndian } - } - return Double(bitPattern: bits) - } - - private static func readPointList( - _ data: Data.SubSequence, - offset: inout Data.Index, - littleEndian: Bool - ) -> String? { - guard let numPoints = readUInt32(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - var coords: [String] = [] - for _ in 0 ..< numPoints { - guard let x = readFloat64(data, offset: &offset, littleEndian: littleEndian), - let y = readFloat64(data, offset: &offset, littleEndian: littleEndian) else { - return nil - } - coords.append("\(formatCoord(x)) \(formatCoord(y))") - } - return coords.joined(separator: ", ") - } - - private static func formatCoord(_ value: Double) -> String { - if value == value.rounded() && abs(value) < 1e15 { - return String(format: "%.1f", value) - } - let formatted = String(format: "%.15g", value) - return formatted - } - - static func hexString(_ data: Data) -> String { - if data.isEmpty { return "" } - return "0x" + data.map { String(format: "%02X", $0) }.joined() - } -} diff --git a/TableProTests/Core/Menu/MainMenuBuilderTests.swift b/TableProTests/Core/Menu/MainMenuBuilderTests.swift index 03d66c10d8..d088d362d4 100644 --- a/TableProTests/Core/Menu/MainMenuBuilderTests.swift +++ b/TableProTests/Core/Menu/MainMenuBuilderTests.swift @@ -419,6 +419,17 @@ struct MainMenuValidationTests { #expect(enabled(#selector(MainSplitViewController.toggleFilterBar(_:)), context)) } + @Test("Highlight Rules needs a connected data grid with columns") + func highlightRulesNeedsDataGrid() { + var context = MenuValidationContext() + context.canPresentHighlightRules = true + #expect(!enabled(#selector(MainSplitViewController.showHighlightRules(_:)), context)) + context.isConnected = true + #expect(enabled(#selector(MainSplitViewController.showHighlightRules(_:)), context)) + context.canPresentHighlightRules = false + #expect(!enabled(#selector(MainSplitViewController.showHighlightRules(_:)), context)) + } + @Test("Capability flags gate driver-specific commands") func capabilitiesGateCommands() { var context = MenuValidationContext() @@ -445,6 +456,8 @@ struct MainMenuValidationTests { private func capableContext() -> MenuValidationContext { var context = MenuValidationContext() context.canUseTableResultCommands = true + context.canPresentHighlightRules = true + context.canNavigatePages = true context.isQueryTab = true context.hasResultRows = true context.hasQueryText = true @@ -489,7 +502,8 @@ struct MainMenuValidationTests { #selector(MainSplitViewController.openContainerSwitcher(_:)), #selector(MainSplitViewController.showServerDashboard(_:)), #selector(MainSplitViewController.showUsersAndRoles(_:)), - #selector(MainSplitViewController.toggleFilterBar(_:)) + #selector(MainSplitViewController.toggleFilterBar(_:)), + #selector(MainSplitViewController.showHighlightRules(_:)) ] } diff --git a/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift b/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift index ef6d5a6e67..c5300773f3 100644 --- a/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift +++ b/TableProTests/Core/Plugins/PluginMetadataRegistryTypeCountTests.swift @@ -11,7 +11,7 @@ import Testing /// this registry, `docs/snippets/driver-counts.mdx`, and the marketing site. Nothing at runtime /// reconciles them, and by August 2026 they read 28, 27 and 25 at once. /// -/// The answer is 34, and the reason it once read 28 is worth keeping. Turso is served by +/// The answer is 35, and the reason it once read 28 is worth keeping. Turso is served by /// the libSQL plugin and was the only alias in `reverseTypeIndex` with no curated entry of its /// own, so it was the only type the picker could not offer before its plugin was installed. /// ScyllaDB is the shape every other alias already had: an alias of Cassandra with a curated @@ -24,7 +24,7 @@ import Testing /// `docs/scripts/check-docs-against-source.py` reads the registry and holds the docs half. /// /// The count is taken from the built-in defaults rather than from `allRegisteredTypeIds()`. -/// Both answer 34 under XCTest, where no plugin bundle ever loads, but the registry is a +/// Both answer 35 under XCTest, where no plugin bundle ever loads, but the registry is a /// process-global singleton and suites that register a synthetic type run alongside this one. @MainActor @Suite("PluginMetadataRegistry engine count") @@ -34,7 +34,7 @@ struct PluginMetadataRegistryTypeCountTests { "CockroachDB", "Dameng", "Databend", "DuckDB", "DynamoDB", "Elasticsearch", "etcd", "Kafka", "libSQL", "MariaDB", "MongoDB", "MySQL", "Oracle", "PGlite", "PostgreSQL", "Redis", "Redshift", "ScyllaDB", "Snowflake", "Spanner", "SQL Server", "SQLite", "SurrealDB", "Teradata", "TiDB", "Trino", - "Turso", "Typesense" + "Turso", "Typesense", "Weaviate" ] private static func builtInTypeIds() -> Set { @@ -43,10 +43,10 @@ struct PluginMetadataRegistryTypeCountTests { return Set(curated + registry) } - @Test("The app ships 34 database types before any plugin loads") + @Test("The app ships 35 database types before any plugin loads") func builtInDefaultsCoverTwentyNineTypes() { let ids = Self.builtInTypeIds() - #expect(ids.count == 34) + #expect(ids.count == 35) #expect(ids == Self.expectedTypeIds) } diff --git a/TableProTests/Core/Services/Highlight/HighlightConditionTests.swift b/TableProTests/Core/Services/Highlight/HighlightConditionTests.swift new file mode 100644 index 0000000000..c18f9d2f9f --- /dev/null +++ b/TableProTests/Core/Services/Highlight/HighlightConditionTests.swift @@ -0,0 +1,147 @@ +// +// HighlightConditionTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@Suite("Highlight condition matching") +struct HighlightConditionTests { + private func matches( + _ value: PluginCellValue, + _ filterOperator: FilterOperator, + _ operand: String = "", + second: String? = nil, + caseSensitive: Bool? = nil, + type: ColumnType? = .text(rawType: "VARCHAR") + ) -> Bool { + let rule = HighlightRule( + columnName: "c", + filterOperator: filterOperator, + value: operand, + secondValue: second, + isCaseSensitive: caseSensitive + ) + return HighlightCondition(rule: rule, columnType: type).matches(value) + } + + @Test("Equality on text is exact and case-sensitive by default") + func textEquality() { + #expect(matches("paid", .equal, "paid")) + #expect(!matches("Paid", .equal, "paid")) + #expect(matches("Paid", .equal, "paid", caseSensitive: false)) + #expect(matches("pending", .notEqual, "paid")) + #expect(!matches("007", .equal, "7")) + } + + @Test("A padded value matches exactly as stored, so a quick rule matches its own cell") + func paddedValuesMatchAsStored() { + #expect(matches("abc ", .equal, "abc ", type: .text(rawType: "CHAR(10)"))) + #expect(!matches("abc", .equal, "abc ", type: .text(rawType: "CHAR(10)"))) + #expect(matches(" ", .equal, " ")) + #expect(matches("42", .equal, " 42 ", type: .integer(rawType: "INT"))) + } + + @Test("NULL fails every comparison and matches only IS NULL and IS EMPTY") + func nullSemantics() { + #expect(matches(.null, .isNull)) + #expect(matches(.null, .isEmpty)) + #expect(!matches(.null, .isNotNull)) + #expect(!matches(.null, .notEqual, "paid")) + #expect(!matches(.null, .greaterThan, "1", type: .integer(rawType: "INT"))) + #expect(!matches(.null, .notContains, "x")) + #expect(!matches(.null, .notInList, "a, b")) + } + + @Test("The literal NULL means IS NULL on a column that is not text") + func nullLiteral() { + #expect(matches(.null, .equal, "NULL", type: .integer(rawType: "INT"))) + #expect(!matches("5", .equal, "NULL", type: .integer(rawType: "INT"))) + #expect(matches("5", .notEqual, "null", type: .integer(rawType: "INT"))) + #expect(!matches(.null, .equal, "NULL")) + #expect(matches("NULL", .equal, "NULL")) + } + + @Test("Numbers compare numerically on a numeric column") + func numericColumns() { + let integer = ColumnType.integer(rawType: "INT") + #expect(matches("1000", .greaterThan, "999", type: integer)) + #expect(matches("1.0", .equal, "1", type: .decimal(rawType: "DECIMAL"))) + #expect(matches("5", .between, "1", second: "10", type: integer)) + #expect(!matches("11", .between, "1", second: "10", type: integer)) + #expect(matches("10", .lessOrEqual, "10", type: integer)) + } + + @Test("Ordering on text compares numerically only when both sides are numbers") + func orderingOnText() { + #expect(matches("1000", .greaterThan, "999")) + #expect(matches("banana", .greaterThan, "apple")) + #expect(!matches("apple", .greaterThan, "banana")) + } + + @Test("Boolean columns accept every spelling of true and false") + func booleans() { + let boolean = ColumnType.boolean(rawType: "BOOLEAN") + #expect(matches("t", .equal, "true", type: boolean)) + #expect(matches("1", .equal, "yes", type: boolean)) + #expect(matches("false", .equal, "0", type: boolean)) + #expect(!matches("f", .equal, "true", type: boolean)) + #expect(matches("1", .equal, "true", type: .integer(rawType: "TINYINT(1)"))) + } + + @Test("Pattern operators ignore case by default and honour Match Case") + func patterns() { + #expect(matches("Hello World", .contains, "world")) + #expect(!matches("Hello World", .contains, "world", caseSensitive: true)) + #expect(matches("Hello", .startsWith, "he")) + #expect(matches("Hello", .endsWith, "LLO")) + #expect(!matches("Hello", .endsWith, "hel")) + #expect(matches("Hello", .notContains, "xyz")) + } + + @Test("Empty means NULL or an empty string on text, and only NULL elsewhere") + func emptiness() { + #expect(matches("", .isEmpty)) + #expect(!matches("x", .isEmpty)) + #expect(matches("x", .isNotEmpty)) + #expect(!matches("", .isNotEmpty)) + #expect(!matches("", .isEmpty, type: .integer(rawType: "INT"))) + #expect(matches("", .isNotEmpty, type: .integer(rawType: "INT"))) + } + + @Test("IN and NOT IN split on commas and trim each item") + func lists() { + #expect(matches("b", .inList, "a, b ,c")) + #expect(!matches("d", .inList, "a, b, c")) + #expect(matches("d", .notInList, "a, b, c")) + #expect(!matches("a", .notInList, "a, b")) + #expect(matches(.null, .inList, "a, NULL", type: .integer(rawType: "INT"))) + } + + @Test("A regular expression searches the value, and an invalid one matches nothing") + func regex() { + #expect(matches("order-42", .regex, "\\d+$")) + #expect(!matches("order", .regex, "\\d+$")) + #expect(matches("ABC", .regex, "abc", caseSensitive: false)) + #expect(!matches("anything", .regex, "(unclosed")) + } + + @Test("A binary value matches only IS NULL and IS NOT NULL") + func binary() { + let bytes = PluginCellValue.bytes(Data([0x01, 0x02])) + #expect(matches(bytes, .isNotNull)) + #expect(!matches(bytes, .isNull)) + #expect(!matches(bytes, .equal, "0x0102")) + #expect(!matches(bytes, .contains, "01")) + } + + @Test("A search past the cap only looks at the leading part of a very long value") + func searchIsCapped() { + let long = String(repeating: "a", count: HighlightCondition.searchLimit + 50) + "needle" + #expect(!matches(.text(long), .contains, "needle")) + #expect(matches(.text("needle" + long), .contains, "needle")) + } +} diff --git a/TableProTests/Core/Services/Highlight/HighlightRuleSetTests.swift b/TableProTests/Core/Services/Highlight/HighlightRuleSetTests.swift new file mode 100644 index 0000000000..b8f5d9713e --- /dev/null +++ b/TableProTests/Core/Services/Highlight/HighlightRuleSetTests.swift @@ -0,0 +1,176 @@ +// +// HighlightRuleSetTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@Suite("Highlight rule set") +struct HighlightRuleSetTests { + private let columns = ["id", "status", "total"] + private let types: [ColumnType] = [.integer(rawType: "INT"), .text(rawType: "VARCHAR"), .decimal(rawType: "DECIMAL")] + + private func row(_ values: PluginCellValue...) -> ContiguousArray { + ContiguousArray(values) + } + + @Test("The first matching row rule sets the row's color, and reordering flips it") + func firstRowRuleWins() { + let paid = HighlightRule(columnName: "status", value: "paid", color: .green) + let big = HighlightRule(columnName: "total", filterOperator: .greaterThan, value: "100", color: .red) + let values = row("1", "paid", "500") + + let paidFirst = HighlightRuleSet(rules: [paid, big], columns: columns, columnTypes: types) + let bigFirst = HighlightRuleSet(rules: [big, paid], columns: columns, columnTypes: types) + + #expect(paidFirst.highlight(for: values).rowColor == .green) + #expect(bigFirst.highlight(for: values).rowColor == .red) + } + + @Test("A cell rule colours its own column and leaves the row rule in place") + func cellRulesColourTheirColumn() { + let rowRule = HighlightRule(columnName: "status", value: "paid", color: .green) + let cellRule = HighlightRule( + columnName: "total", filterOperator: .greaterThan, value: "100", color: .red, target: .cell + ) + let highlight = HighlightRuleSet(rules: [rowRule, cellRule], columns: columns, columnTypes: types) + .highlight(for: row("1", "paid", "500")) + + #expect(highlight.rowColor == .green) + #expect(highlight.cellRule(forColumn: 2)?.color == .red) + #expect(highlight.cellRule(forColumn: 1) == nil) + #expect(highlight.describingRule(forColumn: 2) == cellRule) + #expect(highlight.describingRule(forColumn: 0) == rowRule) + } + + @Test("Disabled and incomplete rules never match") + func disabledAndIncompleteRules() { + let disabled = HighlightRule(isEnabled: false, columnName: "status", value: "paid", color: .green) + let incomplete = HighlightRule(columnName: "status", value: "", color: .red) + let set = HighlightRuleSet(rules: [disabled, incomplete], columns: columns, columnTypes: types) + + #expect(set.isEmpty) + #expect(set.highlight(for: row("1", "paid", "5")) == .none) + } + + @Test("A rule whose column is not in the result is reported, not dropped") + func missingColumnIsUnresolved() { + let rule = HighlightRule(columnName: "archived", filterOperator: .isNotNull, color: .gray) + let set = HighlightRuleSet(rules: [rule], columns: columns, columnTypes: types) + + #expect(set.unresolvedRuleIDs == [rule.id]) + #expect(set.highlight(for: row("1", "paid", "5")) == .none) + } + + @Test("A duplicated column name resolves by occurrence") + func duplicateColumnsResolveByOccurrence() { + let duplicated = ["status", "status"] + let textTypes: [ColumnType] = [.text(rawType: nil), .text(rawType: nil)] + let second = HighlightRule( + columnName: "status", columnOccurrence: 1, value: "paid", color: .blue, target: .cell + ) + let highlight = HighlightRuleSet(rules: [second], columns: duplicated, columnTypes: textTypes) + .highlight(for: row("paid", "paid")) + + #expect(highlight.cellRule(forColumn: 0) == nil) + #expect(highlight.cellRule(forColumn: 1)?.color == .blue) + #expect(HighlightRuleSet.occurrence(ofColumnAt: 1, in: duplicated) == 1) + #expect(HighlightRuleSet.columnIndex(named: "status", occurrence: 2, in: duplicated) == nil) + } +} + +@Suite("Highlight rule descriptions and quick rules") +@MainActor +struct HighlightRuleDescriptionTests { + @Test("A comparison reads as column, symbol and quoted value") + func comparisonTitle() { + let rule = HighlightRule(columnName: "status", value: "paid") + #expect(HighlightRuleDescription.condition(of: rule) == "status = “paid”") + } + + @Test("An operator without a value reads as its name") + func valuelessTitle() { + let rule = HighlightRule(columnName: "notes", filterOperator: .isNull) + #expect(HighlightRuleDescription.condition(of: rule) == "notes is NULL") + } + + @Test("A long value is truncated in menu titles only") + func longValuesTruncate() { + let value = String(repeating: "x", count: 50) + let rule = HighlightRule(columnName: "notes", value: value) + let title = HighlightRuleDescription.condition(of: rule, valueLimit: HighlightRuleDescription.menuValueLimit) + + #expect(title == "notes = “\(String(repeating: "x", count: 32))…”") + #expect(HighlightRuleDescription.condition(of: rule).contains(value)) + } + + @Test("The quick rule follows the clicked cell's raw value") + func quickRuleFromCell() { + let text = HighlightMenuBuilder.quickRule( + columnName: "status", columnOccurrence: 0, columnType: .text(rawType: "VARCHAR"), value: "paid", target: .row, color: .green + ) + let null = HighlightMenuBuilder.quickRule( + columnName: "status", columnOccurrence: 0, columnType: .text(rawType: "VARCHAR"), value: .null, target: .cell, color: .red + ) + let empty = HighlightMenuBuilder.quickRule( + columnName: "status", columnOccurrence: 0, columnType: .text(rawType: "VARCHAR"), value: "", target: .row, color: .red + ) + let binary = HighlightMenuBuilder.quickRule( + columnName: "blob", columnOccurrence: 0, columnType: .text(rawType: "VARCHAR"), value: .bytes(Data([1])), target: .row, color: .red + ) + + #expect(text?.filterOperator == .equal) + #expect(text?.value == "paid") + #expect(null?.filterOperator == .isNull) + #expect(null?.target == .cell) + #expect(empty?.filterOperator == .isEmpty) + #expect(binary == nil) + } + + @Test("A quick rule cannot be built from a value the rule would read as NULL") + func quickRuleRefusesTheNullKeyword() { + let json = HighlightMenuBuilder.quickRule( + columnName: "payload", columnOccurrence: 0, columnType: .json(rawType: "JSONB"), + value: "null", target: .row, color: .red + ) + let text = HighlightMenuBuilder.quickRule( + columnName: "note", columnOccurrence: 0, columnType: .text(rawType: "VARCHAR"), + value: "null", target: .row, color: .red + ) + + #expect(json == nil) + #expect(text?.value == "null") + } + + @Test("Menu sections name the target and the condition") + func sectionTitles() { + let row = HighlightRule(columnName: "status", value: "paid", target: .row) + let cell = HighlightRule(columnName: "status", value: "paid", target: .cell) + + #expect(HighlightMenuBuilder.sectionTitle(for: row) == "Rows Where status = “paid”") + #expect(HighlightMenuBuilder.sectionTitle(for: cell) == "Cells Where status = “paid”") + } + + @Test("Two rules share a condition regardless of their color") + func sameCondition() { + let green = HighlightRule(columnName: "status", value: "paid", color: .green) + var red = green + red.color = .red + let cell = HighlightRule(columnName: "status", value: "paid", color: .green, target: .cell) + + #expect(HighlightRule(columnName: "status", value: "paid", color: .red).hasSameCondition(as: green)) + #expect(red.hasSameCondition(as: green)) + #expect(!cell.hasSameCondition(as: green)) + } + + @Test("Duplicated columns get a numbered label") + func columnOptions() { + let options = HighlightColumnOption.options(for: ["id", "status", "status"]) + + #expect(options.map(\.label) == ["id", "status (1)", "status (2)"]) + #expect(options.map(\.occurrence) == [0, 0, 1]) + } +} diff --git a/TableProTests/Core/Storage/HighlightRuleStorageTests.swift b/TableProTests/Core/Storage/HighlightRuleStorageTests.swift new file mode 100644 index 0000000000..f970093cb6 --- /dev/null +++ b/TableProTests/Core/Storage/HighlightRuleStorageTests.swift @@ -0,0 +1,132 @@ +// +// HighlightRuleStorageTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@Suite("Highlight rule storage") +@MainActor +struct HighlightRuleStorageTests { + private let directory: URL + private let connectionId = UUID() + + init() { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("HighlightRuleStorageTests-\(UUID().uuidString)", isDirectory: true) + } + + private func scope(table: String, database: String? = "shop", schema: String? = "public") -> TableScope { + TableScope(connectionId: connectionId, database: database, schema: schema, table: table) + } + + private var fileURL: URL { + directory.appendingPathComponent("\(connectionId.uuidString).json") + } + + @Test("Rules round-trip through a fresh store") + func roundTrip() { + let rules = [ + HighlightRule(columnName: "status", value: "paid", color: .green), + HighlightRule(columnName: "total", filterOperator: .greaterThan, value: "10", color: .red, target: .cell) + ] + HighlightRuleStorage(storageDirectory: directory).setRules(rules, for: scope(table: "orders")) + + let reloaded = HighlightRuleStorage(storageDirectory: directory) + #expect(reloaded.rules(for: scope(table: "orders")) == rules) + #expect(reloaded.rules(for: scope(table: "customers")).isEmpty) + } + + @Test("Clearing the last rule removes the connection's file") + func clearingRemovesFile() { + let storage = HighlightRuleStorage(storageDirectory: directory) + storage.setRules([HighlightRule(columnName: "status", value: "paid")], for: scope(table: "orders")) + #expect(FileManager.default.fileExists(atPath: fileURL.path)) + + storage.setRules([], for: scope(table: "orders")) + #expect(!FileManager.default.fileExists(atPath: fileURL.path)) + } + + @Test("A table rename moves its rules to the new name") + func renameMovesRules() { + let storage = HighlightRuleStorage(storageDirectory: directory) + let rules = [HighlightRule(columnName: "status", value: "paid")] + storage.setRules(rules, for: scope(table: "orders")) + + storage.rename(from: scope(table: "orders"), to: scope(table: "purchases")) + + let reloaded = HighlightRuleStorage(storageDirectory: directory) + #expect(reloaded.rules(for: scope(table: "orders")).isEmpty) + #expect(reloaded.rules(for: scope(table: "purchases")) == rules) + } + + @Test("A schema rename moves every table's rules in it") + func renameScopeMovesEveryTable() { + let storage = HighlightRuleStorage(storageDirectory: directory) + let rules = [HighlightRule(columnName: "status", value: "paid")] + storage.setRules(rules, for: scope(table: "orders")) + storage.setRules(rules, for: scope(table: "items")) + + storage.renameScope( + connectionId: connectionId, fromDatabase: "shop", fromSchema: "public", + toDatabase: "shop", toSchema: "sales" + ) + + #expect(storage.rules(for: scope(table: "orders", schema: "sales")) == rules) + #expect(storage.rules(for: scope(table: "items", schema: "sales")) == rules) + #expect(storage.rules(for: scope(table: "orders")).isEmpty) + } + + @Test("Deleting a connection removes its rules") + func removingConnectionPurges() { + let storage = HighlightRuleStorage(storageDirectory: directory) + storage.setRules([HighlightRule(columnName: "status", value: "paid")], for: scope(table: "orders")) + + storage.removeRules(for: [connectionId]) + + #expect(storage.rules(for: scope(table: "orders")).isEmpty) + #expect(!FileManager.default.fileExists(atPath: fileURL.path)) + } + + @Test("Every change moves the observed revision") + func revisionMoves() { + let storage = HighlightRuleStorage(storageDirectory: directory) + let before = storage.revision + storage.setRules([HighlightRule(columnName: "status", value: "paid")], for: scope(table: "orders")) + #expect(storage.revision != before) + } + + @Test("An unreadable file is set aside rather than overwritten") + func unreadableFileIsPreserved() throws { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try Data("{ not json".utf8).write(to: fileURL) + + let storage = HighlightRuleStorage(storageDirectory: directory) + #expect(storage.rules(for: scope(table: "orders")).isEmpty) + + let preserved = directory.appendingPathComponent("\(connectionId.uuidString).unreadable.json") + #expect(FileManager.default.fileExists(atPath: preserved.path)) + #expect(try String(contentsOf: preserved, encoding: .utf8) == "{ not json") + } + + @Test("A rule the app cannot decode is skipped and the rest survive") + func undecodableRuleIsSkipped() throws { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let key = scope(table: "orders").storageComponent + let json = """ + {"\(key)": [ + {"columnName": "status", "filterOperator": "=", "value": "paid", "color": "green"}, + {"columnName": "status", "filterOperator": "SOUNDS LIKE", "value": "x", "color": "green"}, + {"columnName": "status", "filterOperator": "=", "value": "late", "color": "chartreuse"} + ]} + """ + try Data(json.utf8).write(to: fileURL) + + let rules = HighlightRuleStorage(storageDirectory: directory).rules(for: scope(table: "orders")) + #expect(rules.count == 1) + #expect(rules.first?.value == "paid") + #expect(rules.first?.color == .green) + } +} diff --git a/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 6b5ad141ab..704293ee76 100644 --- a/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -545,6 +545,52 @@ struct QueryClassifierNonSqlTests { ) } + @Test("Weaviate GraphQL reads are safe and uuid deletes are destructive") + func weaviateTiers() { + #expect(!QueryClassifier.isWriteQuery("{ Get { Article { title } } }", databaseType: .weaviate)) + #expect(!QueryClassifier.isWriteQuery("query { Get { Article { title } } }", databaseType: .weaviate)) + #expect(!QueryClassifier.isWriteQuery("GET /v1/schema", databaseType: .weaviate)) + #expect(!QueryClassifier.isWriteQuery("POST /v1/graphql {}", databaseType: .weaviate)) + #expect(QueryClassifier.isWriteQuery("mutation { delete { Article } }", databaseType: .weaviate)) + #expect(QueryClassifier.isWriteQuery("POST /v1/objects {}", databaseType: .weaviate)) + #expect(QueryClassifier.classifyTier("DELETE /v1/objects/abc", databaseType: .weaviate) == .destructive) + let search = "WEAVIATE_SEARCH:e30=" + #expect(!QueryClassifier.isWriteQuery(search, databaseType: .weaviate)) + } + + @Test("A console body declaring a mutation is classified the same as a bare one") + func weaviateConsoleBodyIsRead() { + #expect(QueryClassifier.isWriteQuery( + "POST /v1/graphql\nmutation { delete { Article } }", + databaseType: .weaviate + )) + #expect(QueryClassifier.isWriteQuery( + "POST /v1/graphql\n{\"query\": \"mutation { delete { Article } }\"}", + databaseType: .weaviate + )) + #expect(!QueryClassifier.isWriteQuery( + "POST /v1/graphql\n{\"query\": \"{ Get { Article { title } } }\"}", + databaseType: .weaviate + )) + #expect(!QueryClassifier.isWriteQuery( + "POST /v1/graphql?pretty\n{ Get { Article { title } } }", + databaseType: .weaviate + )) + } + + @Test("A mutation inside a query envelope is a write however it is typed") + func weaviateEnvelopeIsRead() { + #expect(QueryClassifier.isWriteQuery( + "{\"query\": \"mutation { delete { Article } }\"}", + databaseType: .weaviate + )) + #expect(!QueryClassifier.isWriteQuery( + "{\"query\": \"{ Get { Article { title } } }\"}", + databaseType: .weaviate + )) + #expect(!QueryClassifier.isWriteQuery("{ Get { Article { title } } }", databaseType: .weaviate)) + } + @Test("Elasticsearch stored scripts are flagged as code execution on both verbs") func elasticsearchScriptsAreFlagged() { #expect( diff --git a/TableProTests/Models/ResultStatusModelTests.swift b/TableProTests/Models/ResultStatusModelTests.swift index b5d378e0d9..00d7005949 100644 --- a/TableProTests/Models/ResultStatusModelTests.swift +++ b/TableProTests/Models/ResultStatusModelTests.swift @@ -199,6 +199,21 @@ struct ResultStatusModelTests { #expect(!structure.controls.showsPagination) } + @Test("Highlight rules are offered only where the data grid draws the result") + func highlightRulesFollowTheDataGrid() { + let table = makeSnapshot(rowCount: 10) + #expect(model(table, viewMode: .data).controls.showsHighlightRules) + #expect(!model(table, viewMode: .json).controls.showsHighlightRules) + #expect(!model(table, viewMode: .chart).controls.showsHighlightRules) + #expect(!model(table, viewMode: .structure).controls.showsHighlightRules) + + let query = makeSnapshot(tabType: .query, rowCount: 3, hasTableName: false) + #expect(model(query, viewMode: .data).controls.showsHighlightRules) + + let noResult = makeSnapshot(tabType: .query, rowCount: 0, hasColumns: false, hasTableName: false) + #expect(!model(noResult, viewMode: .data).controls.showsHighlightRules) + } + @Test("A query tab never offers table-only controls") func queryTabHasNoTableControls() { var pagination = PaginationState(pageSize: 1_000) diff --git a/TableProTests/Plugins/LibPQPluginErrorTests.swift b/TableProTests/Plugins/LibPQPluginErrorTests.swift new file mode 100644 index 0000000000..3bab0016e4 --- /dev/null +++ b/TableProTests/Plugins/LibPQPluginErrorTests.swift @@ -0,0 +1,65 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@Suite("LibPQPluginError result fields") +struct LibPQPluginErrorTests { + private static let undefinedFunctionFields: [Int32: String] = [ + Int32(UInt8(ascii: "C")): "42883", + Int32(UInt8(ascii: "P")): "134", + Int32(UInt8(ascii: "D")): "no function matches" + ] + + private static func decoded(_ fields: [Int32: String], message: String = "ERROR: failed") -> LibPQPluginError { + LibPQPluginError(message: message) { fields[$0] } + } + + @Test("SQLSTATE comes from the C field, never the statement position") + func sqlStateReadsTheSQLStateField() { + let error = Self.decoded(Self.undefinedFunctionFields) + #expect(error.sqlState == "42883") + #expect(error.pluginSqlState == "42883") + } + + @Test("Detail comes from the D field") + func detailReadsTheDetailField() { + let error = Self.decoded(Self.undefinedFunctionFields) + #expect(error.detail == "no function matches") + } + + @Test("Only the SQLSTATE and detail fields are read") + func readsNoOtherField() { + var requested: [Int32] = [] + _ = LibPQPluginError(message: "ERROR: failed") { field in + requested.append(field) + return nil + } + #expect(Set(requested) == [Int32(UInt8(ascii: "C")), Int32(UInt8(ascii: "D"))]) + } + + @Test("A result with no diagnostic fields carries no SQLSTATE") + func missingFieldsStayNil() { + let error = Self.decoded([:]) + #expect(error.sqlState == nil) + #expect(error.detail == nil) + } + + @Test("The error description names the SQLSTATE") + func descriptionNamesTheSQLState() { + let error = Self.decoded( + Self.undefinedFunctionFields, + message: "ERROR: function to_regclass(text) does not exist" + ) + #expect(error.errorDescription?.contains("(SQLSTATE: 42883)") == true) + } + + @Test("A decoded read-only transaction error is diagnosed as server-enforced read-only") + func readOnlyTransactionIsDiagnosed() { + let error = Self.decoded( + [Int32(UInt8(ascii: "C")): "25006"], + message: "ERROR: cannot execute INSERT in a read-only transaction" + ) + #expect(DatabaseWriteRejectionDiagnosis.classify(error) != nil) + } +} diff --git a/TableProTests/Plugins/MariaDBFieldClassifierTests.swift b/TableProTests/Plugins/MariaDBFieldClassifierTests.swift index 883dbc64fd..f570a3fd43 100644 --- a/TableProTests/Plugins/MariaDBFieldClassifierTests.swift +++ b/TableProTests/Plugins/MariaDBFieldClassifierTests.swift @@ -3,13 +3,10 @@ // TableProTests // -#if canImport(MySQLDriverPlugin) import Foundation import TableProPluginKit import Testing -@testable import MySQLDriverPlugin - @Suite("MariaDBFieldClassifier") struct MariaDBFieldClassifierTests { @Test("makeColumnMeta reads PRIMARY KEY, NOT NULL, and AUTO_INCREMENT flags") @@ -82,7 +79,6 @@ struct MariaDBFieldClassifierTests { bytes.withUnsafeBytes { MariaDBFieldClassifier.bitFieldToString($0) } } - @Test("BLOB family with binary charset routes to binary") func blobFamilyBinary() { for typeRaw: UInt32 in [249, 250, 251, 252] { @@ -147,4 +143,3 @@ struct MariaDBFieldClassifierTests { #expect(!MariaDBFieldClassifier.isBinary(typeRaw: 255, charset: 63)) // GEOMETRY (handled upstream) } } -#endif diff --git a/TableProTests/Plugins/MariaDBTypeNameTests.swift b/TableProTests/Plugins/MariaDBTypeNameTests.swift index 2fb80741ae..af00cc4630 100644 --- a/TableProTests/Plugins/MariaDBTypeNameTests.swift +++ b/TableProTests/Plugins/MariaDBTypeNameTests.swift @@ -3,11 +3,8 @@ // TableProTests // -#if canImport(MySQLDriverPlugin) import Testing -@testable import MySQLDriverPlugin - @Suite("MariaDB type name resolution") struct MariaDBTypeNameTests { private func resolve(typeRaw: UInt32, charsetnr: UInt32 = 33, flags: UInt = 0, length: UInt = 0) -> String { @@ -117,4 +114,3 @@ struct MariaDBTypeNameTests { #expect(resolve(typeRaw: 252, charsetnr: 33, flags: 0, length: 100_000) == "LONGTEXT") } } -#endif diff --git a/TableProTests/Plugins/MySQLCharacterSetTests.swift b/TableProTests/Plugins/MySQLCharacterSetTests.swift new file mode 100644 index 0000000000..c6717f1f09 --- /dev/null +++ b/TableProTests/Plugins/MySQLCharacterSetTests.swift @@ -0,0 +1,73 @@ +// +// MySQLCharacterSetTests.swift +// TableProTests +// + +import Foundation +import Testing + +@Suite("MySQL character set decoding") +struct MySQLCharacterSetTests { + private func decode(_ bytes: [UInt8], as name: String) -> String { + bytes.withUnsafeBytes { MySQLCharacterSet(serverName: name).decode($0) } + } + + @Test("Server names are normalized, and utf8 means utf8mb3") + func namesAreNormalized() { + #expect(MySQLCharacterSet(serverName: "UTF8").name == "utf8mb3") + #expect(MySQLCharacterSet(serverName: " latin1 ") == .latin1) + #expect(MySQLCharacterSet(serverName: "utf8mb4") == .utf8mb4) + } + + @Test("UTF-8 text decodes as UTF-8") + func utf8Decodes() { + #expect(decode(Array("メール・記事紐付け".utf8), as: "utf8mb4") == "メール・記事紐付け") + #expect(decode(Array("😀".utf8), as: "utf8mb4") == "😀") + } + + @Test("Invalid UTF-8 is marked with a replacement character, not reinvented as Latin 1") + func invalidUTF8IsNotFabricated() { + #expect(decode([0x61, 0xFF, 0x62], as: "utf8mb4") == "a\u{FFFD}b") + #expect(decode([0x63, 0x61, 0x66, 0xE9], as: "utf8mb3") == "caf\u{FFFD}") + } + + @Test("A latin1 column holding UTF-8 bytes reads as that UTF-8") + func latin1HoldingUTF8() { + #expect(decode([0xE3, 0x83, 0xA1], as: "latin1") == "メ") + } + + @Test("A latin1 column holding Latin 1 text uses MySQL's own latin1, which is cp1252") + func latin1IsWindows1252() { + #expect(decode([0x69, 0x74, 0x92, 0x73, 0x20, 0x35, 0x80], as: "latin1") == "it’s 5€") + #expect(decode([0x63, 0x61, 0x66, 0xE9], as: "latin1") == "café") + } + + @Test("Charsets Foundation decodes byte-for-byte like the server use Foundation") + func foundationCharsets() { + #expect(decode([0xCF, 0xF0, 0xE8, 0xE2, 0xE5, 0xF2], as: "cp1251") == "Привет") + #expect(decode([0xB1, 0xE6], as: "latin2") == "ąć") + #expect(decode([0x83, 0x81, 0x81, 0x5B, 0x83, 0x8B, 0x5C], as: "cp932") == "メール\\") + #expect(decode([0xD6, 0xD0, 0xCE, 0xC4], as: "gbk") == "中文") + #expect(decode([0x30, 0xE1, 0x30, 0xFC], as: "utf16") == "メー") + #expect(decode([0xE1, 0x30, 0xFC, 0x30], as: "utf16le") == "メー") + } + + @Test("A byte a single-byte charset leaves undefined becomes one replacement character") + func undefinedSingleByteKeepsTheRest() { + #expect(decode([0x61, 0x81, 0x62], as: "cp1250") == "a\u{FFFD}b") + } + + @Test("Charsets whose Foundation mapping disagrees with the server are not in the table") + func mismatchedCharsetsAreExcluded() { + let decodable = Set(MySQLCharacterSet.singleByteDecodedNames + MySQLCharacterSet.multiByteDecodedNames) + for name in ["sjis", "ujis", "eucjpms", "big5", "euckr", "greek", "hebrew", "koi8r", "koi8u", "cp866", "latin7", "tis620"] { + #expect(!decodable.contains(name), "\(name)") + } + } + + @Test("A charset without a verified decoder reads valid UTF-8 as UTF-8") + func unknownCharsetFallsBackToUTF8() { + #expect(decode(Array("abc".utf8), as: "armscii8") == "abc") + #expect(decode([0x61, 0xFF], as: "armscii8") == "a\u{FFFD}") + } +} diff --git a/TableProTests/Plugins/MySQLColumnDecodingTests.swift b/TableProTests/Plugins/MySQLColumnDecodingTests.swift new file mode 100644 index 0000000000..37be254ad9 --- /dev/null +++ b/TableProTests/Plugins/MySQLColumnDecodingTests.swift @@ -0,0 +1,107 @@ +// +// MySQLColumnDecodingTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite("MySQL column decoding") +struct MySQLColumnDecodingTests { + private static let doubleEncodedMail = String(bytes: [0xC3, 0xA3, 0xC6, 0x92, 0xC2, 0xA1], encoding: .utf8) ?? "" + + private func decoding(type: UInt32, charset: UInt32, name: String? = "utf8mb4") -> MySQLColumnDecoding { + MySQLColumnDecoding(typeRaw: type, charsetnr: charset, characterSetName: name) + } + + private func decode( + _ bytes: [UInt8], + with decoding: MySQLColumnDecoding, + encoding: MySQLConnectionEncoding = .utf8 + ) -> PluginCellValue { + bytes.withUnsafeBytes { decoding.decode($0, encoding: encoding) } + } + + @Test("Each column kind gets its own decoding") + func columnKinds() { + #expect(decoding(type: 255, charset: 63) == .geometry) + #expect(decoding(type: 16, charset: 63) == .bit) + #expect(decoding(type: 252, charset: 63) == .bytes) + #expect(decoding(type: 253, charset: 63) == .bytes) + #expect(decoding(type: 253, charset: 8, name: "latin1") == .text(.latin1)) + #expect(decoding(type: 252, charset: 255, name: "utf8mb4") == .text(.utf8mb4)) + } + + @Test("Numbers, dates and JSON carry the binary charset and decode as text") + func binaryCharsetScalarsAreText() { + for type: UInt32 in [3, 8, 12, 245, 246] { + #expect(decoding(type: type, charset: 63, name: "binary") == .text(.utf8mb4)) + } + } + + @Test("Databend's booleans, hex-encoded binary and text geometry keep their own decoding") + func databendShapes() { + let boolean = MySQLColumnDecoding(typeRaw: 2, length: 1, charsetnr: 63, characterSetName: "binary", flavor: .databend) + #expect(boolean == .databendBoolean) + #expect(decode(Array("1".utf8), with: boolean) == .text("true")) + + let binary = MySQLColumnDecoding(typeRaw: 252, charsetnr: 63, characterSetName: "binary", flavor: .databend) + #expect(binary == .databendHexBytes) + #expect(decode(Array("CAFE".utf8), with: binary) == .bytes(Data([0xCA, 0xFE]))) + + let geometry = MySQLColumnDecoding(typeRaw: 255, charsetnr: 63, characterSetName: "binary", flavor: .databend) + #expect(geometry == .text(.utf8mb4)) + #expect(MySQLColumnDecoding(typeRaw: 2, length: 1, charsetnr: 63, characterSetName: "binary") == .text(.utf8mb4)) + } + + @Test("A collation id libmariadb does not know decodes as the session's UTF-8") + func unknownCollationIsUTF8() { + #expect(decoding(type: 253, charset: 309, name: nil) == .text(.utf8mb4)) + } + + @Test("The default encoding shows exactly what the server stored") + func defaultShowsStoredText() { + let stored = Array(Self.doubleEncodedMail.utf8) + #expect(decode(stored, with: .text(.utf8mb4)) == .text(Self.doubleEncodedMail)) + } + + @Test("UTF-8 via Latin 1 repairs text columns only") + func legacyModeRepairsText() { + let stored = Array(Self.doubleEncodedMail.utf8) + #expect(decode(stored, with: .text(.utf8mb4), encoding: .utf8ViaLatin1) == .text("メ")) + #expect(decode(stored, with: .bytes, encoding: .utf8ViaLatin1) == .bytes(Data(stored))) + } + + @Test("A result row keeps NULLs and decodes every other cell by its column") + func resultRow() { + var columns = MySQLResultColumns() + columns.append(name: "id", typeCode: 3, typeName: "INT", decoding: .text(.utf8mb4), flags: mysqlPriKeyFlag) + columns.append(name: "note", typeCode: 253, typeName: "VARCHAR", decoding: .text(.latin1), flags: 0) + columns.append(name: "missing", typeCode: 253, typeName: "VARCHAR", decoding: .text(.utf8mb4), flags: 0) + let cells: [[UInt8]?] = [Array("7".utf8), [0x63, 0x61, 0x66, 0xE9], nil] + let buffers = cells.map { cell in + cell.map { bytes in + let buffer = UnsafeMutableRawBufferPointer.allocate(byteCount: bytes.count, alignment: 1) + buffer.copyBytes(from: bytes) + return buffer + } + } + defer { buffers.forEach { $0?.deallocate() } } + + let row = columns.row(encoding: .utf8) { index in + buffers[index].map { UnsafeRawBufferPointer($0) } + } + + #expect(row == [.text("7"), .text("café"), .null]) + #expect(columns.metadata.first?.isPrimaryKey == true) + } + + @Test("Names and messages read as UTF-8, or as MySQL latin1 when a latin1 session sent them") + func sessionText() { + let utf8 = Array("列名".utf8) + let latin1: [UInt8] = [0x63, 0x61, 0x66, 0xE9] + #expect(utf8.withUnsafeBytes { mysqlSessionText($0, encoding: .utf8) } == "列名") + #expect(latin1.withUnsafeBytes { mysqlSessionText($0, encoding: .utf8) } == "café") + } +} diff --git a/TableProTests/Plugins/MySQLConnectionEncodingTests.swift b/TableProTests/Plugins/MySQLConnectionEncodingTests.swift new file mode 100644 index 0000000000..52218ceecd --- /dev/null +++ b/TableProTests/Plugins/MySQLConnectionEncodingTests.swift @@ -0,0 +1,82 @@ +// +// MySQLConnectionEncodingTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@Suite("MySQL connection encoding") +struct MySQLConnectionEncodingTests { + @Test("A missing, empty or unknown field value is plain UTF-8") + func fieldValueParsing() { + #expect(MySQLConnectionEncoding(fieldValue: nil) == .utf8) + #expect(MySQLConnectionEncoding(fieldValue: "") == .utf8) + #expect(MySQLConnectionEncoding(fieldValue: "sjis") == .utf8) + #expect(MySQLConnectionEncoding(fieldValue: "utf8ViaLatin1") == .utf8ViaLatin1) + } + + @Test("Only UTF-8 via Latin 1 changes the client character set, and only the client's") + func sessionStatements() { + #expect(MySQLConnectionEncoding.utf8.sessionStatements.isEmpty) + #expect(MySQLConnectionEncoding.utf8ViaLatin1.sessionStatements == ["SET character_set_client = latin1"]) + #expect(MySQLConnectionEncoding.sessionCharacterSetName == "utf8mb4") + #expect(MySQLConnectionEncoding.sessionFallbackStatement == "SET NAMES utf8") + } + + @Test("The field is an advanced dropdown whose values the driver reads back") + func fieldShape() throws { + let field = MySQLConnectionEncoding.connectionField + #expect(field.id == MySQLConnectionEncoding.fieldId) + #expect(field.section == .advanced) + guard case .dropdown(let options) = field.fieldType else { + Issue.record("expected a dropdown") + return + } + #expect(options.map(\.value) == MySQLConnectionEncoding.allCases.map(\.rawValue)) + for option in options { + #expect(MySQLConnectionEncoding(fieldValue: option.value).rawValue == option.value) + } + } + + @Test("MySQL and MariaDB curate the same encoding field the plugin declares") + func curatedFieldsMatchThePlugin() throws { + let expected = try Self.encoded(MySQLConnectionEncoding.connectionField) + let curated = PluginMetadataRegistry.curatedDefaults() + for typeId in ["MySQL", "MariaDB"] { + let snapshot = try #require(curated.first { $0.typeId == typeId }?.snapshot) + let field = try #require( + snapshot.connection.additionalConnectionFields.first { $0.id == MySQLConnectionEncoding.fieldId }, + "\(typeId) curates no encoding field" + ) + #expect(try Self.encoded(field) == expected, "\(typeId)") + } + } + + @Test("The MySQL plugin declares the shared encoding field") + func pluginDeclaresTheField() throws { + let source = try String(contentsOf: Self.repositoryRoot().appendingPathComponent( + "Plugins/MySQLDriverPlugin/MySQLPlugin.swift" + ), encoding: .utf8) + #expect(source.contains("MySQLConnectionEncoding.connectionField")) + } + + private static func encoded(_ field: ConnectionField) throws -> Data { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return try encoder.encode(field) + } + + private static func repositoryRoot(file: StaticString = #filePath) throws -> URL { + var directory = URL(fileURLWithPath: "\(file)").deletingLastPathComponent() + while directory.path != "/" { + if FileManager.default.fileExists(atPath: directory.appendingPathComponent("project.yml").path) { + return directory + } + directory = directory.deletingLastPathComponent() + } + throw CocoaError(.fileNoSuchFile) + } +} diff --git a/TableProTests/Plugins/MySQLLatin1Tests.swift b/TableProTests/Plugins/MySQLLatin1Tests.swift new file mode 100644 index 0000000000..e6e15332eb --- /dev/null +++ b/TableProTests/Plugins/MySQLLatin1Tests.swift @@ -0,0 +1,101 @@ +// +// MySQLLatin1Tests.swift +// TableProTests +// + +import Foundation +import Testing + +@Suite("MySQL latin1") +struct MySQLLatin1Tests { + private static let serverUTF8ForUpperHalf: [String] = [ + "E282AC", "C281", "E2809A", "C692", "E2809E", "E280A6", "E280A0", "E280A1", + "CB86", "E280B0", "C5A0", "E280B9", "C592", "C28D", "C5BD", "C28F", + "C290", "E28098", "E28099", "E2809C", "E2809D", "E280A2", "E28093", "E28094", + "CB9C", "E284A2", "C5A1", "E280BA", "C593", "C29D", "C5BE", "C5B8", + "C2A0", "C2A1", "C2A2", "C2A3", "C2A4", "C2A5", "C2A6", "C2A7", + "C2A8", "C2A9", "C2AA", "C2AB", "C2AC", "C2AD", "C2AE", "C2AF", + "C2B0", "C2B1", "C2B2", "C2B3", "C2B4", "C2B5", "C2B6", "C2B7", + "C2B8", "C2B9", "C2BA", "C2BB", "C2BC", "C2BD", "C2BE", "C2BF", + "C380", "C381", "C382", "C383", "C384", "C385", "C386", "C387", + "C388", "C389", "C38A", "C38B", "C38C", "C38D", "C38E", "C38F", + "C390", "C391", "C392", "C393", "C394", "C395", "C396", "C397", + "C398", "C399", "C39A", "C39B", "C39C", "C39D", "C39E", "C39F", + "C3A0", "C3A1", "C3A2", "C3A3", "C3A4", "C3A5", "C3A6", "C3A7", + "C3A8", "C3A9", "C3AA", "C3AB", "C3AC", "C3AD", "C3AE", "C3AF", + "C3B0", "C3B1", "C3B2", "C3B3", "C3B4", "C3B5", "C3B6", "C3B7", + "C3B8", "C3B9", "C3BA", "C3BB", "C3BC", "C3BD", "C3BE", "C3BF" + ] + + private static let reporterMojibakeUTF8 = + "C3A3C692C2A1C3A3C692C2BCC3A3C692C2ABC3A3C692C2BBC3A8C2A8CB9CC3A4C2BAE280B9C3A7C2B4C290C3A4C2BBCB9CC3A3C281E28098" + + private func hex(_ text: String) -> String { + text.utf8.map { String(format: "%02X", $0) }.joined() + } + + private func text(fromHex hex: String) -> String { + var bytes: [UInt8] = [] + var index = hex.startIndex + while index < hex.endIndex { + let next = hex.index(index, offsetBy: 2) + bytes.append(UInt8(hex[index.. URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory + } + + @Test("A MySQL dump declares utf8mb4 the way mysqldump does, and puts the session back") + func mysqlDeclaresUTF8MB4() { + let declaration = SQLExportEncodingDeclaration.forDialect(.mysql) + #expect(declaration.prologue.contains("/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;")) + #expect(declaration.prologue.contains("/*!40101 SET NAMES utf8 */;")) + #expect(declaration.prologue.contains("/*!50503 SET NAMES utf8mb4 */;")) + #expect(declaration.epilogue.contains("/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;")) + #expect(declaration.epilogue.contains("/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;")) + #expect(declaration.epilogue.contains("/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;")) + } + + @Test("Other dialects write no declaration") + func otherDialectsDeclareNothing() { + for dialect: SqlDialect in [.sqlite, .generic, .postgres] { + #expect(SQLExportEncodingDeclaration.forDialect(dialect) == .empty) + } + } + + @Test("An unsplit dump opens with the declaration and closes by restoring the session") + func unsplitDumpIsWrapped() throws { + let directory = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: directory) } + let declaration = SQLExportEncodingDeclaration.forDialect(.mysql) + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter( + destination: destination, splitSizeMegabytes: 0, encodingDeclaration: declaration + ) + try writer.write("INSERT INTO `t` VALUES ('メール');\n") + try writer.commit() + + let dump = try String(contentsOf: destination, encoding: .utf8) + #expect(dump == declaration.prologue + "INSERT INTO `t` VALUES ('メール');\n" + declaration.epilogue) + } + + @Test("Every part of a split dump carries its own declaration, so each restores on its own") + func everyPartIsWrapped() throws { + let directory = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: directory) } + let declaration = SQLExportEncodingDeclaration.forDialect(.mysql) + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter( + destination: destination, splitSizeMegabytes: 1, encodingDeclaration: declaration + ) + let chunk = String(repeating: "x", count: 700 * 1_024) + try writer.write("A\(chunk);\n") + try writer.write("B\(chunk);\n") + let parts = try writer.commit() + + #expect(parts.count == 2) + for (index, part) in parts.enumerated() { + let text = try String(contentsOf: part, encoding: .utf8) + #expect(text.hasPrefix(declaration.prologue), "part \(index + 1)") + #expect(text.hasSuffix(declaration.epilogue), "part \(index + 1)") + #expect(text.contains(index == 0 ? "A" : "B")) + } + } + + @Test("No part grows past the cap once its closing declaration is added") + func epilogueCountsTowardTheCap() throws { + let directory = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: directory) } + let declaration = SQLExportEncodingDeclaration.forDialect(.mysql) + let cap = 1_024 * 1_024 + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter( + destination: destination, splitSizeMegabytes: 1, encodingDeclaration: declaration + ) + let firstLength = cap - declaration.prologue.utf8.count - declaration.epilogue.utf8.count - 5 + try writer.write(String(repeating: "a", count: firstLength - 2) + ";\n") + try writer.write("SELECT 1;\n") + let parts = try writer.commit() + + #expect(parts.count == 2) + for part in parts { + let size = try FileManager.default.attributesOfItem(atPath: part.path)[.size] as? Int ?? 0 + #expect(size <= cap, "\(part.lastPathComponent) is \(size) bytes") + } + } + + @Test("A statement larger than the cap still lands in a part of its own, not after an empty one") + func oversizedStatementDoesNotLeaveAnEmptyPart() throws { + let directory = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: directory) } + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter( + destination: destination, splitSizeMegabytes: 1, + encodingDeclaration: .forDialect(.mysql) + ) + try writer.write(String(repeating: "y", count: 2 * 1_024 * 1_024) + ";\n") + let parts = try writer.commit() + + #expect(parts.count == 1) + #expect(!writer.didSplit) + } +} diff --git a/TableProTests/Plugins/WeaviateConnectionFieldsTests.swift b/TableProTests/Plugins/WeaviateConnectionFieldsTests.swift new file mode 100644 index 0000000000..3046f3bd04 --- /dev/null +++ b/TableProTests/Plugins/WeaviateConnectionFieldsTests.swift @@ -0,0 +1,141 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProWeaviateCore +import Testing + +@Suite("Weaviate registry snapshot") +struct WeaviateRegistrySnapshotTests { + private func snapshot() throws -> PluginMetadataSnapshot { + let defaults = PluginMetadataRegistry.shared.registryPluginDefaults() + return try #require(defaults.first { $0.typeId == "Weaviate" }).snapshot + } + + @Test("Weaviate is a collection engine on port 8080 with no SQL dialect") + func connectionShape() throws { + let snapshot = try snapshot() + #expect(snapshot.defaultPort == 8_080) + #expect(snapshot.editor.sqlDialect == nil) + #expect(snapshot.queryLanguageName == "GraphQL") + #expect(snapshot.schema.tableEntityName == "Collections") + #expect(snapshot.schema.defaultPrimaryKeyColumn == "uuid") + #expect(snapshot.schema.immutableColumns == ["uuid", "vector"]) + #expect(!snapshot.supportsForeignKeys) + #expect(!snapshot.capabilities.supportsSSH) + #expect(snapshot.capabilities.supportsSSL) + #expect(snapshot.connection.category == .document) + #expect(snapshot.iconName == "weaviate-icon") + } + + @Test("Auth field ids are Weaviate-prefixed and do not collide with Elasticsearch") + func authFieldIdsArePrefixed() throws { + let ids = try snapshot().connection.additionalConnectionFields.map(\.id) + #expect(ids == [ + WeaviateFieldID.authMethod, + WeaviateFieldID.apiKey, + WeaviateFieldID.skipTLSVerify + ]) + #expect(!ids.contains("esAuthMethod")) + #expect(!ids.contains("esApiKey")) + let elasticsearch = try #require( + PluginMetadataRegistry.shared.registryPluginDefaults().first { $0.typeId == "Elasticsearch" } + ) + let esIds = elasticsearch.snapshot.connection.additionalConnectionFields.map(\.id) + #expect(Set(ids).isDisjoint(with: Set(esIds))) + } +} + +@Suite("Weaviate connection fields") +struct WeaviateConnectionFieldsTests { + private func fields() throws -> [ConnectionField] { + let defaults = PluginMetadataRegistry.shared.registryPluginDefaults() + let entry = try #require(defaults.first { $0.typeId == "Weaviate" }) + return entry.snapshot.connection.additionalConnectionFields + } + + @Test("Auth method defaults to none") + func authMethodDefaultsToNone() throws { + let fields = try fields() + let method = try #require(fields.first { $0.id == WeaviateFieldID.authMethod }) + #expect(method.defaultValue == "none") + guard case .dropdown(let options) = method.fieldType else { + Issue.record("Expected a dropdown field type") + return + } + #expect(options.map(\.value) == ["none", "apiKey"]) + } + + @Test("The API key replaces both built-in credential rows") + func apiKeyReplacesUsernameAndPassword() throws { + let fields = try fields() + let apiKey = try #require(fields.first { $0.id == WeaviateFieldID.apiKey }) + #expect(apiKey.isSecure) + #expect(!apiKey.isRequired) + #expect(apiKey.hidesPassword) + #expect(fields.hidesPassword(forValues: [:])) + #expect(fields.hidesUsername(forValues: [:])) + #expect(fields.hidesPassword(forValues: [WeaviateFieldID.authMethod: "none"])) + #expect(fields.hidesPassword(forValues: [WeaviateFieldID.authMethod: "apiKey"])) + #expect(fields.hidesUsername(forValues: [WeaviateFieldID.authMethod: "none"])) + #expect(fields.hidesUsername(forValues: [WeaviateFieldID.authMethod: "apiKey"])) + } + + @Test("The snapshot hides the built-in password for every auth method") + @MainActor + func snapshotHidesBuiltInPassword() throws { + let defaults = PluginMetadataRegistry.shared.registryPluginDefaults() + let snapshot = try #require(defaults.first { $0.typeId == "Weaviate" }).snapshot + #expect(snapshot.connection.hidesBuiltInPassword) + for method in ["none", "apiKey"] { + var connection = DatabaseConnection(name: "Weaviate", type: .weaviate) + connection.additionalFields = [WeaviateFieldID.authMethod: method] + #expect(PluginManager.shared.hidesPassword(for: connection), "\(method)") + } + } +} + +@Suite("Weaviate field parity") +struct WeaviateFieldParityTests { + private static let repoRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + + private func source(_ path: String) throws -> String { + try String(contentsOf: Self.repoRoot.appendingPathComponent(path), encoding: .utf8) + } + + /// The plugin's own field list replaces the registry snapshot the moment the plugin is + /// installed, and no test can link both copies, so the two sources are compared as text. + @Test("The plugin's field list matches the registry copy") + func pluginCopyMatchesRegistryCopy() throws { + let plugin = try source("Plugins/WeaviateDriverPlugin/WeaviatePlugin.swift") + let registry = try source("TablePro/Core/Plugins/PluginMetadataRegistry+WeaviateDefaults.swift") + for field in [WeaviateFieldID.authMethod, WeaviateFieldID.apiKey, WeaviateFieldID.skipTLSVerify] { + #expect(registry.contains("id: \"\(field)\""), Comment(rawValue: field)) + } + #expect(plugin.contains("id: WeaviateFieldID.authMethod")) + #expect(plugin.contains("id: WeaviateFieldID.apiKey")) + #expect(plugin.contains("id: WeaviateFieldID.skipTLSVerify")) + for copy in [plugin, registry] { + #expect(copy.contains("hidesPassword: true")) + #expect(copy.contains("withHidesUsername(true)")) + #expect(copy.contains("apiKey")) + } + } +} + +@Suite("Weaviate plugin manifest") +struct WeaviatePluginManifestTests { + @Test("Info.plist declares PluginKit 25 and the Weaviate type id") + func plistDeclaresType() throws { + let url = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("Plugins/WeaviateDriverPlugin/Info.plist") + let plist = try #require(NSDictionary(contentsOf: url) as? [String: Any]) + #expect(plist["TableProPluginKitVersion"] as? Int == 25) + #expect(plist["TableProProvidesDatabaseTypeIds"] as? [String] == ["Weaviate"]) + } +} diff --git a/TableProTests/Views/Main/ResultStatusBarLayoutTests.swift b/TableProTests/Views/Main/ResultStatusBarLayoutTests.swift index 577126b481..eb163d7508 100644 --- a/TableProTests/Views/Main/ResultStatusBarLayoutTests.swift +++ b/TableProTests/Views/Main/ResultStatusBarLayoutTests.swift @@ -51,6 +51,14 @@ struct ResultStatusBarLayoutTests { onReset: {}, onJumpToColumn: nil ), + highlightState: StatusBarHighlightState( + rules: [], + columns: hasColumns ? ["id", "name"] : [], + isPersisted: tabType == .table, + presentationRequest: 0, + onChange: { _ in }, + onDismiss: {} + ), paginationCallbacks: PaginationCallbacks( onFirst: {}, onPrevious: {}, diff --git a/TableProTests/Views/Results/DataGridCellAppearanceTests.swift b/TableProTests/Views/Results/DataGridCellAppearanceTests.swift index 731f8fbc37..56a47edb1b 100644 --- a/TableProTests/Views/Results/DataGridCellAppearanceTests.swift +++ b/TableProTests/Views/Results/DataGridCellAppearanceTests.swift @@ -179,4 +179,70 @@ struct DataGridCellAppearanceTests { func noAccessoryNoRole() { #expect(resolve().accessoryRole == nil) } + + // MARK: - Highlight rules + + private func highlighted( + column: Int, + color: HighlightColor = .green, + isDeleted: Bool = false, + isInserted: Bool = false, + modifiedColumns: Set = [] + ) -> RowVisualState { + let rule = HighlightRule(columnName: "status", value: "paid", color: color, target: .cell) + return RowVisualState( + isDeleted: isDeleted, + isInserted: isInserted, + modifiedColumns: modifiedColumns, + highlight: RowHighlight(rowRule: nil, cellRules: [column: rule]) + ) + } + + @Test("A cell a highlight rule matches takes the rule's wash") + func highlightedCellTakesTheWash() { + let appearance = resolve(visualState: highlighted(column: 1), columnIndex: 1) + + #expect(appearance.backgroundTint == HighlightColor.green.washColor) + #expect(resolve(visualState: highlighted(column: 1), columnIndex: 2).backgroundTint == nil) + } + + @Test("A modified cell keeps the modified tint over a highlight") + func modifiedTintOutranksHighlight() { + let appearance = resolve(visualState: highlighted(column: 0, modifiedColumns: [0]), columnIndex: 0) + + #expect(appearance.backgroundTint == palette.modifiedColumnTint) + } + + @Test("A find match and a selection both outrank a highlight") + func findAndSelectionOutrankHighlight() { + let found = resolve(visualState: highlighted(column: 0), isCurrentFindMatch: true, columnIndex: 0) + let selected = resolve(visualState: highlighted(column: 0), columnIndex: 0, onEmphasizedSelection: true) + + #expect(found.backgroundTint == palette.findMatchTint) + #expect(selected.backgroundTint == nil) + } + + @Test("A pending insert or delete shows no cell highlight, so it cannot pass for one") + func pendingRowsShowNoCellHighlight() { + let inserted = resolve(visualState: highlighted(column: 0, isInserted: true), columnIndex: 0) + let deleted = resolve(visualState: highlighted(column: 0, isDeleted: true), columnIndex: 0) + + #expect(inserted.backgroundTint == nil) + #expect(deleted.backgroundTint == nil) + } + + @Test("Only a highlight that is drawn is named to VoiceOver") + func drawnHighlightRule() { + let rowRule = HighlightRule(columnName: "status", value: "paid", color: .green) + let cellRule = HighlightRule(columnName: "total", value: "9", color: .red, target: .cell) + let highlight = RowHighlight(rowRule: rowRule, cellRules: [1: cellRule]) + let plain = RowVisualState(isDeleted: false, isInserted: false, modifiedColumns: [], highlight: highlight) + let modified = RowVisualState(isDeleted: false, isInserted: false, modifiedColumns: [1], highlight: highlight) + let deleted = RowVisualState(isDeleted: true, isInserted: false, modifiedColumns: [], highlight: highlight) + + #expect(plain.drawnHighlightRule(forColumn: 1) == cellRule) + #expect(plain.drawnHighlightRule(forColumn: 0) == rowRule) + #expect(modified.drawnHighlightRule(forColumn: 1) == rowRule) + #expect(deleted.drawnHighlightRule(forColumn: 1) == nil) + } } diff --git a/TableProTests/Views/Results/DataGridRowTintThemeTests.swift b/TableProTests/Views/Results/DataGridRowTintThemeTests.swift index 059bc25999..9997f2c42d 100644 --- a/TableProTests/Views/Results/DataGridRowTintThemeTests.swift +++ b/TableProTests/Views/Results/DataGridRowTintThemeTests.swift @@ -4,9 +4,28 @@ // import AppKit +import SwiftUI @testable import TablePro import Testing +@MainActor +private final class FixedVisualStateDelegate: DataGridViewDelegate { + var state: RowVisualState + + init(state: RowVisualState) { + self.state = state + } + + func dataGridVisualState(forRow row: Int) -> RowVisualState? { state } +} + +@MainActor +private final class NoopColumnLayoutPersister: ColumnLayoutPersisting { + func load(for key: ColumnLayoutTableKey) -> ColumnLayoutState? { nil } + func save(_ layout: ColumnLayoutState, for key: ColumnLayoutTableKey) {} + func clear(for key: ColumnLayoutTableKey) {} +} + /// These tests activate a theme on the shared `ThemeEngine`. What keeps that from reaching a suite /// running in parallel is that both bodies are synchronous and `@MainActor`, so nothing else on the /// main actor can interleave between activating the test theme and restoring the original one. @@ -17,8 +36,29 @@ import Testing struct DataGridRowTintThemeTests { private static let deleted = RowVisualState(isDeleted: true, isInserted: false, modifiedColumns: []) - private func makeRowView() -> DataGridRowView { - DataGridRowView(frame: NSRect(x: 0, y: 0, width: 120, height: 24)) + private final class Harness { + let delegate: FixedVisualStateDelegate + let coordinator: TableViewCoordinator + let rowView: DataGridRowView + + @MainActor + init(state: RowVisualState) { + delegate = FixedVisualStateDelegate(state: state) + coordinator = TableViewCoordinator( + changeManager: AnyChangeManager(DataChangeManager()), + isEditable: false, + selectedRowIndices: .constant([]), + delegate: delegate, + layoutPersister: NoopColumnLayoutPersister() + ) + rowView = DataGridRowView(frame: NSRect(x: 0, y: 0, width: 120, height: 24)) + rowView.coordinator = coordinator + rowView.rowIndex = 0 + } + } + + private func makeRowView(state: RowVisualState) -> Harness { + Harness(state: state) } private func renderedTint(of rowView: DataGridRowView) throws -> NSColor { @@ -42,18 +82,53 @@ struct DataGridRowTintThemeTests { defer { engine.activateTheme(original) } engine.activateTheme(theme(original, id: "test.tint.red", deletedHex: "#FF0000")) - let rowView = makeRowView() - rowView.applyVisualState(Self.deleted) - let firstTint = try renderedTint(of: rowView) + let harness = makeRowView(state: Self.deleted) + let firstTint = try renderedTint(of: harness.rowView) engine.activateTheme(theme(original, id: "test.tint.blue", deletedHex: "#0000FF")) - rowView.applyVisualState(Self.deleted) - let secondTint = try renderedTint(of: rowView) + harness.rowView.invalidateVisualState() + let secondTint = try renderedTint(of: harness.rowView) #expect(firstTint.redComponent > secondTint.redComponent) #expect(secondTint.blueComponent > firstTint.blueComponent) } + @Test("A row paints the state its coordinator reports now, not one pushed into it earlier") + func rowPaintsTheLiveState() throws { + let engine = ThemeEngine.shared + let original = engine.activeTheme + defer { engine.activateTheme(original) } + engine.activateTheme(theme(original, id: "test.tint.red", deletedHex: "#FF0000")) + + let harness = makeRowView(state: .empty) + let before = try renderedTint(of: harness.rowView) + harness.delegate.state = Self.deleted + let after = try renderedTint(of: harness.rowView) + + #expect(before.alphaComponent == 0) + #expect(after.redComponent > 0.5) + #expect(harness.rowView.visualState == Self.deleted) + } + + @Test("A pending delete keeps its wash over a matching highlight rule") + func pendingDeleteOutranksHighlightWash() throws { + let engine = ThemeEngine.shared + let original = engine.activeTheme + defer { engine.activateTheme(original) } + engine.activateTheme(theme(original, id: "test.tint.red", deletedHex: "#FF0000")) + + let highlight = RowHighlight( + rowRule: HighlightRule(columnName: "status", value: "paid", color: .blue), + cellRules: [:] + ) + let harness = makeRowView(state: Self.deleted.highlighted(highlight)) + let tint = try renderedTint(of: harness.rowView) + + #expect(tint.redComponent > tint.blueComponent) + #expect(Self.deleted.highlighted(highlight).tint == engine.colors.dataGrid.deleted) + #expect(RowVisualState.empty.highlighted(highlight).tint == HighlightColor.blue.washColor) + } + @Test("A row with no deleted or inserted state stays untinted across a theme change") func plainRowStaysUntinted() throws { let engine = ThemeEngine.shared @@ -61,13 +136,12 @@ struct DataGridRowTintThemeTests { defer { engine.activateTheme(original) } engine.activateTheme(theme(original, id: "test.tint.red", deletedHex: "#FF0000")) - let rowView = makeRowView() - rowView.applyVisualState(.empty) - let firstTint = try renderedTint(of: rowView) + let harness = makeRowView(state: .empty) + let firstTint = try renderedTint(of: harness.rowView) engine.activateTheme(theme(original, id: "test.tint.blue", deletedHex: "#0000FF")) - rowView.applyVisualState(.empty) - let secondTint = try renderedTint(of: rowView) + harness.rowView.invalidateVisualState() + let secondTint = try renderedTint(of: harness.rowView) #expect(firstTint.redComponent == secondTint.redComponent) #expect(firstTint.blueComponent == secondTint.blueComponent) diff --git a/TableProTests/Views/Results/DataGridUpdateSnapshotTests.swift b/TableProTests/Views/Results/DataGridUpdateSnapshotTests.swift index e37360eff5..d8a139b418 100644 --- a/TableProTests/Views/Results/DataGridUpdateSnapshotTests.swift +++ b/TableProTests/Views/Results/DataGridUpdateSnapshotTests.swift @@ -16,6 +16,7 @@ struct DataGridUpdateSnapshotTests { reloadVersion: Int = 0, contentRevision: Int = 0, displayFormats: [ValueDisplayFormat?] = [], + highlightRules: [HighlightRule] = [], columnComments: [String: String] = [:] ) -> DataGridUpdateSnapshot { DataGridUpdateSnapshot( @@ -24,6 +25,7 @@ struct DataGridUpdateSnapshotTests { columns: columns, valueFilteredIDsCount: nil, displayFormats: displayFormats, + highlightRules: highlightRules, configuration: DataGridConfiguration(), isEditable: true, rowReorder: .disabled, @@ -79,6 +81,20 @@ struct DataGridUpdateSnapshotTests { #expect(raw != uuid) } + @Test("A highlight rule change invalidates the update snapshot") + func highlightRuleChangesSnapshot() { + let rule = HighlightRule(columnName: "type", value: "admin", color: .green) + var recolored = rule + recolored.color = .red + + let highlighted = makeSnapshot(highlightRules: [rule]) + let rebuilt = makeSnapshot(highlightRules: [rule]) + + #expect(makeSnapshot() != highlighted) + #expect(highlighted != makeSnapshot(highlightRules: [recolored])) + #expect(highlighted == rebuilt) + } + @Test("Display format cache entries are scoped to a pinned result set") func displayFormatCacheUsesResultSetIdentity() { let firstResult = UUID() diff --git a/TableProTests/Views/Results/TableViewCoordinatorHighlightTests.swift b/TableProTests/Views/Results/TableViewCoordinatorHighlightTests.swift new file mode 100644 index 0000000000..0d644c8514 --- /dev/null +++ b/TableProTests/Views/Results/TableViewCoordinatorHighlightTests.swift @@ -0,0 +1,145 @@ +// +// TableViewCoordinatorHighlightTests.swift +// TableProTests +// + +import AppKit +import SwiftUI +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +private final class HighlightTestPersister: ColumnLayoutPersisting { + func load(for key: ColumnLayoutTableKey) -> ColumnLayoutState? { nil } + func save(_ layout: ColumnLayoutState, for key: ColumnLayoutTableKey) {} + func clear(for key: ColumnLayoutTableKey) {} +} + +@MainActor +private final class StructureStateDelegate: DataGridViewDelegate { + func dataGridVisualState(forRow row: Int) -> RowVisualState? { .empty } +} + +@MainActor +private final class HighlightGrid { + var tableRows: TableRows + let coordinator: TableViewCoordinator + + init(statuses: [String], delegate: (any DataGridViewDelegate)? = nil) { + let rows = ContiguousArray(statuses.enumerated().map { index, status in + Row(id: .existing(index), values: [.text("\(index)"), .text(status)]) + }) + tableRows = TableRows( + rows: rows, + columns: ["id", "status"], + columnTypes: [.integer(rawType: "INT"), .text(rawType: "VARCHAR")] + ) + coordinator = TableViewCoordinator( + changeManager: AnyChangeManager(DataChangeManager()), + isEditable: true, + selectedRowIndices: .constant([]), + delegate: delegate, + layoutPersister: HighlightTestPersister() + ) + coordinator.tableRowsProvider = { [weak self] in self?.tableRows ?? TableRows() } + coordinator.tableRowsMutator = { [weak self] mutation in + guard let self else { return } + mutation(&self.tableRows) + } + coordinator.rebuildColumnMetadataCache(from: tableRows) + coordinator.updateCache() + } + + @discardableResult + func apply(_ rules: [HighlightRule]) -> Bool { + coordinator.syncHighlightRules(rules, tableRows: tableRows) + } + + func rowColor(_ row: Int) -> HighlightColor? { + coordinator.visualState(for: row).highlight.rowColor + } +} + +@Suite("Grid coordinator highlight rules") +@MainActor +struct TableViewCoordinatorHighlightTests { + private let paid = HighlightRule(columnName: "status", value: "paid", color: .green) + + @Test("Only the rows a rule matches carry its highlight") + func matchingRowsCarryTheHighlight() { + let grid = HighlightGrid(statuses: ["paid", "pending", "paid"]) + grid.apply([paid]) + + #expect(grid.rowColor(0) == .green) + #expect(grid.rowColor(1) == nil) + #expect(grid.rowColor(2) == .green) + } + + @Test("Changing the rules reports a change and recolours rows already evaluated") + func changingRulesRecolours() { + let grid = HighlightGrid(statuses: ["paid"]) + #expect(grid.apply([paid])) + #expect(grid.rowColor(0) == .green) + + var recolored = paid + recolored.color = .red + #expect(grid.apply([recolored])) + #expect(grid.rowColor(0) == .red) + #expect(!grid.apply([recolored])) + } + + @Test("An edit that flips a rule is reflected once the edit commits") + func editFlipsTheHighlight() { + let grid = HighlightGrid(statuses: ["pending"]) + grid.apply([paid]) + #expect(grid.rowColor(0) == nil) + + grid.coordinator.commitTypedCellEdit(row: 0, columnIndex: 1, newValue: .text("paid")) + + #expect(grid.tableRows.rows[0].values[1] == .text("paid")) + #expect(grid.rowColor(0) == .green) + } + + @Test("New rows under the same positional ids are evaluated afresh once the cache is dropped") + func positionalIdsAreNotServedStaleHighlights() { + let grid = HighlightGrid(statuses: ["paid"]) + grid.apply([paid]) + #expect(grid.rowColor(0) == .green) + + grid.tableRows.rows[0].values[1] = .text("pending") + grid.coordinator.invalidateDisplayCache() + + #expect(grid.rowColor(0) == nil) + } + + @Test("A fresh display state for a new page carries no highlights from the old one") + func freshDisplayStateStartsClean() { + let grid = HighlightGrid(statuses: ["paid"]) + grid.apply([paid]) + #expect(grid.rowColor(0) == .green) + + grid.tableRows.rows[0].values[1] = .text("pending") + grid.coordinator.adoptDisplayState(DataGridDisplayState()) + grid.apply([paid]) + + #expect(grid.rowColor(0) == nil) + } + + @Test("A grid whose owner supplies its own row state is never highlighted") + func delegateStateSuppressesHighlights() { + let delegate = StructureStateDelegate() + let grid = HighlightGrid(statuses: ["paid"], delegate: delegate) + grid.apply([paid]) + + #expect(grid.rowColor(0) == nil) + } + + @Test("The accessibility description names the rule that coloured the cell") + func accessibilityDescription() { + let grid = HighlightGrid(statuses: ["paid"]) + grid.apply([paid]) + + #expect(grid.coordinator.highlightDescription(row: 0, columnIndex: 0) == "status = “paid”") + } +} diff --git a/TableProUITests/HighlightRulesUITests.swift b/TableProUITests/HighlightRulesUITests.swift new file mode 100644 index 0000000000..5cb6e155c4 --- /dev/null +++ b/TableProUITests/HighlightRulesUITests.swift @@ -0,0 +1,106 @@ +// +// HighlightRulesUITests.swift +// TableProUITests +// + +import AppKit +import XCTest + +final class HighlightRulesUITests: UITestCase { + func testAddingARuleFromTheStatusBarKeepsItForTheTable() throws { + let app = try launchWithSampleDatabase() + let window = try readyWindow(of: app) + _ = try albumGrid(in: window) + + openRulesFromStatusBar(in: window) + let add = window.buttons["highlight-rules-add"].firstMatch + XCTAssertTrue(waitUntilHittable(add, timeout: 10), "The popover must offer Add Rule") + XCTAssertFalse(ruleCheckbox(in: window).exists, "A table nobody highlighted starts with no rules") + add.click() + XCTAssertTrue(ruleCheckbox(in: window).waitToExist(timeout: 10), "Add Rule must list a new rule") + app.typeText("1") + + app.typeKey(.escape, modifierFlags: []) + XCTAssertTrue(add.waitForNonExistence(timeout: 5), "Escape in the value field must close the popover") + + openRulesFromStatusBar(in: window) + XCTAssertTrue( + ruleCheckbox(in: window).waitToExist(timeout: 10), + "A rule belongs to the table, so reopening the popover lists it again" + ) + + let reopenedAdd = window.buttons["highlight-rules-add"].firstMatch + XCTAssertTrue(waitUntilHittable(reopenedAdd, timeout: 10)) + reopenedAdd.click() + XCTAssertTrue(waitForPredicate(timeout: 10) { self.ruleCheckboxes(in: window).count == 2 }) + app.typeKey(.escape, modifierFlags: []) + XCTAssertTrue(reopenedAdd.waitForNonExistence(timeout: 5)) + + openRulesFromStatusBar(in: window) + XCTAssertTrue(ruleCheckbox(in: window).waitToExist(timeout: 10)) + XCTAssertEqual(ruleCheckboxes(in: window).count, 1, "A rule closed without a value is not kept") + } + + func testTheCellMenuOffersHighlightAndOpensTheRules() throws { + let app = try launchWithSampleDatabase() + let window = try readyWindow(of: app) + let grid = try albumGrid(in: window) + + let cell = gridPoint(in: grid, of: window, dy: 70) + cell.click() + Thread.sleep(forTimeInterval: NSEvent.doubleClickInterval) + cell.rightClick() + + let highlight = window.menus.menuItems["Highlight"].firstMatch + XCTAssertTrue(highlight.waitToExist(timeout: 15), "A cell's context menu must offer Highlight") + highlight.hover() + + let showRules = contextMenuItem("Highlight Rules…", in: app) + XCTAssertTrue(waitUntilHittable(showRules, timeout: 10), "The Highlight submenu must offer Highlight Rules…") + showRules.click() + + XCTAssertTrue( + window.buttons["highlight-rules-add"].firstMatch.waitToExist(timeout: 10), + "Highlight Rules… must open the rules popover" + ) + } + + // MARK: - Helpers + + private func readyWindow(of app: XCUIApplication) throws -> XCUIElement { + let window = app.windows.matching(NSPredicate(format: "identifier != %@", "welcome")).firstMatch + XCTAssertTrue(window.waitToExist(timeout: 60), "The sample database produced no window") + XCTAssertTrue( + waitForPredicate(timeout: 30) { window.outlines.firstMatch.outlineRows.count > 1 }, + "The object browser must list the sample database's tables" + ) + return window + } + + private func albumGrid(in window: XCUIElement) throws -> XCUIElement { + let row = window.outlines.firstMatch.staticTexts + .matching(NSPredicate(format: "value == %@", "Table: Album")) + .firstMatch + XCTAssertTrue(row.waitToExist(timeout: 20), "The object browser must list Album") + clickAtCenter(row) + + let grid = window.tables.matching(identifier: "data-grid").firstMatch + XCTAssertTrue(grid.waitToExist(timeout: 30), "Album produced no data grid") + XCTAssertTrue(waitForClickableRows(in: grid), "Album must load rows before a cell can be highlighted") + return grid + } + + private func openRulesFromStatusBar(in window: XCUIElement) { + let button = window.buttons["result-status-highlight"] + XCTAssertTrue(waitUntilHittable(button, timeout: 15), "The status bar must offer Highlight") + button.click() + } + + private func ruleCheckbox(in window: XCUIElement) -> XCUIElement { + ruleCheckboxes(in: window).firstMatch + } + + private func ruleCheckboxes(in window: XCUIElement) -> XCUIElementQuery { + window.checkBoxes.matching(identifier: "highlight-rule-enabled") + } +} diff --git a/docs/connections/connection-form.mdx b/docs/connections/connection-form.mdx index adf4952cea..04b99c7ecb 100644 --- a/docs/connections/connection-form.mdx +++ b/docs/connections/connection-form.mdx @@ -128,6 +128,7 @@ Metadata connections, the extra ones TablePro opens to read a database's object | [SurrealDB](/databases/surrealdb) | 8000 | Yes | Yes | Yes | No | Yes | Yes | | [Elasticsearch](/databases/elasticsearch) | 9200 | No | Yes | No | No | No | No | | [Typesense](/databases/typesense) | 8108 | No | Yes | No | No | No | No | +| [Weaviate](/databases/weaviate) | 8080 | No | Yes | No | No | No | No | | [Snowflake](/databases/snowflake) | 443 | No | No | No | No | No | No | | [SQLite](/databases/sqlite) | File | No | No | No | No | No | No | | [DuckDB](/databases/duckdb) | File | No | No | No | No | No | No | diff --git a/docs/connections/ssh-tunneling.mdx b/docs/connections/ssh-tunneling.mdx index 84626b658d..fbdbc365d1 100644 --- a/docs/connections/ssh-tunneling.mdx +++ b/docs/connections/ssh-tunneling.mdx @@ -37,7 +37,7 @@ To share one SSH config across connections, save it with **Save Current as Profi Network section with SSH Tunnel selected and a saved profile in the Profile picker -**SSH Tunnel** is not offered on SQLite, PGlite, libSQL, Beancount, BigQuery, Spanner, Cloudflare D1, Cloudflare R2 SQL, DynamoDB, Elasticsearch, Typesense, or Snowflake: each is reached over a local file, a loopback socket, or a vendor HTTP API. +**SSH Tunnel** is not offered on SQLite, PGlite, libSQL, Beancount, BigQuery, Spanner, Cloudflare D1, Cloudflare R2 SQL, DynamoDB, Elasticsearch, Typesense, Weaviate, or Snowflake: each is reached over a local file, a loopback socket, or a vendor HTTP API. ## Authentication methods diff --git a/docs/connections/ssl.mdx b/docs/connections/ssl.mdx index 470bb8c21c..81b6c35e8e 100644 --- a/docs/connections/ssl.mdx +++ b/docs/connections/ssl.mdx @@ -43,7 +43,7 @@ A new connection starts on the mode that matches the driver's own default, and t | MySQL, MariaDB | Preferred | Tries TLS, then retries plain on an SSL handshake error. Auth and network errors are not retried | | SQL Server | Preferred | FreeTDS `encryption=request`, falls back to plain | | Teradata | Disabled | Opens a TLS transport, retries on a plain socket if it fails to come up | -| MongoDB, Redis, Cassandra, ClickHouse, Elasticsearch, Typesense, SurrealDB | Disabled | Nothing. No fallback exists, so Preferred forces TLS exactly like Required | +| MongoDB, Redis, Cassandra, ClickHouse, Elasticsearch, Typesense, SurrealDB, Weaviate | Disabled | Nothing. No fallback exists, so Preferred forces TLS exactly like Required | | etcd | Disabled | Nothing. The driver never reads these fields. Set **TLS Mode** on the Options section instead, and see [etcd](/databases/etcd) | | Trino | Disabled | Sends every request over HTTPS with no fallback, again like Required | | Oracle | Disabled | Connects in plain TCP, so it behaves like Disabled. A red warning appears under the picker; use Required to enforce TCPS | diff --git a/docs/databases/index.mdx b/docs/databases/index.mdx index c0d2e822b9..a5cb337dc9 100644 --- a/docs/databases/index.mdx +++ b/docs/databases/index.mdx @@ -1,11 +1,11 @@ --- title: Supported Databases -description: All 34 engines TablePro connects to, their default ports, and which ones need a plugin +description: All 35 engines TablePro connects to, their default ports, and which ones need a plugin --- import DriverCounts from "/snippets/driver-counts.mdx"; -Thirty-four engines, and every one of them is free to use. What differs between them is where the +Thirty-five engines, and every one of them is free to use. What differs between them is where the driver comes from, not what the license covers. @@ -48,6 +48,7 @@ driver comes from, not what the license covers. | [Trino](/databases/trino) | 8080 | Plugin | | [Typesense](/databases/typesense) | 8108 | Plugin | | [Turso](/databases/libsql) | API-based | Plugin | +| [Weaviate](/databases/weaviate) | 8080 | Plugin | Rows sharing a page share a driver. MariaDB reads as MySQL, ScyllaDB as Cassandra, Turso as libSQL, and Redshift, CockroachDB and PGlite all speak the PostgreSQL wire protocol. TiDB and Databend diff --git a/docs/databases/mysql.mdx b/docs/databases/mysql.mdx index 40e694b46d..a930f51cef 100644 --- a/docs/databases/mysql.mdx +++ b/docs/databases/mysql.mdx @@ -26,7 +26,7 @@ MySQL 8 accounts on `caching_sha2_password` connect on the first try, with no au | **Password** | empty | Stored in the macOS Keychain | | **Database** | empty | Optional. Leave it empty to browse every database | -The session character set is `utf8mb4`, so emoji and non-Latin text round-trip untouched. A connect attempt gives up after 10 seconds. +Every connection sets its session character set to `utf8mb4` once it logs in, over any `init_connect` the server runs, so emoji and non-Latin text round-trip untouched. A connect attempt gives up after 10 seconds. ## Connection URL @@ -76,6 +76,40 @@ Reconnecting costs a TCP connect, the TLS handshake and authentication: measured A release is refused, with the reason, while the session holds anything a reconnect would destroy: an open transaction, a temporary table, a prepared statement, a `GET_LOCK`, `LOCK TABLES`, a user variable, a changed session setting, or a stored routine call, whose body is opaque. +## Garbled non-Latin text + +A comment or value that reads `メール` where `メール` belongs was written by a client that sent UTF-8 while telling the server it was sending Latin 1. A `mysql` command-line client without a UTF-8 locale does that, and so does a MySQL 5.7 container loading its `docker-entrypoint-initdb.d` scripts, and so does any client on a server whose `init_connect` runs `SET NAMES latin1`. The server stored the garbled form, so every UTF-8 client shows the same thing. + +To work with such a database the way that client did, set **Encoding** to **UTF-8 via Latin 1** in **Options** and reconnect. Text written through Latin 1 then reads correctly, text stored correctly still reads correctly, and whatever you save is stored the way the old client stored it, so the application that wrote the data keeps reading it. + + +With **UTF-8 via Latin 1**, a correctly stored value you edit is saved in the garbled form, and a table or column whose non-Latin name was stored correctly cannot be opened. Use it on a database written through Latin 1, never on one where applications write UTF-8. + + +An SQL export taken with **UTF-8 via Latin 1** holds the text as it reads, in UTF-8. Restoring it gives a database with the text fixed, which the old client then reads as `?`. Restore it where you are moving off that client, not as a backup of the database it still writes to. + +To fix the stored text instead, convert it in place. The `WHERE` clause skips values that were stored correctly: + +```sql +UPDATE orders +SET note = CONVERT(CAST(CONVERT(note USING latin1) AS BINARY) USING utf8mb4) +WHERE note = CONVERT(CONVERT(note USING latin1) USING utf8mb4) + AND CONVERT(CAST(CONVERT(note USING latin1) AS BINARY) USING utf8mb4) IS NOT NULL; +``` + +Table comments take one `ALTER TABLE` each. This query writes them for the current database; run the statements it returns: + +```sql +SELECT CONCAT('ALTER TABLE `', TABLE_NAME, '` COMMENT = ', + QUOTE(CONVERT(CAST(CONVERT(TABLE_COMMENT USING latin1) AS BINARY) USING utf8mb4)), ';') +FROM information_schema.TABLES +WHERE TABLE_SCHEMA = DATABASE() AND TABLE_COMMENT <> '' + AND TABLE_COMMENT = CONVERT(CONVERT(TABLE_COMMENT USING latin1) USING utf8mb4) + AND CONVERT(CAST(CONVERT(TABLE_COMMENT USING latin1) AS BINARY) USING utf8mb4) IS NOT NULL; +``` + +A column comment is part of the column's definition. Copy it from the **Structure** tab while **UTF-8 via Latin 1** is on, then switch back to **UTF-8**, reconnect, and paste it into the same column. + ## SSL/TLS New connections default to **Preferred**: TLS first, dropping to plain text only after an SSL handshake error. Pick **Verify CA** with the provider's certificate for strict validation. See [SSL/TLS](/connections/ssl). diff --git a/docs/databases/weaviate.mdx b/docs/databases/weaviate.mdx new file mode 100644 index 0000000000..abb2d26107 --- /dev/null +++ b/docs/databases/weaviate.mdx @@ -0,0 +1,132 @@ +--- +title: Weaviate +description: Connect to Weaviate, browse collections, edit objects by uuid, and run GraphQL +--- + +import RegistryPlugin from "/snippets/registry-plugin.mdx"; + +Collections are tables, objects are rows, and `uuid` is the primary key. The editor speaks GraphQL, not SQL. Vector and hybrid search go through `/v1/graphql`. Browse and row edits use `/v1/objects`. + + + +## Quick setup + +Click **New Connection…**, select **Weaviate**, enter host and port, pick an **Auth Method**, then click **Save & Connect**. + +There is no Database field. One connection reaches one Weaviate instance, and its collections are the objects. + +## Connection settings + +| Field | Description | +|-------|-------------| +| **Host** | Node hostname. `localhost` for Docker, a Weaviate Cloud hostname for hosted clusters | +| **Port** | `8080` by default. Weaviate Cloud answers on `443` | +| **Auth Method** | None, or API Key | +| **API Key** | Sent as an `Authorization: Bearer` header, and only when **Auth Method** is API Key | +| **Skip TLS Verification** | Advanced section. Trusts any certificate, even under **Verify CA** or **Verify Identity** | + +## Authentication + +| Auth Method | What is sent | +|-------------|--------------| +| **None** | Nothing. For a local node with anonymous access | +| **API Key** | An `Authorization: Bearer` header. Paste the Weaviate Cloud key, or the key the local node was started with | + +Weaviate Cloud requires an API key and HTTPS. Set **SSL Mode** to **Required** (or **Verify Identity** if you have the CA) and **Auth Method** to API Key. + +## Browsing collections + +The sidebar lists collections from `GET /v1/schema`. + +Every grid has `uuid` first and `vector` last. `uuid` is the primary key. Both columns are read-only. Property columns come from the collection schema. Arrays and objects render as JSON in the cell. + +A vector is shown as a JSON number array. It is not a SQL BLOB and the inline editor does not write it back. + +Unfiltered pages load through `GET /v1/objects`. A column filter or a sort becomes a GraphQL `Get` with `where` and `sort`. + +## Filtering + +Filter values are typed from the collection schema: an `int` property filters as a number, a `date` property needs a full RFC 3339 timestamp such as `2024-01-31T00:00:00Z`, and an array property filters on one element. + +| Operator | What Weaviate runs | +|----------|--------------------| +| `=`, `!=` | `Equal`, `NotEqual` | +| `>`, `>=`, `<`, `<=` | Numbers and dates only | +| **CONTAINS**, **NOT CONTAINS**, **STARTS WITH**, **ENDS WITH** | `Like` with asterisk wildcards, text properties only | +| **IN**, **NOT IN** | `ContainsAny`, `ContainsNone` over a comma-separated list | +| **BETWEEN** | Two bounds, both inclusive | +| **IS NULL**, **IS NOT NULL** | `IsNull`, which needs `indexNullState` on the collection | +| **IS EMPTY**, **IS NOT EMPTY** | `len()`, which needs `indexPropertyLength` on the collection | + +**REGEX** has no Weaviate equivalent, and a filter Weaviate cannot run reports why instead of returning the whole collection. + +Grid edits are REST calls keyed by `uuid`. + +| Change | Request | +|--------|---------| +| New row | `POST /v1/objects`, with an `id` if you typed one | +| Edited cells | `PATCH /v1/objects/{uuid}?class=Collection` | +| Deleted row | `DELETE /v1/objects/{uuid}?class=Collection` | + +An update or delete with no `uuid` is skipped and logged. + +## GraphQL editor + +Type a GraphQL operation and run it. Weaviate's GraphQL API reads only: `{ Get { … } }` and `query { … }` work, and there is no mutation type. Writes go through REST. + +```graphql +{ + Get { + Article( + nearText: { concepts: ["search term"] } + limit: 10 + ) { + title + _additional { id distance } + } + } +} +``` + +`Get` responses render as a grid of the fields the query selected, with `_additional.id` mapped to `uuid`. The rest of `_additional`, such as `distance` and `score`, each become a column. Anything else, including `Aggregate`, is shown as formatted JSON. + +A REST line also runs, the way the other search drivers do: + +```http +GET /v1/schema +``` + +A path without `/v1` gets it, so `GET /nodes` reaches `/v1/nodes`. A body goes on the same line as the path or on the lines under it. + +## SSL/TLS + +New connections start on **Disabled**, plain HTTP. Every other mode goes over HTTPS. **Preferred** and **Required (skip verify)** accept a self-signed certificate. See [SSL/TLS](/connections/ssl). + +## Limitations + +- Oracle-style SQL is not available. Use GraphQL, or the REST console for `/v1/schema` and `/v1/objects`. +- gRPC is not used. +- Cross-references are properties, not foreign keys. There are no routines, triggers, or schema edits from Structure. +- Multi-tenancy is not exposed. Name a tenant in GraphQL if the collection requires one. +- Paging stops at row 10,000. Weaviate refuses an offset and limit that add up past `QUERY_MAXIMUM_RESULTS`, which defaults to 10,000. +- No [SSH tunnel](/connections/ssh-tunneling). +- The plugin is registry-only. It is not on iPhone or iPad. + +## Troubleshooting + +### Authentication failed: … + +The API key was rejected, or the node expects a key and **Auth Method** is None. Weaviate Cloud needs both HTTPS and an API key. + +### The connection drops after the API key changes + +A revoked or rotated key fails the next health check, because the check reads `/v1/meta` rather than the unauthenticated readiness endpoint. Paste the new key and connect again. + +### Connection failed: … + +A TLS or network failure. For a self-signed certificate set **SSL Mode** to **Required (skip verify)**, or turn on **Skip TLS Verification**. + +## Related + +- [Import & Export](/features/import-export) +- [Filtering](/features/filtering) diff --git a/docs/docs.json b/docs/docs.json index 43abd690e3..43c7fc135d 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -175,7 +175,8 @@ "databases/teradata", "databases/tidb", "databases/trino", - "databases/typesense" + "databases/typesense", + "databases/weaviate" ] }, { diff --git a/docs/features/data-grid.mdx b/docs/features/data-grid.mdx index dd8110ceae..9fc8d6c10c 100644 --- a/docs/features/data-grid.mdx +++ b/docs/features/data-grid.mdx @@ -1,9 +1,9 @@ --- title: Data Grid -description: Sort, size, hide, chart and copy the rows a table or a query puts in the grid +description: Sort, size, hide, highlight, chart and copy the rows a table or a query puts in the grid --- -Most of the grid's controls sit in the status bar beneath it. The view switcher is at the leading edge, the row count in the middle, then the columns, filter and page buttons at the trailing edge. +Most of the grid's controls sit in the status bar beneath it. The view switcher is at the leading edge, the row count in the middle, then the columns, highlight, filter and page buttons at the trailing edge. Data grid @@ -40,6 +40,23 @@ The header menu carries two filters that answer different questions. A value filter lives as long as the result does: switching view mode or tab keeps it, replacing the result clears it. **Fetch All** keeps it and applies it to the rows it loads. +## Highlighting + +Right-click a cell and open **Highlight**. A color under **Rows Where status = “paid”** tints every row holding that value; a color under **Cells Where status = “paid”** tints only that cell. The palette marks the color a matching rule already uses, and **Remove Highlight** takes the rule away. + +For anything other than an exact match, click the highlighter button in the status bar or choose **View > Highlight Rules**. A rule is a column, an operator from the [filter bar](/features/filtering), a value, a color, and **Row** or **Cell**. **Highlight Values…** in the header menu starts a rule on that column. A rule left without a value is dropped when the popover closes. + + + Highlight Rules popover listing three rules over an Invoice grid with tinted rows and cells + Highlight Rules popover listing three rules over an Invoice grid with tinted rows and cells + + +Rules run top to bottom and the first match colors the row, so drag the rule that should win to the top. A rule picked from the cell menu goes in first. A cell rule tints its own cell over the row's color. Rules read the stored value rather than the text a **Display As** format shows: a numeric column compares as numbers, a boolean column accepts `true`, `1`, `t` and `yes` alike, and `NULL` never satisfies a comparison: match it with **is NULL** or **is empty**. + +A row waiting to be inserted or deleted keeps its [change tracking](/features/change-tracking) tint over any rule, and a selected row shows the selection. On a table you edit, give row rules a color other than the green and red those tints use, so a highlight never reads as a pending change. + +Rules belong to the table, scoped to the connection, database, and schema, and follow the table through a rename. A query result that comes from one table uses that table's rules. Rules on any other query result are not saved, and go when the tab closes or the app quits. Saved rules stay on this Mac and do not sync. + ## Columns Drag a border to resize a column, or double-click it to fit the content; **Size to Fit** and **Size All Columns to Fit** on the header menu do the same. A fitted column stops at half the visible grid width. diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 0bfb69afd4..cc3664e5dc 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -133,6 +133,8 @@ As SQL, a results export writes `INSERT` statements only. A result set is the ou Splitting writes `dump.part1.sql`, `dump.part2.sql` and so on, rotating between statements so no part ends mid-`INSERT`. Restore the parts in order. A gzipped export is one file, so the two settings do not combine and the summary says so. + A MySQL or MariaDB dump is UTF-8 and says so: every part opens with `SET NAMES utf8mb4` and ends by putting the session's character set back, as `mysqldump` does. `mysql < dump.sql` then restores Japanese, emoji and other non-Latin text intact, even from a client whose default is Latin 1. + One snapshot opens `START TRANSACTION WITH CONSISTENT SNAPSHOT` on MySQL, `BEGIN ISOLATION LEVEL REPEATABLE READ` on PostgreSQL, and a deferred transaction on SQLite. It holds that transaction open for the whole export. Excluding the counter drops `AUTO_INCREMENT=` from the table options and leaves the column's own `AUTO_INCREMENT` attribute alone. Restoring rows sets the counter one past the highest key in the data, so a source counter that had run ahead of its rows, after deletes or a reset, does not carry over. diff --git a/docs/images/highlight-rules-dark.png b/docs/images/highlight-rules-dark.png new file mode 100644 index 0000000000..7f5c21fc9b Binary files /dev/null and b/docs/images/highlight-rules-dark.png differ diff --git a/docs/images/highlight-rules.png b/docs/images/highlight-rules.png new file mode 100644 index 0000000000..21febb50ad Binary files /dev/null and b/docs/images/highlight-rules.png differ diff --git a/docs/index.mdx b/docs/index.mdx index 4f67682ea0..062214abf9 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -1,6 +1,6 @@ --- title: Introduction -description: Native macOS database client for MySQL, PostgreSQL, SQLite, MongoDB, Redis, and 29 more +description: Native macOS database client for MySQL, PostgreSQL, SQLite, MongoDB, Redis, and 30 more --- import DriverCounts from "/snippets/driver-counts.mdx"; diff --git a/docs/snippets/driver-counts.mdx b/docs/snippets/driver-counts.mdx index 36442e09d9..b7e36c271a 100644 --- a/docs/snippets/driver-counts.mdx +++ b/docs/snippets/driver-counts.mdx @@ -1,3 +1,3 @@ -Five drivers ship inside the app and cover eleven databases. Twenty-one registry plugins cover the -other twenty-three and install on the first connection that needs one. See +Five drivers ship inside the app and cover eleven databases. Twenty-two registry plugins cover the +other twenty-four and install on the first connection that needs one. See [Plugins & Themes](/features/plugins). diff --git a/project.yml b/project.yml index c8a381d0bc..9bbb020930 100644 --- a/project.yml +++ b/project.yml @@ -476,6 +476,13 @@ targets: - Plugins/MongoDBDriverPlugin/MongoScriptText.swift - Plugins/MongoDBDriverPlugin/MongoStreamProjection.swift - Plugins/OracleDriverPlugin/OracleObjectQueries.swift + - Plugins/MySQLDriverPlugin/GeometryWKBParser.swift + - Plugins/MySQLDriverPlugin/MariaDBFieldClassifier.swift + - Plugins/MySQLDriverPlugin/MariaDBFieldMetadata.swift + - Plugins/MySQLDriverPlugin/MySQLCharacterSet.swift + - Plugins/MySQLDriverPlugin/MySQLColumnDecoding.swift + - Plugins/MySQLDriverPlugin/MySQLConnectionEncoding.swift + - Plugins/MySQLDriverPlugin/MySQLLatin1.swift - Plugins/MySQLDriverPlugin/MySQLColumnDefinitionSQL.swift - Plugins/MySQLDriverPlugin/MySQLCreateTableSQL.swift - Plugins/MySQLDriverPlugin/MySQLIdleRelease.swift @@ -501,6 +508,7 @@ targets: - Plugins/MSSQLDriverPlugin/MSSQLCheckConstraintDefinition.swift - Plugins/PostgreSQLDriverPlugin/ColumnQueryShape.swift - Plugins/PostgreSQLDriverPlugin/LibPQByteaDecoder.swift + - Plugins/PostgreSQLDriverPlugin/LibPQPluginError.swift - Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift - Plugins/PostgreSQLDriverPlugin/PostGISSpatialRewrite.swift - Plugins/PostgreSQLDriverPlugin/PostgreSQLCapabilities.swift @@ -563,6 +571,7 @@ targets: - Plugins/RedisDriverPlugin/RedisStatementGenerator.swift - Plugins/RedisDriverPlugin/RedisTopologyDiagnostics.swift - Plugins/SQLExportPlugin/SQLExportDDLRewriter.swift + - Plugins/SQLExportPlugin/SQLExportEncodingDeclaration.swift - Plugins/SQLExportPlugin/SQLExportFileWriter.swift - Plugins/SQLExportPlugin/SQLExportInsertMode.swift - Plugins/SQLExportPlugin/SQLExportModels.swift @@ -609,7 +618,7 @@ targets: dependencies: - target: TablePro - package: TableProCore - products: [TableProMSSQLCore, TableProNumberFormatting] + products: [TableProMSSQLCore, TableProNumberFormatting, TableProWeaviateCore] # The Kafka integration suite drives the real driver, so the test target links what # the plugin target links: zstd for decompression and NIO for the transport. - package: zstd @@ -1232,6 +1241,15 @@ targets: - package: TableProCore product: TableProNumberFormatting + WeaviateDriverPlugin: + templates: [DriverPlugin] + templateAttributes: + folder: WeaviateDriverPlugin + principalClass: WeaviatePlugin + dependencies: + - package: TableProCore + product: TableProWeaviateCore + # Compile-checks every plugin, including the registry-only ones the app does not # embed. CI builds this scheme so a registry plugin cannot rot between releases. aggregateTargets: @@ -1276,6 +1294,7 @@ aggregateTargets: - TeradataDriver - TrinoDriverPlugin - TypesenseDriverPlugin + - WeaviateDriverPlugin - XLSXExport scheme: {} diff --git a/scripts/check-mysql-charset-decoding.sh b/scripts/check-mysql-charset-decoding.sh new file mode 100755 index 0000000000..3ee16221ad --- /dev/null +++ b/scripts/check-mysql-charset-decoding.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# Compare the MySQL driver's character-set decoders against a real server. +# +# MySQLCharacterSet maps a server charset name to a Foundation encoding by hand, and MySQLLatin1 +# carries MySQL's own latin1 table, which is cp1252 with five bytes passed through as C1 +# controls. Both are transcriptions of what the server does, and Foundation's idea of a charset +# disagrees with MySQL's for several names that look identical (latin1, greek, hebrew, sjis). This +# asks the server how it converts every byte of every single-byte charset in the table, and a set +# of sample strings for the multibyte ones, and fails if the Swift decoders disagree. +# +# Usage: +# scripts/check-mysql-charset-decoding.sh [host] [port] [user] +# +# Needs the mysql client, xcrun swiftc, and a MySQL 8 or MariaDB 10.5+ server. The password, if +# any, comes from MYSQL_PWD. A byte the server leaves undefined may decode to a replacement +# character or pass through as its own code point, as MySQL's latin1 does. Exits non-zero on a +# disagreement. + +set -uo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-3306}" +USER_NAME="${3:-root}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PLUGIN="$ROOT/Plugins/MySQLDriverPlugin" + +command -v mysql > /dev/null || { + echo "mysql client not found" >&2 + exit 3 +} + +MYSQL=(mysql --no-defaults -h "$HOST" -P "$PORT" -u "$USER_NAME" -N -B -r --default-character-set=utf8mb4) +if ! "${MYSQL[@]}" -e "SELECT 1" > /dev/null 2>&1; then + echo "no MySQL at $HOST:$PORT for $USER_NAME" >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +cat > "$WORK/main.swift" <<'SWIFT' +import Foundation + +func bytes(fromHex hex: Substring) -> [UInt8]? { + guard hex.count % 2 == 0 else { return nil } + var result: [UInt8] = [] + var index = hex.startIndex + while index < hex.endIndex { + let next = hex.index(index, offsetBy: 2) + guard let byte = UInt8(hex[index.. String { + input.withUnsafeBytes { MySQLCharacterSet(serverName: name).decode($0) } +} + +let arguments = CommandLine.arguments +if arguments.count == 2, arguments[1] == "names" { + for name in ["latin1"] + MySQLCharacterSet.singleByteDecodedNames { + print("\(name) single") + } + for name in MySQLCharacterSet.multiByteDecodedNames { + print("\(name) multi") + } + exit(0) +} + +var failures = 0 +for line in (try String(contentsOfFile: arguments[1], encoding: .utf8)).split(separator: "\n") { + let fields = line.split(separator: "\t", omittingEmptySubsequences: false) + guard fields.count >= 3 else { continue } + let name = String(fields[0]) + switch fields[1] { + case "byte": + guard let byte = UInt8(fields[2], radix: 16), fields.count == 4, + fields[3] != "NULL", let server = bytes(fromHex: fields[3]) else { continue } + let undefined = server == [0x3F] && byte != 0x3F + let local = decode([byte], name) + let passThrough = String(Unicode.Scalar(byte)) + let acceptable = undefined ? ["\u{FFFD}", "?", passThrough] : [String(decoding: server, as: UTF8.self)] + if !acceptable.contains(local) { + print("\(name) byte \(fields[2]): server \(fields[3]) local \(local.unicodeScalars.map { String($0.value, radix: 16) })") + failures += 1 + } + case "sample": + guard fields.count == 5, let encoded = bytes(fromHex: fields[3]), let back = bytes(fromHex: fields[4]) else { + continue + } + let sample = String(fields[2]) + guard String(decoding: back, as: UTF8.self) == sample else { continue } + let local = decode(encoded, name) + if local != sample { + print("\(name) sample \(sample): local \(local)") + failures += 1 + } + default: + continue + } +} +print(failures == 0 ? "OK" : "\(failures) disagreements") +exit(failures == 0 ? 0 : 1) +SWIFT + +xcrun swiftc -O -o "$WORK/check" "$WORK/main.swift" \ + "$PLUGIN/MySQLCharacterSet.swift" "$PLUGIN/MySQLLatin1.swift" > "$WORK/build.log" 2>&1 || { + cat "$WORK/build.log" >&2 + exit 3 +} + +SAMPLES=("メール・記事紐付け" "～" "〜" "①" "髙" "¥" "\\" "‖" "¬" "㈱" '“' "€" "中文简体" "繁體中文" + "한국어" "Привет" "ґєії" "ąčęėįšųūž" "ğışİ" "łóźżćńśŁ" "àéüß" "😀" "ｱｲｳ" "∑" "×" $'\xe2\x80\x94' "£") + +NAMES="$("$WORK/check" names)" && [ -n "$NAMES" ] || { + echo "the decoder listed no character sets" >&2 + exit 3 +} + +: > "$WORK/server.tsv" +: > "$WORK/mysql.err" +while read -r NAME KIND; do + KNOWN="$("${MYSQL[@]}" -e "SELECT COUNT(*) FROM information_schema.CHARACTER_SETS WHERE CHARACTER_SET_NAME = '$NAME'" 2>> "$WORK/mysql.err")" + if [ "$KNOWN" != "1" ]; then + echo "skipped $NAME: this server has no such character set" + continue + fi + if [ "$KIND" = "single" ]; then + COLUMNS="" + for BYTE in $(seq 0 255); do + HEX="$(printf '%02X' "$BYTE")" + COLUMNS="$COLUMNS${COLUMNS:+,}CONCAT('$NAME\tbyte\t$HEX\t', IFNULL(HEX(CONVERT(CONVERT(UNHEX('$HEX') USING $NAME) USING utf8mb4)), 'NULL'))" + done + "${MYSQL[@]}" -e "SELECT $COLUMNS" 2>> "$WORK/mysql.err" | tr '\t' '\n' | paste - - - - >> "$WORK/server.tsv" + fi + for SAMPLE in "${SAMPLES[@]}"; do + LITERAL="${SAMPLE//\\/\\\\}" + "${MYSQL[@]}" -e "SELECT '$NAME', 'sample', _utf8mb4'$LITERAL', HEX(CONVERT(_utf8mb4'$LITERAL' USING $NAME)), HEX(CONVERT(CONVERT(_utf8mb4'$LITERAL' USING $NAME) USING utf8mb4))" \ + >> "$WORK/server.tsv" 2>> "$WORK/mysql.err" + done +done <<< "$NAMES" + +if grep -v -e '^WARNING' -e '^$' "$WORK/mysql.err" > /dev/null; then + grep -v -e '^WARNING' "$WORK/mysql.err" >&2 + exit 3 +fi + +"$WORK/check" "$WORK/server.tsv"