diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f675d6a8..c7877c64b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Count exactly** next to an estimated row total, to replace the estimate with a real count when you want it. + +### Fixed + +- Opening a large MongoDB collection no longer hangs. Row counts that back the pagination display now give up after 5 seconds and leave the estimate in place instead of holding the tab. +- Stop now cancels a running MongoDB query on the server, rather than leaving it running until it finishes on its own. +- A MongoDB query that runs out of time now says so, and points at the missing index that usually causes it. +- Running a MongoDB query with no limit no longer pulls the whole collection into memory before applying the row limit. +- Exporting a MongoDB query that has a skip or limit now exports that range, not the whole collection. +- A row count estimate of zero from a table that was never analyzed no longer displays as an empty table. + ## [0.61.0] - 2026-07-30 ### Added diff --git a/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift b/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift index 015260a9f..9713dc4bb 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift @@ -111,6 +111,14 @@ extension MongoDBConnection { } defer { bson_destroy(optsBson) } + let session = attachCancellableSession(client: client, opts: optsBson) + defer { + if let session { + releaseSessionLsid() + mongoc_client_session_destroy(session) + } + } + let col = try getCollection(client, database: database, collection: collection) defer { mongoc_collection_destroy(col) } @@ -124,6 +132,20 @@ extension MongoDBConnection { return try iterateCursor(cursor) } + /// Binds the operation to a server-side session so `cancelCurrentQuery` can abort it with + /// `killSessions`. Returns nil when the deployment does not support sessions, in which case the + /// operation still runs, just without server-side cancellation. + func attachCancellableSession(client: OpaquePointer, opts: OpaquePointer) -> OpaquePointer? { + var error = bson_error_t() + guard let session = mongoc_client_start_session(client, nil, &error) else { return nil } + guard mongoc_client_session_append(session, opts, &error) else { + mongoc_client_session_destroy(session) + return nil + } + adoptSessionLsid(session) + return session + } + func aggregateSync( client: OpaquePointer, database: String, collection: String, pipeline: String ) throws -> (docs: [[String: Any]], isTruncated: Bool) { @@ -157,7 +179,8 @@ extension MongoDBConnection { } func countDocumentsSync( - client: OpaquePointer, database: String, collection: String, filter: String + client: OpaquePointer, database: String, collection: String, + filter: String, background: Bool ) throws -> Int64 { try checkCancelled() @@ -169,19 +192,61 @@ extension MongoDBConnection { let col = try getCollection(client, database: database, collection: collection) defer { mongoc_collection_destroy(col) } - let timeoutMS = queryTimeoutMS + let maxTimeMS = effectiveMaxTimeMS(background: background) + let isUnfiltered = filter.trimmingCharacters(in: .whitespacesAndNewlines) == "{}" + + let hinted = try runCountDocuments( + collection: col, filter: filterBson, maxTimeMS: maxTimeMS, hintIdIndex: isUnfiltered + ) + try checkCancelled() + switch hinted { + case .count(let value): + return value + case .failed(let error): + guard isUnfiltered, MongoDBTimeoutPolicy.shouldRetryWithoutHint(errorCode: error.code) else { + throw error + } + } + + let unhinted = try runCountDocuments( + collection: col, filter: filterBson, maxTimeMS: maxTimeMS, hintIdIndex: false + ) + try checkCancelled() + switch unhinted { + case .count(let value): + return value + case .failed(let error): + throw error + } + } + + private enum CountOutcome { + case count(Int64) + case failed(MongoDBError) + } + + private func runCountDocuments( + collection col: OpaquePointer, filter: OpaquePointer, + maxTimeMS: Int32?, hintIdIndex: Bool + ) throws -> CountOutcome { + var optsFields: [String] = [] + if let maxTimeMS { + optsFields.append("\"maxTimeMS\": \(maxTimeMS)") + } + if hintIdIndex { + optsFields.append("\"hint\": \"_id_\"") + } + var optsBson: OpaquePointer? - if timeoutMS > 0 { - optsBson = jsonToBson("{\"maxTimeMS\": \(timeoutMS)}") + if !optsFields.isEmpty { + optsBson = jsonToBson("{\(optsFields.joined(separator: ", "))}") } - defer { if let opts = optsBson { bson_destroy(opts) } } + defer { if let optsBson { bson_destroy(optsBson) } } var error = bson_error_t() - let count = mongoc_collection_count_documents(col, filterBson, optsBson, nil, nil, &error) - - try checkCancelled() - guard count >= 0 else { throw makeError(error) } - return count + let count = mongoc_collection_count_documents(col, filter, optsBson, nil, nil, &error) + guard count >= 0 else { return .failed(makeError(error)) } + return .count(count) } func insertOneSync( @@ -367,6 +432,7 @@ extension MongoDBConnection { var docPtr: OpaquePointer? var sample: [[String: Any]] = [] var projection: MongoStreamProjection? + var emitted = 0 while mongoc_cursor_next(cursor, &docPtr) { if Task.isCancelled { @@ -380,12 +446,18 @@ extension MongoDBConnection { if let projection { continuation.yield(.rows([projection.row(for: dict, convert: streamCellValue)])) + emitted += 1 + if emitted >= PluginRowLimits.emergencyMax { + logger.warning("Streamed result truncated at \(PluginRowLimits.emergencyMax) documents") + break + } continue } sample.append(dict) if sample.count >= MongoStreamProjection.sampleSize { projection = openStream(sample: sample, continuation: continuation) + emitted += sample.count sample = [] } } diff --git a/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift b/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift index 082e95f02..1ae6d94c4 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift @@ -68,12 +68,17 @@ final class MongoDBConnection: @unchecked Sendable { private let replicaSet: String? private let extraUriParams: [String: String] + private let controlQueue = DispatchQueue(label: "com.TablePro.mongodb.control", qos: .userInitiated) + private let stateLock = NSLock() private var _isConnected: Bool = false private var _isShuttingDown: Bool = false private var _cachedServerVersion: String? private var _isCancelled: Bool = false private var _queryTimeoutMS: Int32 = 0 + #if canImport(CLibMongoc) + private var _activeSessionLsid: OpaquePointer? + #endif var isConnected: Bool { stateLock.lock() @@ -106,6 +111,10 @@ final class MongoDBConnection: @unchecked Sendable { stateLock.unlock() } + func effectiveMaxTimeMS(background: Bool) -> Int32? { + MongoDBTimeoutPolicy.resolveMaxTimeMS(ambientMS: queryTimeoutMS, background: background) + } + // MARK: - Initialization init( @@ -339,6 +348,8 @@ final class MongoDBConnection: @unchecked Sendable { #if canImport(CLibMongoc) let handle = client client = nil + let staleLsid = _activeSessionLsid + _activeSessionLsid = nil #endif _isConnected = false _cachedServerVersion = nil @@ -347,6 +358,7 @@ final class MongoDBConnection: @unchecked Sendable { stateLock.unlock() #if canImport(CLibMongoc) + if let staleLsid { bson_destroy(staleLsid) } if let handle = handle { queue.async { mongoc_client_destroy(handle) } } @@ -358,8 +370,63 @@ final class MongoDBConnection: @unchecked Sendable { func cancelCurrentQuery() { stateLock.lock() _isCancelled = true + #if canImport(CLibMongoc) + let lsid = _activeSessionLsid.map { bson_copy($0) } + #endif stateLock.unlock() + + #if canImport(CLibMongoc) + guard let lsid else { return } + killSession(lsid: lsid) + #endif + } + + #if canImport(CLibMongoc) + func adoptSessionLsid(_ session: OpaquePointer) { + guard let lsid = mongoc_client_session_get_lsid(session) else { return } + let copy = bson_copy(lsid) + stateLock.lock() + let previous = _activeSessionLsid + _activeSessionLsid = copy + stateLock.unlock() + if let previous { bson_destroy(previous) } + } + + func releaseSessionLsid() { + stateLock.lock() + let previous = _activeSessionLsid + _activeSessionLsid = nil + stateLock.unlock() + if let previous { bson_destroy(previous) } + } + + /// Aborts the in-flight operation server-side. The connection's own queue is blocked inside a + /// libmongoc call at this point, so the `killSessions` command needs its own client. + private func killSession(lsid: OpaquePointer) { + let uriString = buildUri() + controlQueue.async { + defer { bson_destroy(lsid) } + guard let controlClient = mongoc_client_new(uriString) else { return } + defer { mongoc_client_destroy(controlClient) } + + let command = bson_new() + defer { bson_destroy(command) } + let sessions = bson_new() + defer { bson_destroy(sessions) } + + bson_append_document(sessions, "0", -1, lsid) + bson_append_array(command, "killSessions", -1, sessions) + + let reply = bson_new() + defer { bson_destroy(reply) } + var error = bson_error_t() + let killed = mongoc_client_command_simple(controlClient, "admin", command, nil, reply, &error) + if !killed { + logger.warning("killSessions failed with code \(error.code, privacy: .public)") + } + } } + #endif /// Throws if cancellation was requested, resetting the flag atomically. /// Safe to call from any thread. @@ -493,7 +560,12 @@ final class MongoDBConnection: @unchecked Sendable { #endif } - func countDocuments(database: String, collection: String, filter: String) async throws -> Int64 { + func countDocuments( + database: String, + collection: String, + filter: String, + background: Bool + ) async throws -> Int64 { #if canImport(CLibMongoc) resetCancellation() return try await pluginDispatchAsync(on: queue) { [self] in @@ -502,7 +574,8 @@ final class MongoDBConnection: @unchecked Sendable { } try checkCancelled() let count = try countDocumentsSync( - client: client, database: database, collection: collection, filter: filter + client: client, database: database, collection: collection, + filter: filter, background: background ) try checkCancelled() return count @@ -512,7 +585,11 @@ final class MongoDBConnection: @unchecked Sendable { #endif } - func estimatedDocumentCount(database: String, collection: String) async throws -> Int64 { + func estimatedDocumentCount( + database: String, + collection: String, + background: Bool + ) async throws -> Int64 { #if canImport(CLibMongoc) resetCancellation() return try await pluginDispatchAsync(on: queue) { [self] in @@ -523,8 +600,14 @@ final class MongoDBConnection: @unchecked Sendable { let col = try getCollection(client, database: database, collection: collection) defer { mongoc_collection_destroy(col) } + var opts: OpaquePointer? + if let maxTimeMS = effectiveMaxTimeMS(background: background) { + opts = jsonToBson("{\"maxTimeMS\": \(maxTimeMS)}") + } + defer { if let opts { bson_destroy(opts) } } + var error = bson_error_t() - let count = mongoc_collection_estimated_document_count(col, nil, nil, nil, &error) + let count = mongoc_collection_estimated_document_count(col, opts, nil, nil, &error) if count < 0 { throw makeError(error) } @@ -639,7 +722,9 @@ final class MongoDBConnection: @unchecked Sendable { collection: String, filter: String, sort: String?, - projection: String? + projection: String?, + skip: Int = 0, + limit: Int? = nil ) -> AsyncThrowingStream { #if canImport(CLibMongoc) let queue = self.queue @@ -683,6 +768,12 @@ final class MongoDBConnection: @unchecked Sendable { let obj = try? JSONSerialization.jsonObject(with: data) { optsJson["projection"] = obj } + if skip > 0 { + optsJson["skip"] = skip + } + if let limit, limit > 0 { + optsJson["limit"] = limit + } let timeoutMS = queryTimeoutMS if timeoutMS > 0 { optsJson["maxTimeMS"] = timeoutMS diff --git a/Plugins/MongoDBDriverPlugin/MongoDBFindLimitPolicy.swift b/Plugins/MongoDBDriverPlugin/MongoDBFindLimitPolicy.swift new file mode 100644 index 000000000..1391c1fc1 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBFindLimitPolicy.swift @@ -0,0 +1,28 @@ +// +// MongoDBFindLimitPolicy.swift +// MongoDBDriverPlugin +// + +import Foundation +import TableProPluginKit + +enum MongoDBFindLimitPolicy { + /// Server-side limit for a user query. One document beyond the row cap is requested so the + /// caller can tell "exactly a full page" from "there is more", without ever asking the server + /// for the whole collection. + static func fetchLimit(parsedLimit: Int?, rowCap: Int?) -> Int { + var candidates: [Int] = [] + if let parsedLimit, parsedLimit > 0 { + candidates.append(parsedLimit) + } + if let rowCap, rowCap > 0 { + candidates.append(rowCap + 1) + } + return min(candidates.min() ?? PluginRowLimits.emergencyMax, PluginRowLimits.emergencyMax) + } + + static func isTruncated(rowCount: Int, rowCap: Int?) -> Bool { + guard let rowCap, rowCap > 0 else { return false } + return rowCount > rowCap + } +} diff --git a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index a1e63d634..444fea62f 100644 --- a/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -133,6 +133,64 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { try await execute(query: query) } + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]? + ) async throws -> PluginQueryResult { + let startTime = Date() + + guard let conn = mongoConnection else { + throw MongoDBPluginError.notConnected + } + + let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.lowercased() != "select 1", + case .find(let collection, let filter, var options) = try MongoShellParser.parse(trimmed) else { + return try await capToRowCap(execute(query: query), rowCap: rowCap) + } + + options.limit = MongoDBFindLimitPolicy.fetchLimit(parsedLimit: options.limit, rowCap: rowCap) + + do { + let result = try await executeOperation( + .find(collection: collection, filter: filter, options: options), + connection: conn, + startTime: startTime + ) + return capToRowCap(result, rowCap: rowCap) + } catch { + throw mapExecutionError(error) + } + } + + private func capToRowCap(_ result: PluginQueryResult, rowCap: Int?) -> PluginQueryResult { + guard let rowCap, MongoDBFindLimitPolicy.isTruncated(rowCount: result.rows.count, rowCap: rowCap) else { + return result + } + return PluginQueryResult( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + rows: Array(result.rows.prefix(rowCap)), + rowsAffected: result.rowsAffected, + executionTime: result.executionTime, + isTruncated: true, + statusMessage: result.statusMessage + ) + } + + private func mapExecutionError(_ error: Error) -> Error { + guard let mongoError = error as? MongoDBError, + MongoDBTimeoutPolicy.isTimeoutCode(mongoError.code), + let maxTimeMS = mongoConnection?.effectiveMaxTimeMS(background: false) else { + return error + } + return MongoDBError( + code: mongoError.code, + message: MongoDBTimeoutPolicy.timeoutMessage(maxTimeMS: maxTimeMS) + ) + } + // MARK: - Query Cancellation func cancelQuery() throws { @@ -288,7 +346,9 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { throw MongoDBPluginError.notConnected } - let count = try await conn.estimatedDocumentCount(database: currentDb, collection: table) + let count = try await conn.estimatedDocumentCount( + database: currentDb, collection: table, background: true + ) return Int(count) } @@ -296,13 +356,33 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { table: String, filters: [(column: String, op: String, value: String)], logicMode: String + ) async throws -> Int? { + try await documentCount(table: table, filters: filters, logicMode: logicMode, background: true) + } + + func fetchExactRowCount( + table: String, + schema: String?, + filters: [(column: String, op: String, value: String)], + logicMode: String + ) async throws -> Int? { + try await documentCount(table: table, filters: filters, logicMode: logicMode, background: false) + } + + private func documentCount( + table: String, + filters: [(column: String, op: String, value: String)], + logicMode: String, + background: Bool ) async throws -> Int? { guard let conn = mongoConnection else { throw MongoDBPluginError.notConnected } let filterJson = MongoDBQueryBuilder().buildFilterDocument(from: filters, logicMode: logicMode) - let count = try await conn.countDocuments(database: currentDb, collection: table, filter: filterJson) + let count = try await conn.countDocuments( + database: currentDb, collection: table, filter: filterJson, background: background + ) return Int(count) } @@ -640,7 +720,8 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { case .find(let collection, let filter, let options): return conn.streamFind( database: db, collection: collection, filter: filter, - sort: options.sort, projection: options.projection + sort: options.sort, projection: options.projection, + skip: options.skip ?? 0, limit: options.limit ) case .aggregate(let collection, let pipeline): return conn.streamAggregate( @@ -705,7 +786,9 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return buildPluginResult(from: result.docs, startTime: startTime, isTruncated: result.isTruncated) case .countDocuments(let collection, let filter): - let count = try await conn.countDocuments(database: db, collection: collection, filter: filter) + let count = try await conn.countDocuments( + database: db, collection: collection, filter: filter, background: false + ) return PluginQueryResult( columns: ["count"], columnTypeNames: ["Int64"], rows: [[.text(String(count))]], rowsAffected: 0, @@ -788,6 +871,19 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { executionTime: Date().timeIntervalSince(startTime) ) + default: + return try await executeCommandOperation(operation, connection: conn, startTime: startTime) + } + } + + private func executeCommandOperation( + _ operation: MongoOperation, + connection conn: MongoDBConnection, + startTime: Date + ) async throws -> PluginQueryResult { + let db = currentDb + + switch operation { case .createIndex(let collection, let keys, let options): var indexDoc = "{\"key\": \(keys)" if let opts = options { @@ -857,6 +953,9 @@ final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { rows: [["1"]], rowsAffected: 0, executionTime: Date().timeIntervalSince(startTime) ) + + default: + throw MongoDBPluginError.unsupportedOperation } } diff --git a/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift b/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift new file mode 100644 index 000000000..74b0e3723 --- /dev/null +++ b/Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift @@ -0,0 +1,49 @@ +// +// MongoDBTimeoutPolicy.swift +// MongoDBDriverPlugin +// + +import Foundation + +enum MongoDBServerErrorCode { + static let badValue: UInt32 = 2 + static let indexNotFound: UInt32 = 27 + static let maxTimeMSExpired: UInt32 = 50 + static let cursorKilled: UInt32 = 237 +} + +enum MongoDBTimeoutPolicy { + static let backgroundMaxTimeMS: Int32 = 5_000 + + static func resolveMaxTimeMS(ambientMS: Int32, background: Bool) -> Int32? { + guard background else { + return ambientMS > 0 ? ambientMS : nil + } + guard ambientMS > 0 else { + return backgroundMaxTimeMS + } + return min(ambientMS, backgroundMaxTimeMS) + } + + static func shouldRetryWithoutHint(errorCode: UInt32) -> Bool { + errorCode == MongoDBServerErrorCode.badValue || errorCode == MongoDBServerErrorCode.indexNotFound + } + + static func isTimeoutCode(_ errorCode: UInt32) -> Bool { + errorCode == MongoDBServerErrorCode.maxTimeMSExpired + } + + static func timeoutMessage(maxTimeMS: Int32) -> String { + let seconds = max(1, Int((Double(maxTimeMS) / 1_000).rounded())) + return String( + format: String( + localized: """ + The query timed out after %d seconds. Sorting or filtering on a field with no index \ + makes MongoDB read the whole collection, even when you only ask for one page. \ + Add an index for that field, or raise the query timeout in Settings. + """ + ), + seconds + ) + } +} diff --git a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 361d75d3a..740e08bf0 100644 --- a/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -132,6 +132,8 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { func buildFilteredQuery(table: String, schema: String?, filters: [(column: String, op: String, value: String)], logicMode: String, sortColumns: [(columnIndex: Int, ascending: Bool)], columns: [String], limit: Int, offset: Int) -> String? // Filtered row count (optional, for NoSQL plugins; SQL plugins use COUNT(*) WHERE) func fetchFilteredRowCount(table: String, filters: [(column: String, op: String, value: String)], logicMode: String) async throws -> Int? + // User-initiated exact row count (allowed to be slow; background count caps must not apply) + func fetchExactRowCount(table: String, schema: String?, filters: [(column: String, op: String, value: String)], logicMode: String) async throws -> Int? // Statement generation (optional, for NoSQL plugins) func generateStatements(table: String, columns: [String], primaryKeyColumns: [String], changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set) -> [(statement: String, parameters: [PluginCellValue])]? func generateStatements(table: String, schema: String?, columns: [String], primaryKeyColumns: [String], changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set) -> [(statement: String, parameters: [PluginCellValue])]? @@ -321,6 +323,9 @@ public extension PluginDatabaseDriver { buildFilteredQuery(table: table, filters: filters, logicMode: logicMode, sortColumns: sortColumns, columns: columns, limit: limit, offset: offset) } func fetchFilteredRowCount(table: String, filters: [(column: String, op: String, value: String)], logicMode: String) async throws -> Int? { nil } + func fetchExactRowCount(table: String, schema: String?, filters: [(column: String, op: String, value: String)], logicMode: String) async throws -> Int? { + try await fetchFilteredRowCount(table: table, filters: filters, logicMode: logicMode) + } func generateStatements(table: String, columns: [String], primaryKeyColumns: [String], changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set) -> [(statement: String, parameters: [PluginCellValue])]? { nil } func generateStatements(table: String, schema: String?, columns: [String], primaryKeyColumns: [String], changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set) -> [(statement: String, parameters: [PluginCellValue])]? { generateStatements( diff --git a/TablePro/Core/Coordinators/PaginationCoordinator.swift b/TablePro/Core/Coordinators/PaginationCoordinator.swift index eae727b89..27d1380f7 100644 --- a/TablePro/Core/Coordinators/PaginationCoordinator.swift +++ b/TablePro/Core/Coordinators/PaginationCoordinator.swift @@ -129,9 +129,11 @@ final class PaginationCoordinator { // MARK: - Cancel Current Query func cancelCurrentQuery() { - let hadInFlightTask = parent.currentQueryTask != nil + let hadInFlightTask = parent.currentQueryTask != nil || parent.currentRowCountTask != nil parent.currentQueryTask?.cancel() parent.currentQueryTask = nil + parent.currentRowCountTask?.cancel() + parent.currentRowCountTask = nil parent.queryGeneration += 1 if hadInFlightTask, let driver = DatabaseManager.shared.driver(for: parent.connectionId) { try? driver.cancelQuery() @@ -139,15 +141,74 @@ final class PaginationCoordinator { parent.toolbarState.setExecuting(false) for idx in parent.tabManager.tabs.indices { if parent.tabManager.tabs[idx].execution.isExecuting - || parent.tabManager.tabs[idx].pagination.isLoadingMore { + || parent.tabManager.tabs[idx].pagination.isLoadingMore + || parent.tabManager.tabs[idx].pagination.isCountingExact { parent.tabManager.mutate(at: idx) { tab in tab.execution.isExecuting = false tab.pagination.isLoadingMore = false + tab.pagination.isCountingExact = false } } } } + // MARK: - Exact Row Count + + func requestExactRowCount() { + guard let (tab, index) = parent.tabManager.selectedTabAndIndex, + tab.tabType == .table, + !tab.pagination.isCountingExact, + let tableName = tab.tableContext.tableName, !tableName.isEmpty else { return } + + let tabId = tab.id + let schemaName = tab.tableContext.schemaName + let filters = tab.filterState.hasAppliedFilters ? tab.filterState.appliedFilters : [] + let logicMode = tab.filterState.filterLogicMode + let isNonSQL = PluginManager.shared.editorLanguage(for: parent.connection.type) != .sql + let countSQL = isNonSQL ? nil : parent.queryBuilder.buildFilteredCountQuery( + tableName: tableName, schemaName: schemaName, filters: filters, logicMode: logicMode + ) + + parent.tabManager.mutate(at: index) { $0.pagination.isCountingExact = true } + + parent.currentRowCountTask = Task(priority: .userInitiated) { [parent] in + let count = await Self.exactRowCount( + connectionId: parent.connectionId, + tableName: tableName, + filters: filters, + logicMode: logicMode, + countSQL: countSQL + ) + + guard !Task.isCancelled else { return } + parent.tabManager.mutate(tabId: tabId) { tab in + tab.pagination.isCountingExact = false + guard let count, count >= 0 else { return } + tab.pagination.totalRowCount = count + tab.pagination.isApproximateRowCount = false + } + } + } + + private static func exactRowCount( + connectionId: UUID, + tableName: String, + filters: [TableFilter], + logicMode: FilterLogicMode, + countSQL: String? + ) async -> Int? { + try? await DatabaseManager.shared.withMetadataDriver(connectionId: connectionId, workload: .bulk) { driver in + guard let countSQL else { + return try await driver.fetchExactRowCount( + table: tableName, filters: filters, logicMode: logicMode + ) + } + let result = try await driver.execute(query: countSQL) + guard let countStr = result.rows.first?.first?.asText else { return Int?.none } + return Int(countStr) + } + } + // MARK: - Fetch All Rows func fetchAllRows() { diff --git a/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift b/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift index be9c34b9e..e648a6f61 100644 --- a/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift +++ b/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift @@ -404,7 +404,7 @@ extension QueryExecutionCoordinator { ) { let isNonSQL = PluginManager.shared.editorLanguage(for: connectionType) != .sql - Task(priority: .utility) { [weak self, parent] in + parent.currentRowCountTask = Task(priority: .utility) { [weak self, parent] in guard let self else { return } guard !parent.isTearingDown else { return } @@ -465,14 +465,9 @@ extension QueryExecutionCoordinator { await MainActor.run { guard capturedGeneration == parent.queryGeneration else { return } parent.tabManager.mutate(tabId: tabId) { tab in - switch outcome { - case let .count(value, isApproximate): - tab.pagination.totalRowCount = value - tab.pagination.isApproximateRowCount = isApproximate - case .clear: - tab.pagination.totalRowCount = nil - tab.pagination.isApproximateRowCount = false - } + let applied = outcome.appliedTotal + tab.pagination.totalRowCount = applied.total + tab.pagination.isApproximateRowCount = applied.isApproximate } } } @@ -555,7 +550,15 @@ enum RowCountPlan: Equatable { case filteredNonSQL(filters: [TableFilter], logicMode: FilterLogicMode) } -private enum RowCountOutcome { +enum RowCountOutcome: Equatable { case count(Int, isApproximate: Bool) case clear + + /// An estimate of zero or less means "unknown", never "this table is empty": an un-analyzed + /// table reports 0 or -1 while holding millions of rows. An exact zero is trustworthy. + var appliedTotal: (total: Int?, isApproximate: Bool) { + guard case let .count(value, isApproximate) = self else { return (nil, false) } + guard value > 0 || (value == 0 && !isApproximate) else { return (nil, false) } + return (value, isApproximate) + } } diff --git a/TablePro/Core/Database/DatabaseDriver.swift b/TablePro/Core/Database/DatabaseDriver.swift index 97faa7972..3ec2e705e 100644 --- a/TablePro/Core/Database/DatabaseDriver.swift +++ b/TablePro/Core/Database/DatabaseDriver.swift @@ -115,6 +115,10 @@ protocol DatabaseDriver: AnyObject, Sendable { /// Returns nil when the driver can't count a filtered set, so the caller falls back. func fetchFilteredRowCount(table: String, filters: [TableFilter], logicMode: FilterLogicMode) async throws -> Int? + /// Fetch an exact row count for a user-initiated request. Drivers that cap their automatic + /// counts to keep browsing responsive must not apply that cap here. + func fetchExactRowCount(table: String, filters: [TableFilter], logicMode: FilterLogicMode) async throws -> Int? + /// Fetch the DDL (CREATE TABLE statement) for a specific table func fetchTableDDL(table: String) async throws -> String @@ -406,6 +410,9 @@ extension DatabaseDriver { func fetchApproximateRowCount(table: String) async throws -> Int? { nil } func fetchFilteredRowCount(table: String, filters: [TableFilter], logicMode: FilterLogicMode) async throws -> Int? { nil } + func fetchExactRowCount(table: String, filters: [TableFilter], logicMode: FilterLogicMode) async throws -> Int? { + try await fetchFilteredRowCount(table: table, filters: filters, logicMode: logicMode) + } func supportedMaintenanceOperations() -> [String]? { nil } func maintenanceStatements(operation: String, table: String?, options: [String: String]) -> [String]? { nil } diff --git a/TablePro/Core/Plugins/PluginDriverAdapter.swift b/TablePro/Core/Plugins/PluginDriverAdapter.swift index de01405e5..205d5519f 100644 --- a/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -316,6 +316,18 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable { ) } + func fetchExactRowCount(table: String, filters: [TableFilter], logicMode: FilterLogicMode) async throws -> Int? { + let tuples = filters + .filter { $0.isEnabled && !$0.columnName.isEmpty } + .map(\.asPluginFilterTuple) + return try await pluginDriver.fetchExactRowCount( + table: table, + schema: pluginDriver.currentSchema, + filters: tuples, + logicMode: logicMode == .and ? "and" : "or" + ) + } + func fetchTableDDL(table: String) async throws -> String { try await pluginDriver.fetchTableDDL(table: table, schema: pluginDriver.currentSchema) } diff --git a/TablePro/Models/Query/QueryTabState.swift b/TablePro/Models/Query/QueryTabState.swift index 2349da3a9..c63f63d03 100644 --- a/TablePro/Models/Query/QueryTabState.swift +++ b/TablePro/Models/Query/QueryTabState.swift @@ -185,6 +185,7 @@ struct PaginationState: Equatable { var currentOffset: Int = 0 // Current OFFSET for SQL query var isLoading: Bool = false var isApproximateRowCount: Bool = false // True when totalRowCount is from fast estimate + var isCountingExact: Bool = false // True while a user-requested exact count is running // Result truncation state (query tabs) var hasMoreRows: Bool = false diff --git a/TablePro/Views/Main/Child/MainEditorContentView.swift b/TablePro/Views/Main/Child/MainEditorContentView.swift index 7982c4da6..580ff2a35 100644 --- a/TablePro/Views/Main/Child/MainEditorContentView.swift +++ b/TablePro/Views/Main/Child/MainEditorContentView.swift @@ -799,7 +799,8 @@ struct MainEditorContentView: View { onLast: onLastPage, onPageSizeChange: onPageSizeChange, onShowAll: onShowAll, - onGoToPage: onGoToPage + onGoToPage: onGoToPage, + onRequestExactCount: { coordinator.paginationCoordinator.requestExactRowCount() } ), columnState: StatusBarColumnState( hidden: tab.columnLayout.hiddenColumns, diff --git a/TablePro/Views/Main/Child/MainStatusBarView.swift b/TablePro/Views/Main/Child/MainStatusBarView.swift index 700acb621..181750a68 100644 --- a/TablePro/Views/Main/Child/MainStatusBarView.swift +++ b/TablePro/Views/Main/Child/MainStatusBarView.swift @@ -15,6 +15,7 @@ struct PaginationCallbacks { let onPageSizeChange: (Int) -> Void let onShowAll: () -> Void let onGoToPage: (Int) -> Void + let onRequestExactCount: () -> Void } struct StatusBarColumnState { @@ -116,6 +117,24 @@ struct MainStatusBarView: View { .foregroundStyle(.secondary) } + if snapshot.tabType == .table, !snapshot.pagination.isLoadingMore, + snapshot.pagination.isApproximateRowCount || snapshot.pagination.totalRowCount == nil { + if snapshot.pagination.isCountingExact { + ProgressView() + .controlSize(.small) + .accessibilityLabel(String(localized: "Counting rows")) + } else { + Button { + paginationCallbacks.onRequestExactCount() + } label: { + Text("Count exactly") + .font(.caption) + } + .buttonStyle(.link) + .help(String(localized: "Replace the estimate with an exact row count.")) + } + } + if snapshot.tabType == .query && snapshot.pagination.hasMoreRows && !snapshot.pagination.isLoadingMore { Text("ยท") .font(.caption) diff --git a/TablePro/Views/Main/MainContentCoordinator.swift b/TablePro/Views/Main/MainContentCoordinator.swift index e49a01013..2c43b07f1 100644 --- a/TablePro/Views/Main/MainContentCoordinator.swift +++ b/TablePro/Views/Main/MainContentCoordinator.swift @@ -194,6 +194,7 @@ final class MainContentCoordinator { @ObservationIgnored internal var queryGeneration: Int = 0 @ObservationIgnored internal var currentQueryTask: Task? + @ObservationIgnored internal var currentRowCountTask: Task? @ObservationIgnored internal var tableLoadTasks: [UUID: (token: UUID, task: Task)] = [:] @ObservationIgnored internal var redisDatabaseSwitchTask: Task? @ObservationIgnored private var changeManagerUpdateTask: Task? diff --git a/TableProTests/Core/Coordinators/RowCountPlanTests.swift b/TableProTests/Core/Coordinators/RowCountPlanTests.swift index faf57fb98..3f4aa9fe5 100644 --- a/TableProTests/Core/Coordinators/RowCountPlanTests.swift +++ b/TableProTests/Core/Coordinators/RowCountPlanTests.swift @@ -82,3 +82,45 @@ struct RowCountPlanTests { #expect(plan == .filteredNonSQL(filters: state.appliedFilters, logicMode: state.filterLogicMode)) } } + +@Suite("RowCountOutcome") +struct RowCountOutcomeTests { + @Test("A positive estimate is applied and stays marked approximate") + func positiveEstimateApplies() { + let applied = RowCountOutcome.count(4_600_000, isApproximate: true).appliedTotal + #expect(applied.total == 4_600_000) + #expect(applied.isApproximate) + } + + @Test("An estimate of zero means unknown, not an empty table") + func zeroEstimateIsUnknown() { + let applied = RowCountOutcome.count(0, isApproximate: true).appliedTotal + #expect(applied.total == nil) + #expect(!applied.isApproximate) + } + + @Test("A negative estimate is never shown as a total") + func negativeEstimateIsUnknown() { + let applied = RowCountOutcome.count(-1, isApproximate: true).appliedTotal + #expect(applied.total == nil) + } + + @Test("An exact zero is trustworthy and reported as an empty table") + func exactZeroIsApplied() { + let applied = RowCountOutcome.count(0, isApproximate: false).appliedTotal + #expect(applied.total == 0) + #expect(!applied.isApproximate) + } + + @Test("A negative exact count is still rejected") + func negativeExactIsUnknown() { + #expect(RowCountOutcome.count(-5, isApproximate: false).appliedTotal.total == nil) + } + + @Test("Clearing reports an unknown total") + func clearIsUnknown() { + let applied = RowCountOutcome.clear.appliedTotal + #expect(applied.total == nil) + #expect(!applied.isApproximate) + } +} diff --git a/TableProTests/Core/MongoDB/MongoDBFindLimitPolicyTests.swift b/TableProTests/Core/MongoDB/MongoDBFindLimitPolicyTests.swift new file mode 100644 index 000000000..0c4944a61 --- /dev/null +++ b/TableProTests/Core/MongoDB/MongoDBFindLimitPolicyTests.swift @@ -0,0 +1,67 @@ +// +// MongoDBFindLimitPolicyTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite("MongoDB Find Limit Policy") +struct MongoDBFindLimitPolicyTests { + @Suite("fetchLimit") + struct FetchLimitTests { + @Test("A parsed limit is honoured as-is") + func parsedLimitWins() { + #expect(MongoDBFindLimitPolicy.fetchLimit(parsedLimit: 20, rowCap: nil) == 20) + } + + @Test("A row cap asks for one extra document so truncation is detectable") + func rowCapAsksForOneMore() { + #expect(MongoDBFindLimitPolicy.fetchLimit(parsedLimit: nil, rowCap: 500) == 501) + } + + @Test("The smaller of the parsed limit and the row cap wins") + func smallerBoundWins() { + #expect(MongoDBFindLimitPolicy.fetchLimit(parsedLimit: 20, rowCap: 500) == 20) + #expect(MongoDBFindLimitPolicy.fetchLimit(parsedLimit: 5_000, rowCap: 500) == 501) + } + + @Test("An unbounded query never asks for more than the emergency ceiling") + func unboundedFallsBackToCeiling() { + #expect(MongoDBFindLimitPolicy.fetchLimit(parsedLimit: nil, rowCap: nil) == PluginRowLimits.emergencyMax) + } + + @Test("A non-positive bound is ignored rather than sent to the server") + func ignoresNonPositiveBounds() { + #expect(MongoDBFindLimitPolicy.fetchLimit(parsedLimit: 0, rowCap: nil) == PluginRowLimits.emergencyMax) + #expect(MongoDBFindLimitPolicy.fetchLimit(parsedLimit: nil, rowCap: 0) == PluginRowLimits.emergencyMax) + } + + @Test("The emergency ceiling is never exceeded") + func neverExceedsCeiling() { + let limit = MongoDBFindLimitPolicy.fetchLimit( + parsedLimit: nil, rowCap: PluginRowLimits.emergencyMax + ) + #expect(limit == PluginRowLimits.emergencyMax) + } + } + + @Suite("isTruncated") + struct IsTruncatedTests { + @Test("Exactly the cap is not truncated") + func exactlyCapIsNotTruncated() { + #expect(!MongoDBFindLimitPolicy.isTruncated(rowCount: 500, rowCap: 500)) + } + + @Test("One beyond the cap is truncated") + func oneBeyondCapIsTruncated() { + #expect(MongoDBFindLimitPolicy.isTruncated(rowCount: 501, rowCap: 500)) + } + + @Test("No cap means never truncated") + func noCapIsNeverTruncated() { + #expect(!MongoDBFindLimitPolicy.isTruncated(rowCount: 1_000_000, rowCap: nil)) + } + } +} diff --git a/TableProTests/Core/MongoDB/MongoDBTimeoutPolicyTests.swift b/TableProTests/Core/MongoDB/MongoDBTimeoutPolicyTests.swift new file mode 100644 index 000000000..85fce7788 --- /dev/null +++ b/TableProTests/Core/MongoDB/MongoDBTimeoutPolicyTests.swift @@ -0,0 +1,91 @@ +// +// MongoDBTimeoutPolicyTests.swift +// TableProTests +// + +import Foundation +import Testing + +@Suite("MongoDB Timeout Policy") +struct MongoDBTimeoutPolicyTests { + @Suite("resolveMaxTimeMS") + struct ResolveMaxTimeMSTests { + @Test("Background counts are capped even when no ambient timeout is configured") + func backgroundWithNoAmbientTimeout() { + let resolved = MongoDBTimeoutPolicy.resolveMaxTimeMS(ambientMS: 0, background: true) + #expect(resolved == MongoDBTimeoutPolicy.backgroundMaxTimeMS) + } + + @Test("Background counts never exceed the cap") + func backgroundClampsLargeAmbientTimeout() { + let resolved = MongoDBTimeoutPolicy.resolveMaxTimeMS(ambientMS: 60_000, background: true) + #expect(resolved == MongoDBTimeoutPolicy.backgroundMaxTimeMS) + } + + @Test("Background counts honour an ambient timeout shorter than the cap") + func backgroundHonoursShorterAmbientTimeout() { + let resolved = MongoDBTimeoutPolicy.resolveMaxTimeMS(ambientMS: 2_000, background: true) + #expect(resolved == 2_000) + } + + @Test("User-initiated work uses the ambient timeout") + func foregroundUsesAmbientTimeout() { + let resolved = MongoDBTimeoutPolicy.resolveMaxTimeMS(ambientMS: 60_000, background: false) + #expect(resolved == 60_000) + } + + @Test("User-initiated work stays unbounded when the user chose no limit") + func foregroundRespectsNoLimit() { + let resolved = MongoDBTimeoutPolicy.resolveMaxTimeMS(ambientMS: 0, background: false) + #expect(resolved == nil) + } + } + + @Suite("shouldRetryWithoutHint") + struct RetryTests { + @Test("A rejected hint retries without it") + func retriesOnBadValue() { + #expect(MongoDBTimeoutPolicy.shouldRetryWithoutHint(errorCode: MongoDBServerErrorCode.badValue)) + #expect(MongoDBTimeoutPolicy.shouldRetryWithoutHint(errorCode: MongoDBServerErrorCode.indexNotFound)) + } + + @Test("A timeout is authoritative and never retried") + func doesNotRetryOnTimeout() { + #expect(!MongoDBTimeoutPolicy.shouldRetryWithoutHint(errorCode: MongoDBServerErrorCode.maxTimeMSExpired)) + } + + @Test("Unrelated failures are not retried") + func doesNotRetryOnOtherErrors() { + #expect(!MongoDBTimeoutPolicy.shouldRetryWithoutHint(errorCode: MongoDBServerErrorCode.cursorKilled)) + #expect(!MongoDBTimeoutPolicy.shouldRetryWithoutHint(errorCode: 13)) + } + } + + @Suite("timeoutMessage") + struct TimeoutMessageTests { + @Test("Reports the timeout in seconds") + func reportsSeconds() { + let message = MongoDBTimeoutPolicy.timeoutMessage(maxTimeMS: 60_000) + #expect(message.contains("60")) + } + + @Test("A sub-second timeout still reports at least one second") + func clampsToOneSecond() { + let message = MongoDBTimeoutPolicy.timeoutMessage(maxTimeMS: 200) + #expect(message.contains("1")) + } + + @Test("Names the missing index as the likely cause") + func mentionsIndex() { + let message = MongoDBTimeoutPolicy.timeoutMessage(maxTimeMS: 5_000) + #expect(message.localizedCaseInsensitiveContains("index")) + } + } + + @Test("Only MaxTimeMSExpired counts as a timeout") + func timeoutCodeDetection() { + #expect(MongoDBTimeoutPolicy.isTimeoutCode(50)) + #expect(!MongoDBTimeoutPolicy.isTimeoutCode(0)) + #expect(!MongoDBTimeoutPolicy.isTimeoutCode(292)) + } +} diff --git a/TableProTests/PluginTestSources/MongoDBFindLimitPolicy.swift b/TableProTests/PluginTestSources/MongoDBFindLimitPolicy.swift new file mode 120000 index 000000000..fbd279740 --- /dev/null +++ b/TableProTests/PluginTestSources/MongoDBFindLimitPolicy.swift @@ -0,0 +1 @@ +../../Plugins/MongoDBDriverPlugin/MongoDBFindLimitPolicy.swift \ No newline at end of file diff --git a/TableProTests/PluginTestSources/MongoDBTimeoutPolicy.swift b/TableProTests/PluginTestSources/MongoDBTimeoutPolicy.swift new file mode 120000 index 000000000..1d6122cf0 --- /dev/null +++ b/TableProTests/PluginTestSources/MongoDBTimeoutPolicy.swift @@ -0,0 +1 @@ +../../Plugins/MongoDBDriverPlugin/MongoDBTimeoutPolicy.swift \ No newline at end of file diff --git a/TableProTests/Views/Main/MainStatusBarLayoutTests.swift b/TableProTests/Views/Main/MainStatusBarLayoutTests.swift index 0821a36e2..c1dab3aa3 100644 --- a/TableProTests/Views/Main/MainStatusBarLayoutTests.swift +++ b/TableProTests/Views/Main/MainStatusBarLayoutTests.swift @@ -27,7 +27,8 @@ struct MainStatusBarLayoutTests { onLast: {}, onPageSizeChange: { _ in }, onShowAll: {}, - onGoToPage: { _ in } + onGoToPage: { _ in }, + onRequestExactCount: {} ), columnState: StatusBarColumnState( hidden: [], diff --git a/docs/databases/mongodb.mdx b/docs/databases/mongodb.mdx index 9fc1fa9a9..4f231b9b8 100644 --- a/docs/databases/mongodb.mdx +++ b/docs/databases/mongodb.mdx @@ -127,6 +127,10 @@ db.users.countDocuments({"role": "admin"}) **Timeout**: Verify host/port, check network and firewall, whitelist your IP in MongoDB Atlas. +**A collection is slow to open**: A sort or a filter on a field with no index makes MongoDB read every document, even when you only ask for 20 rows, because it has to look at all of them to find which 20 come first. Check the Structure tab for an index on the field you sorted or filtered by, and add one if it is missing. `Cmd+.` stops the query on the server. + +**The row total shows `~`**: That is an estimate read from collection metadata, which is instant. TablePro caps the automatic count at 5 seconds and keeps the estimate if the server takes longer, so browsing is never held up by counting. **Count exactly** runs a real count with your configured query timeout. Views and time-series collections have no metadata count, so their estimate can be missing. + ## Limitations - Transactions are not exposed. Statements always run standalone, on any topology. diff --git a/docs/features/data-grid.mdx b/docs/features/data-grid.mdx index 9265f40a9..9d77cbfa6 100644 --- a/docs/features/data-grid.mdx +++ b/docs/features/data-grid.mdx @@ -145,7 +145,9 @@ Copies follow the grid as shown: hidden columns are left out and columns keep th **Table tabs** page through data. The status bar has a rows-per-page menu (5 to 1,000, a custom size, or **All rows**) and First / Previous / Next / Last controls. Click the page indicator (for example `3 / 12`) to type a page number and jump to it. Set the default page size in **Settings > Data**. -Large tables show an estimated total instead of running a slow `COUNT(*)`. **Settings > Data > Count rows if estimate less than** sets the threshold below which TablePro counts exactly. +Large tables show an estimated total instead of running a slow `COUNT(*)`. **Settings > Data > Count rows if estimate less than** sets the threshold below which TablePro counts exactly. An estimated total is prefixed with `~`. **Count exactly** next to the total replaces an estimate, or fills in a total shown as `?`, with a real count. That count can take a while on a large table, so it runs in the background and `Cmd+.` cancels it. + +An estimate of zero from a table the server has never analyzed is treated as unknown rather than as an empty table, so you get no total instead of a wrong one. **Query tabs** cap results at 10,000 rows by default. Your query is sent exactly as you wrote it; TablePro stops reading once it reaches the cap. A query with its own `LIMIT`, `FETCH FIRST`, or `TOP` is not capped. When the cap trims a result, the status bar shows **Fetch All** to load the rest (with a confirmation, since large results use a lot of memory). Run one query without the cap via **Execute Without Limit** (`Option+Cmd+Enter`). Adjust the cap in **Settings > Data** (**Truncate query results**, **Row cap**).