diff --git a/README.md b/README.md index 80d5ffa..3dfb526 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ import FetchCore import RAGCore import SwiftlyFetch -let library = try await SwiftlyFetchLibrary.default() +let library = SwiftlyFetchLibrary.default() try await library.addDocument( FetchDocumentRecord( @@ -81,7 +81,7 @@ For lower-level semantic retrieval, import `RAGCore` and `RAGKit` directly: import RAGCore import RAGKit -let kb = try await KnowledgeBase.hashingDefault() +let kb = KnowledgeBase.hashingDefault() try await kb.addDocument( Document( @@ -113,7 +113,7 @@ import FetchCore import RAGCore import SwiftlyFetch -let library = try await SwiftlyFetchLibrary.default() +let library = SwiftlyFetchLibrary.default() let mutation = try await library.addDocument( FetchDocumentRecord( @@ -142,7 +142,7 @@ For semantic retrieval, use `KnowledgeBase` from `RAGKit`: import RAGCore import RAGKit -let localKB = try await KnowledgeBase.hashingDefault() +let localKB = KnowledgeBase.hashingDefault() let appleKB = try await KnowledgeBase.naturalLanguageDefault(languageHint: "en") let semanticStore = FileManager.default .temporaryDirectory @@ -205,9 +205,10 @@ Current defaults: - plain text uses paragraph chunking - markdown uses parser-backed heading-aware chunking - markdown link destinations stay out of chunk text by default, but `HeadingAwareMarkdownChunker(linkDestinationMetadataMode: .include)` can record raw destinations in chunk metadata when downstream indexing or fetch-oriented work needs them -- `hashingDefault()` gives a deterministic local path for tests and examples -- `naturalLanguageDefault()` uses the Apple Natural Language backend on supported platforms +- `hashingDefault()` gives a synchronous deterministic local path for tests and examples +- `naturalLanguageDefault()` uses the Apple Natural Language backend on supported platforms and remains throwing because backend setup can fail - `persistentHashingDefault(configuration:dimension:)` and `persistentNaturalLanguageDefault(configuration:languageHint:)` use the same retrieval defaults with a Core Data-backed semantic vector index +- `SwiftlyFetchLibrary.default()` is synchronous and non-throwing because it uses deterministic in-memory defaults; persistent constructors remain async and throwing - metadata filtering supports explicit exclusions, ordered comparisons for `int`, `double`, and `date`, plus case-insensitive `startsWith` and `endsWith` string matching - markdown list items keep heading and immediate lead-in context in chunk text, and also carry structured chunk metadata for list kind, lead-in, ordinal, and heading path - markdown block quotes stay secondary by default, but are promoted into the primary retrieval stream when they make up more than one third of the document's chunkable block structure diff --git a/ROADMAP.md b/ROADMAP.md index 59a05bc..540e8d5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -228,9 +228,9 @@ In Progress - [x] Plan the first `SwiftlyFetch` umbrella facade in maintainer docs. - [x] Add a narrow bridge from `FetchDocumentRecord` to `RAGCore.Document`. - [x] Add an umbrella ingestion surface only after the semantic persisted index is stable. -- [ ] Decide whether the remaining Core Data vector-index XCTest coverage should migrate to Swift Testing now that the current XCTest path is stable. -- [ ] Decide whether `KnowledgeBase.hashingDefault(dimension:)` should keep its current `async throws` shape for API consistency or become a synchronous non-throwing convenience in a future API polish pass. -- [ ] Decide when to optimize `CoreDataVectorIndex.search` beyond the current pragmatic v1 in-memory ranking path, such as by using Core Data fetch batching, predicate pre-filtering, or a future ANN-backed query path. +- [x] Keep the remaining Core Data vector-index coverage on XCTest because the framework-heavy Core Data lanes are stable there, while Swift Testing stays the default for ordinary package behavior. +- [x] Make `KnowledgeBase.hashingDefault(dimension:)` and `SwiftlyFetchLibrary.default()` synchronous and non-throwing because their deterministic in-memory construction paths have no async or throwing setup work. +- [x] Defer `CoreDataVectorIndex.search` optimization beyond the current pragmatic v1 in-memory ranking path until real corpus scale shows a need; likely future paths include Core Data fetch batching, predicate pre-filtering, or a future ANN-backed query path. ### Exit Criteria diff --git a/Sources/RAGKit/KnowledgeBase+NaturalLanguage.swift b/Sources/RAGKit/KnowledgeBase+NaturalLanguage.swift index 26d1207..b3e9d7f 100644 --- a/Sources/RAGKit/KnowledgeBase+NaturalLanguage.swift +++ b/Sources/RAGKit/KnowledgeBase+NaturalLanguage.swift @@ -1,7 +1,7 @@ import RAGCore public extension KnowledgeBase { - static func hashingDefault(dimension: Int = 64) async throws -> KnowledgeBase { + static func hashingDefault(dimension: Int = 64) -> KnowledgeBase { KnowledgeBase( chunker: DefaultChunker(), embedder: HashingEmbedder(dimension: dimension), diff --git a/Sources/SwiftlyFetch/SwiftlyFetchLibrary.swift b/Sources/SwiftlyFetch/SwiftlyFetchLibrary.swift index 9d8dcb7..c09c0e7 100644 --- a/Sources/SwiftlyFetch/SwiftlyFetchLibrary.swift +++ b/Sources/SwiftlyFetch/SwiftlyFetchLibrary.swift @@ -22,8 +22,8 @@ public actor SwiftlyFetchLibrary { self.documentMapper = documentMapper } - public static func `default`() async throws -> SwiftlyFetchLibrary { - try await SwiftlyFetchLibrary( + public static func `default`() -> SwiftlyFetchLibrary { + SwiftlyFetchLibrary( fetchLibrary: .default(), knowledgeBase: .hashingDefault(), retryStore: InMemorySwiftlyFetchSemanticRetryStore() diff --git a/Tests/FetchKitTests/CoreDataFetchDocumentStoreTests.swift b/Tests/FetchKitTests/CoreDataFetchDocumentStoreTests.swift index f890522..d19d0f9 100644 --- a/Tests/FetchKitTests/CoreDataFetchDocumentStoreTests.swift +++ b/Tests/FetchKitTests/CoreDataFetchDocumentStoreTests.swift @@ -3,6 +3,9 @@ import Foundation import XCTest @testable import FetchKit +// Keep the Core Data-backed document-store coverage on XCTest. These tests exercise +// framework-heavy persistence paths, and XCTest has been the stable runner for this +// repository's Core Data lanes after Swift Testing exposed executor-assumption issues. final class CoreDataFetchDocumentStoreTests: XCTestCase { func testCoreDataFetchDocumentStoreRoundTripsRecord() async throws { let store = try await CoreDataFetchDocumentStore() diff --git a/Tests/RAGKitTests/CoreDataVectorIndexTests.swift b/Tests/RAGKitTests/CoreDataVectorIndexTests.swift index d664a85..92ac55c 100644 --- a/Tests/RAGKitTests/CoreDataVectorIndexTests.swift +++ b/Tests/RAGKitTests/CoreDataVectorIndexTests.swift @@ -2,6 +2,9 @@ import RAGCore @testable import RAGKit import XCTest +// Keep the Core Data-backed semantic index coverage on XCTest. These tests exercise +// framework-heavy persistence paths, and XCTest has been the stable runner for this +// repository's Core Data lanes after Swift Testing exposed executor-assumption issues. final class CoreDataVectorIndexTests: XCTestCase { func testCoreDataVectorIndexPersistsChunksAcrossReopen() async throws { let storeURL = temporaryStoreURL() diff --git a/Tests/RAGKitTests/KnowledgeBaseTests.swift b/Tests/RAGKitTests/KnowledgeBaseTests.swift index 7373d42..5ed3653 100644 --- a/Tests/RAGKitTests/KnowledgeBaseTests.swift +++ b/Tests/RAGKitTests/KnowledgeBaseTests.swift @@ -891,7 +891,7 @@ struct KnowledgeBaseRetrievalTests { @Test("KnowledgeBase hashingDefault uses heading-aware markdown chunking by default") func knowledgeBaseHashingDefaultPrefersMarkdownAwareChunking() async throws { - let knowledgeBase = try await KnowledgeBase.hashingDefault(dimension: 32) + let knowledgeBase = KnowledgeBase.hashingDefault(dimension: 32) try await knowledgeBase.addDocument( Document( diff --git a/Tests/SwiftlyFetchTests/SwiftlyFetchLibraryTests.swift b/Tests/SwiftlyFetchTests/SwiftlyFetchLibraryTests.swift index b6f1aa5..9863911 100644 --- a/Tests/SwiftlyFetchTests/SwiftlyFetchLibraryTests.swift +++ b/Tests/SwiftlyFetchTests/SwiftlyFetchLibraryTests.swift @@ -11,7 +11,7 @@ import Testing struct SwiftlyFetchLibraryTests { @Test("Default facade ingests one document into conventional and semantic search") func defaultFacadeIngestsOneDocumentIntoBothSearchModes() async throws { - let library = try await SwiftlyFetchLibrary.default() + let library = SwiftlyFetchLibrary.default() let record = FetchDocumentRecord( id: "doc-apple", title: "Apple Guide", @@ -82,9 +82,9 @@ struct SwiftlyFetchLibraryTests { reason: "Test retry" ) ) - let library = try SwiftlyFetchLibrary( + let library = SwiftlyFetchLibrary( fetchLibrary: fetchLibrary, - knowledgeBase: await KnowledgeBase.hashingDefault(), + knowledgeBase: KnowledgeBase.hashingDefault(), retryStore: retryStore ) @@ -132,9 +132,9 @@ struct SwiftlyFetchLibraryTests { nextRetryAt: Date().addingTimeInterval(-60) ) ) - let library = try SwiftlyFetchLibrary( + let library = SwiftlyFetchLibrary( fetchLibrary: fetchLibrary, - knowledgeBase: await KnowledgeBase.hashingDefault(), + knowledgeBase: KnowledgeBase.hashingDefault(), retryStore: retryStore ) @@ -262,9 +262,9 @@ struct SwiftlyFetchLibraryTests { reason: "Test retry" ) ) - let library = try SwiftlyFetchLibrary( + let library = SwiftlyFetchLibrary( fetchLibrary: FetchKitLibrary(), - knowledgeBase: await KnowledgeBase.hashingDefault(), + knowledgeBase: KnowledgeBase.hashingDefault(), retryStore: retryStore ) @@ -397,7 +397,7 @@ struct SwiftlyFetchLibraryTests { } private func indexedFixtureLibrary() async throws -> SwiftlyFetchLibrary { - let library = try await SwiftlyFetchLibrary.default() + let library = SwiftlyFetchLibrary.default() for record in GutenbergMiniCorpus.records + TinyStoriesMiniCorpus.records { try await library.addDocument(record) diff --git a/docs/maintainers/hybrid-search-persistence-plan.md b/docs/maintainers/hybrid-search-persistence-plan.md index 37ffb82..a4a8eae 100644 --- a/docs/maintainers/hybrid-search-persistence-plan.md +++ b/docs/maintainers/hybrid-search-persistence-plan.md @@ -112,6 +112,20 @@ The first convenience constructors are: These constructors keep the same chunker and embedder defaults as the in-memory defaults while swapping in the Core Data-backed vector index. +The in-memory deterministic hashing constructor is intentionally synchronous and non-throwing: + +- `KnowledgeBase.hashingDefault(dimension:)` + +That constructor only assembles `DefaultChunker`, `HashingEmbedder`, and `InMemoryVectorIndex`, so it should not ask callers to write `try await`. Keep `naturalLanguageDefault(...)` throwing because the Apple Natural Language embedder can fail during setup, and keep the persistent constructors async/throwing because Core Data-backed vector index setup awaits persistent-store loading and can fail. + +For the same reason, the umbrella `SwiftlyFetchLibrary.default()` constructor is synchronous and non-throwing while it uses only in-memory deterministic dependencies. Keep `SwiftlyFetchLibrary.macOSPersistentLibrary(...)` async/throwing because it opens persistent Core Data stores and may retry pending index work. + +## Validation And Search Optimization Decisions + +Keep the Core Data-backed semantic vector-index tests on XCTest for now. Swift Testing remains the default for ordinary package behavior, but the Core Data persistence lanes are framework-heavy and XCTest has been the stable runner after earlier Swift Testing executor-assumption failures in this repository's Core Data-backed validation. Treat this as an intentional validation boundary, not forgotten migration work. + +Keep `CoreDataVectorIndex.search` on the current pragmatic v1 path until real corpus scale shows that it is too slow or too memory-heavy. The current backend loads persisted chunks, decodes embeddings, applies metadata filters, and ranks in memory behind the `VectorIndex` protocol. That is simple and correct for the current package stage. Likely future optimization paths are Core Data fetch batching, predicate pre-filtering for metadata filters, or a separate ANN-backed vector backend if corpus size eventually earns that extra storage/query complexity. + ## Follow-Up Design Work The next architecture work should focus on shared corpus ingestion rather than another standalone index backend. The detailed umbrella plan lives in [swiftlyfetch-facade-plan.md](./swiftlyfetch-facade-plan.md). diff --git a/docs/maintainers/retrieval-package-plan.md b/docs/maintainers/retrieval-package-plan.md index 8ed7da3..a00ec05 100644 --- a/docs/maintainers/retrieval-package-plan.md +++ b/docs/maintainers/retrieval-package-plan.md @@ -158,7 +158,7 @@ Implemented today: - `KnowledgeBase` - `NaturalLanguageEmbedder` - `AppleContextualEmbeddingBackend` - - convenience constructors for `hashingDefault()`, `naturalLanguageDefault()`, `persistentHashingDefault(configuration:dimension:)`, and `persistentNaturalLanguageDefault(configuration:languageHint:)` + - convenience constructors for synchronous deterministic `hashingDefault()`, throwing `naturalLanguageDefault()`, persistent `persistentHashingDefault(configuration:dimension:)`, and persistent `persistentNaturalLanguageDefault(configuration:languageHint:)` - semantic index persistence now exists as a `RAGKit`-owned derived store through `CoreDataVectorIndex`, keeping semantic chunks and embeddings behind the existing `VectorIndex` protocol instead of pushing vector-storage concerns into `FetchKit` - persisted semantic index health now exists as a `RAGKit` concern through document-level status and fingerprints, while retry scheduling remains reserved for the future umbrella ingestion surface - markdown chunking now uses a parser-backed internal section model built on [swift-markdown](https://github.com/swiftlang/swift-markdown) instead of the earlier line-based heading scanner diff --git a/docs/maintainers/swiftlyfetch-facade-plan.md b/docs/maintainers/swiftlyfetch-facade-plan.md index 9596c42..7ce37ef 100644 --- a/docs/maintainers/swiftlyfetch-facade-plan.md +++ b/docs/maintainers/swiftlyfetch-facade-plan.md @@ -63,7 +63,7 @@ public init( Then add a default in-memory constructor for tests and examples: ```swift -public static func `default`() async throws -> SwiftlyFetchLibrary +public static func `default`() -> SwiftlyFetchLibrary ``` On macOS, add a persistent constructor after the injected path is proven: diff --git a/docs/releases/v0.2.0.md b/docs/releases/v0.2.0.md index f09e159..49c4e29 100644 --- a/docs/releases/v0.2.0.md +++ b/docs/releases/v0.2.0.md @@ -6,15 +6,18 @@ - persisted semantic vector index state and document-level semantic health through Core Data-backed `RAGKit` storage - added semantic retry storage, retry cooldown handling, persistent facade construction, and side-by-side `searchAndRetrieve(...)` - expanded corpus-based coverage with a TinyStories-derived fixture source alongside the existing Gutenberg-derived fixture records +- made `KnowledgeBase.hashingDefault(dimension:)` and `SwiftlyFetchLibrary.default()` synchronous and non-throwing because their deterministic in-memory paths have no async or throwing setup work - hardened release resume behavior and refreshed the quick-start documentation with package dependency guidance and promo media ## Breaking Changes -- None. This is a backward-compatible minor release on top of `v0.1.2`. +- `KnowledgeBase.hashingDefault(dimension:)` is now synchronous and non-throwing. Replace `try await KnowledgeBase.hashingDefault(...)` with `KnowledgeBase.hashingDefault(...)`. +- `SwiftlyFetchLibrary.default()` is now synchronous and non-throwing. Replace `try await SwiftlyFetchLibrary.default()` with `SwiftlyFetchLibrary.default()`. ## Migration Or Upgrade Notes - Existing `RAGCore`, `RAGKit`, `FetchCore`, and `FetchKit` callers can keep using those products directly. +- Remove `try await` from `KnowledgeBase.hashingDefault(...)` and `SwiftlyFetchLibrary.default()` call sites. Keep `try await` for `naturalLanguageDefault(...)`, `persistentHashingDefault(...)`, `persistentNaturalLanguageDefault(...)`, and persistent `SwiftlyFetchLibrary` constructors. - New callers that want coordinated corpus writes can import `SwiftlyFetch` and use `SwiftlyFetchLibrary`. - `SwiftlyFetchLibrary.searchAndRetrieve(...)` returns conventional and semantic results side by side; ranked hybrid search remains intentionally reserved for a later score-policy API. - The default umbrella facade still uses deterministic hashing embeddings so tests, previews, and examples do not depend on downloaded Apple embedding assets.