Skip to content

Add local-first Smart Notes dictation #583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wooboo wants to merge 1 commit into from
Closed

Add local-first Smart Notes dictation #583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Fluid.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
7C3697892ED70F9C005874CE /* DynamicNotchKit in Frameworks */ = {isa = PBXBuildFile; productRef = 7C3697882ED70F9C005874CE /* DynamicNotchKit */; };
7C5AF14B2F15041600DE21B0 /* MediaRemoteAdapter in Frameworks */ = {isa = PBXBuildFile; productRef = 7C5AF14A2F15041600DE21B0 /* MediaRemoteAdapter */; };
7C91B0012F42AA0100C0DEF0 /* HotkeyShortcutTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */; };
A1B2C3D42F70000100ABCDEF /* SmartNotesStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D52F70000100ABCDEF /* SmartNotesStoreTests.swift */; };
7CDB0A2D2F3C4D5600FB7CAD /* DictationE2ETests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CDB0A292F3C4D5600FB7CAD /* DictationE2ETests.swift */; };
7CDB0A2E2F3C4D5600FB7CAD /* AudioFixtureLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */; };
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */; };
Expand All @@ -35,6 +36,7 @@
7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; };
7CDB0A202F3C4D5600FB7CAD /* FluidDictationIntegrationTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = FluidDictationIntegrationTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = "<group>"; };
A1B2C3D52F70000100ABCDEF /* SmartNotesStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SmartNotesStoreTests.swift; sourceTree = "<group>"; };
7CDB0A292F3C4D5600FB7CAD /* DictationE2ETests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DictationE2ETests.swift; sourceTree = "<group>"; };
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LLMClientRequestBodyTests.swift; sourceTree = "<group>"; };
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -108,6 +110,7 @@
7CDB0A272F3C4D5600FB7CAD /* Resources */,
7CDB0A292F3C4D5600FB7CAD /* DictationE2ETests.swift */,
7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */,
A1B2C3D52F70000100ABCDEF /* SmartNotesStoreTests.swift */,
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */,
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */,
);
Expand Down Expand Up @@ -264,6 +267,7 @@
7CDB0A2E2F3C4D5600FB7CAD /* AudioFixtureLoader.swift in Sources */,
7CDB0A2D2F3C4D5600FB7CAD /* DictationE2ETests.swift in Sources */,
7C91B0012F42AA0100C0DEF0 /* HotkeyShortcutTests.swift in Sources */,
A1B2C3D42F70000100ABCDEF /* SmartNotesStoreTests.swift in Sources */,
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */,
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */,
);
Expand Down
133 changes: 128 additions & 5 deletions Sources/Fluid/ContentView.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ enum SidebarItem: Hashable {
case customDictionary
case stats
case history
case smartNotes
case changelog
case feedback
case commandMode
Expand All @@ -79,6 +80,7 @@ enum ShortcutRecordingTarget: Hashable {
case edit
case cancel
case pasteLast
case smartNotes
case dictationPrompt(String)
case newPrompt

Expand All @@ -96,6 +98,8 @@ enum ShortcutRecordingTarget: Hashable {
return "Cancel Recording"
case .pasteLast:
return "Paste Last Transcription"
case .smartNotes:
return "Smart Notes"
case .dictationPrompt:
return "Prompt Shortcut"
case .newPrompt:
Expand All @@ -105,7 +109,7 @@ enum ShortcutRecordingTarget: Hashable {

var enablesFeatureOnAssignment: Bool {
switch self {
case .secondaryDictation, .command, .edit, .pasteLast:
case .secondaryDictation, .command, .edit, .pasteLast, .smartNotes:
return true
case .primaryDictation, .cancel, .dictationPrompt, .newPrompt:
return false
Expand All @@ -121,7 +125,7 @@ enum ShortcutRecordingTarget: Hashable {
switch self {
case .primaryDictation, .pasteLast:
return true
case .secondaryDictation, .command, .edit, .cancel, .dictationPrompt, .newPrompt:
case .secondaryDictation, .command, .edit, .cancel, .smartNotes, .dictationPrompt, .newPrompt:
return false
}
}
Expand Down Expand Up @@ -155,6 +159,7 @@ struct ContentView: View {
case promptMode
case edit
case command
case smartNotes
}

private enum DictationOutputRoute: String {
Expand Down Expand Up @@ -192,7 +197,9 @@ struct ContentView: View {
@State private var rewriteModeHotkeyShortcut: HotkeyShortcut = SettingsStore.shared.rewriteModeHotkeyShortcut
@State private var cancelRecordingHotkeyShortcut: HotkeyShortcut = SettingsStore.shared.cancelRecordingHotkeyShortcut
@State private var pasteLastTranscriptionHotkeyShortcut: HotkeyShortcut? = SettingsStore.shared.pasteLastTranscriptionHotkeyShortcut
@State private var smartNotesHotkeyShortcut: HotkeyShortcut? = SettingsStore.shared.smartNotesHotkeyShortcut
@State private var isPasteLastTranscriptionShortcutEnabled: Bool = SettingsStore.shared.pasteLastTranscriptionShortcutEnabled
@State private var isSmartNotesShortcutEnabled: Bool = SettingsStore.shared.smartNotesShortcutEnabled
@State private var isPromptModeShortcutEnabled: Bool = SettingsStore.shared.promptModeShortcutEnabled
@State private var isCommandModeShortcutEnabled: Bool = SettingsStore.shared.commandModeShortcutEnabled
@State private var isRewriteModeShortcutEnabled: Bool = SettingsStore.shared.rewriteModeShortcutEnabled
Expand Down Expand Up @@ -475,6 +482,13 @@ struct ContentView: View {
.onChange(of: self.isPasteLastTranscriptionShortcutEnabled) { newValue in
self.handlePasteLastTranscriptionShortcutEnabledChange(newValue)
}
.onChange(of: self.smartNotesHotkeyShortcut) { _, newValue in
SettingsStore.shared.smartNotesHotkeyShortcut = newValue
self.hotkeyManager?.updateSmartNotesShortcut(newValue)
}
.onChange(of: self.isSmartNotesShortcutEnabled) { newValue in
self.handleSmartNotesShortcutEnabledChange(newValue)
}
}

private func handlePasteLastTranscriptionShortcutEnabledChange(_ isEnabled: Bool) {
Expand All @@ -484,6 +498,21 @@ struct ContentView: View {
}
}

private func handleSmartNotesShortcutEnabledChange(_ isEnabled: Bool) {
SettingsStore.shared.smartNotesShortcutEnabled = isEnabled
self.hotkeyManager?.updateSmartNotesShortcutEnabled(isEnabled)
if !isEnabled, self.activeShortcutRecordingTarget == .smartNotes {
self.clearShortcutRecordingMode()
}
if !isEnabled, self.activeRecordingMode == .smartNotes {
if self.asr.isRunning {
Task { await self.asr.stopWithoutTranscription() }
}
self.clearActiveRecordingMode()
self.menuBarManager.setOverlayMode(.dictation)
}
}

private func handlePromptShortcutEnabledChange(_ isEnabled: Bool) {
SettingsStore.shared.promptModeShortcutEnabled = isEnabled
self.hotkeyManager?.updatePromptModeShortcutEnabled(isEnabled)
Expand Down Expand Up @@ -1028,6 +1057,7 @@ struct ContentView: View {
let optionalConfiguredShortcuts: [(ShortcutRecordingTarget, HotkeyShortcut?)] = [
(.command, self.commandModeHotkeyShortcut),
(.pasteLast, self.pasteLastTranscriptionHotkeyShortcut),
(.smartNotes, self.smartNotesHotkeyShortcut),
]

for (otherTarget, configuredShortcut) in configuredShortcuts where otherTarget != target {
Expand Down Expand Up @@ -1097,6 +1127,10 @@ struct ContentView: View {
// The hotkey manager reads this shortcut directly from SettingsStore, so no manager update is needed.
self.pasteLastTranscriptionHotkeyShortcut = shortcut
SettingsStore.shared.pasteLastTranscriptionHotkeyShortcut = shortcut
case .smartNotes:
self.smartNotesHotkeyShortcut = shortcut
SettingsStore.shared.smartNotesHotkeyShortcut = shortcut
self.hotkeyManager?.updateSmartNotesShortcut(shortcut)
Comment on lines +1130 to +1133

@greptile-apps greptile-apps Bot Jul 11, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Double manager update on shortcut assignment

applyShortcut(.smartNotes) both writes to SettingsStore and calls hotkeyManager?.updateSmartNotesShortcut(shortcut) directly. The onChange(of: smartNotesHotkeyShortcut) handler (line 485-488) then fires and calls the manager a second time. The same pattern applies to setShortcutTargetEnabled(.smartNotes) (line 1181-1184) paired with its onChange → handleSmartNotesShortcutEnabledChange, which also calls updateSmartNotesShortcutEnabled twice.

The existing .pasteLast case avoids this: applyShortcut(.pasteLast) only sets state and persists to SettingsStore; the manager reads the shortcut live from SettingsStore instead of holding its own copy. Either adopt the same read-through approach for Smart Notes, or remove the direct manager calls from applyShortcut/setShortcutTargetEnabled and let onChange remain the single update path.

Prompt To Fix With AI 
This is a comment left during a code review.
Path: Sources/Fluid/ContentView.swift
Line: 1130-1133

Comment:
**Double manager update on shortcut assignment**

`applyShortcut(.smartNotes)` both writes to `SettingsStore` and calls `hotkeyManager?.updateSmartNotesShortcut(shortcut)` directly. The `onChange(of: smartNotesHotkeyShortcut)` handler (line 485-488) then fires and calls the manager a second time. The same pattern applies to `setShortcutTargetEnabled(.smartNotes)` (line 1181-1184) paired with its `onChange` → `handleSmartNotesShortcutEnabledChange`, which also calls `updateSmartNotesShortcutEnabled` twice.

The existing `.pasteLast` case avoids this: `applyShortcut(.pasteLast)` only sets state and persists to `SettingsStore`; the manager reads the shortcut live from `SettingsStore` instead of holding its own copy. Either adopt the same read-through approach for Smart Notes, or remove the direct manager calls from `applyShortcut`/`setShortcutTargetEnabled` and let `onChange` remain the single update path.

How can I resolve this? If you propose a fix, please make it concise.

Fix in Codex

case let .dictationPrompt(key):
guard let selection = SettingsStore.shared.dictationPromptSelection(forConfigurationKey: key) else { return }
var configuration = SettingsStore.shared.dictationPromptConfiguration(for: selection)
Expand Down Expand Up @@ -1144,6 +1178,10 @@ struct ContentView: View {
case .pasteLast:
self.isPasteLastTranscriptionShortcutEnabled = enabled
SettingsStore.shared.pasteLastTranscriptionShortcutEnabled = enabled
case .smartNotes:
self.isSmartNotesShortcutEnabled = enabled
SettingsStore.shared.smartNotesShortcutEnabled = enabled
self.hotkeyManager?.updateSmartNotesShortcutEnabled(enabled)
case .primaryDictation, .cancel, .dictationPrompt, .newPrompt:
break
}
Expand Down Expand Up @@ -1181,6 +1219,7 @@ struct ContentView: View {

Section {
self.sidebarNavigationLink(.commandMode, title: "Command Mode", systemImage: "terminal.fill")
self.sidebarNavigationLink(.smartNotes, title: "Smart Notes", systemImage: "note.text")
self.sidebarNavigationLink(.meetingTools, title: "File Transcription", systemImage: "doc.text.fill")
} header: {
self.sidebarSectionHeader("Use")
Expand Down Expand Up @@ -1295,6 +1334,8 @@ struct ContentView: View {
return AnyView(self.rewriteModeView)
case .history:
return AnyView(TranscriptionHistoryView())
case .smartNotes:
return AnyView(SmartNotesView())
}
}

Expand Down Expand Up @@ -1475,9 +1516,11 @@ struct ContentView: View {
rewriteShortcut: self.$rewriteModeHotkeyShortcut,
cancelRecordingShortcut: self.$cancelRecordingHotkeyShortcut,
pasteLastTranscriptionShortcut: self.$pasteLastTranscriptionHotkeyShortcut,
smartNotesShortcut: self.$smartNotesHotkeyShortcut,
commandModeShortcutEnabled: self.$isCommandModeShortcutEnabled,
rewriteShortcutEnabled: self.$isRewriteModeShortcutEnabled,
pasteLastTranscriptionShortcutEnabled: self.$isPasteLastTranscriptionShortcutEnabled,
smartNotesShortcutEnabled: self.$isSmartNotesShortcutEnabled,
hotkeyManagerInitialized: self.$hotkeyManagerInitialized,
hotkeyMode: self.$hotkeyMode,
enableStreamingPreview: self.$enableStreamingPreview,
Expand Down Expand Up @@ -2064,10 +2107,11 @@ struct ContentView: View {
DebugLogger.shared.info("Output route selected: \(route.rawValue)", source: "ContentView")
self.appBench("stop_path_enter route=\(route.rawValue)")

// Check if we're in rewrite or command mode
// Check if we're in a routed recording mode.
let modeAtStop = self.activeRecordingMode
let wasRewriteMode = modeAtStop == .edit || self.isRecordingForRewrite
let wasCommandMode = modeAtStop == .command || self.isRecordingForCommand
let wasSmartNotesMode = modeAtStop == .smartNotes
let activeDictationSlot = self.currentDictationShortcutSlot(for: modeAtStop)
let promptOverride = self.promptModeOverrideText
let promptTest = DictationPromptTestCoordinator.shared
Expand All @@ -2077,6 +2121,7 @@ struct ContentView: View {
let shouldHideOverlayOnStop = route == .normal &&
!wasRewriteMode &&
!wasCommandMode &&
!wasSmartNotesMode &&
!promptTest.isActive &&
!shouldUseAIOnStop
var didRequestOverlayHideOnStop = false
Expand Down Expand Up @@ -2133,6 +2178,13 @@ struct ContentView: View {
return
}

// Smart Notes is a dedicated output route and must never be redirected into
// prompt testing, typing, clipboard, or transcription history.
if wasSmartNotesMode {
await self.captureSmartNote(from: transcribedText)
return
}

// Prompt Test Mode: reroute dictation hotkey output into the prompt editor (no typing/clipboard/history).
if promptTest.isActive {
promptTest.lastTranscriptionText = transcribedText
Expand Down Expand Up @@ -2955,13 +3007,62 @@ struct ContentView: View {
}
}

private func captureSmartNote(from transcript: String) async {
self.menuBarManager.setProcessing(true)
NotchOverlayManager.shared.updateTranscriptionText("Saving note")

do {
let note = try SmartNotesStore.shared.capture(rawText: transcript)

if SettingsStore.shared.smartNotesAIEnhancementEnabled {
NotchOverlayManager.shared.updateTranscriptionText("Organizing note")
do {
let response = try await self.processTextWithAI(
transcript,
overrideSystemPrompt: Self.smartNotesEnhancementPrompt
)
let enhancement = try SmartNoteEnhancement.parseAIResponse(response)
try SmartNotesStore.shared.apply(enhancement, to: note.id)
} catch {
DebugLogger.shared.error(
"Smart Notes AI enrichment failed; raw note preserved: \(error.localizedDescription)",
source: "ContentView"
)
NotificationService.showSmartNotesFallback(error: error.localizedDescription)
}
}

NotchOverlayManager.shared.updateTranscriptionText("Note saved")
try? await Task.sleep(nanoseconds: 450_000_000)
} catch {
DebugLogger.shared.error(
"Smart Note save failed: \(error.localizedDescription)",
source: "ContentView"
)
NotchOverlayManager.shared.updateTranscriptionText("Note could not be saved")
try? await Task.sleep(nanoseconds: 900_000_000)
}

await self.menuBarManager.finishProcessingAndHideOverlay()
}

private static let smartNotesEnhancementPrompt = """
Turn the voice transcript into a useful structured note without inventing facts or adding advice.
Remove fillers and false starts, preserve the speaker's meaning, and use concise Markdown in the body.
Choose one short category and up to eight lowercase tags.

Return only valid JSON with exactly this shape:
{"title":"Short descriptive title","category":"Category","tags":["tag"],"body":"Markdown note body"}
Do not wrap the JSON in a code fence.
"""

private func setActiveRecordingMode(_ mode: ActiveRecordingMode) {
if mode != .dictate, mode != .promptMode {
self.clearActiveDictationShortcutState()
}
self.activeRecordingMode = mode
switch mode {
case .none, .dictate, .promptMode:
case .none, .dictate, .promptMode, .smartNotes:
self.isRecordingForCommand = false
self.isRecordingForRewrite = false
case .edit:
Expand Down Expand Up @@ -3282,10 +3383,12 @@ struct ContentView: View {
promptModeShortcut: self.promptModeHotkeyShortcut,
commandModeShortcut: self.commandModeHotkeyShortcut,
rewriteModeShortcut: self.rewriteModeHotkeyShortcut,
smartNotesShortcut: self.smartNotesHotkeyShortcut,
promptShortcutAssignments: SettingsStore.shared.dictationPromptShortcutAssignments(),
promptModeShortcutEnabled: self.isPromptModeShortcutEnabled,
commandModeShortcutEnabled: self.isCommandModeShortcutEnabled,
rewriteModeShortcutEnabled: self.isRewriteModeShortcutEnabled,
smartNotesShortcutEnabled: self.isSmartNotesShortcutEnabled,
startRecordingCallback: {
DebugLogger.shared.debug("ContentView: startRecordingCallback invoked by hotkey", source: "ContentView")
self.startRecording()
Expand Down Expand Up @@ -3372,6 +3475,23 @@ struct ContentView: View {
await self.asr.start()
}
},
smartNotesCallback: {
self.captureRecordingContext()
self.setActiveRecordingMode(.smartNotes)
self.menuBarManager.setOverlayMode(.dictation)

guard !self.asr.isRunning else { return }
self.advanceOverlayLifecycle()
TranscriptionSoundPlayer.shared.playStartSound()
Task {
await self.asr.start(onCaptureStarted: {
self.menuBarManager.showRecordingOverlayImmediately()
})
if !self.asr.isRunning {
self.menuBarManager.hideRecordingOverlayImmediately(reason: "smart_notes_start_failed")
}
}
},
isDictateRecordingProvider: {
self.activeRecordingMode == .dictate
},
Expand All @@ -3384,6 +3504,9 @@ struct ContentView: View {
isRewriteRecordingProvider: {
self.activeRecordingMode == .edit
},
isSmartNotesRecordingProvider: {
self.activeRecordingMode == .smartNotes
},
isShortcutCaptureActiveProvider: {
self.isRecordingAnyShortcutCapture
}
Expand Down Expand Up @@ -3616,7 +3739,7 @@ extension ContentView {
return self.activeDictationShortcutSlot ?? .primary
case .promptMode:
return self.activeDictationShortcutSlot ?? .secondary
case .none, .edit, .command:
case .none, .edit, .command, .smartNotes:
return nil
}
}
Expand Down
Loading
Loading