Skip to content

feat: add Retype Mode for remote-desktop/VDI dictation targets #562

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gabornyergesX wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add Retype Mode for remote-desktop/VDI dictation targets #562

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions Fluid.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
7CDB0A2E2F3C4D5600FB7CAD /* AudioFixtureLoader.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */; };
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */; };
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; };
A151A2638402FB94A8B623B0 /* RetypeModeChunkingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B006C71F68032201FC764C29 /* RetypeModeChunkingTests.swift */; };
4B6FB68A15BCDAE48784EFE4 /* RetypeModeSettingsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A1D1884585C2592C6BEA589 /* RetypeModeSettingsTests.swift */; };
4736A7DABDC6DC91AC3ED756 /* RetypeModeFocusSettleTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = FB05EADEF59C2A8730DBB46D /* RetypeModeFocusSettleTests.swift */; };
7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; };
7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; };
7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; };
Expand All @@ -38,6 +41,9 @@
7CDB0A292F3C4D5600FB7CAD /* DictationE2ETests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DictationE2ETests.swift; sourceTree = "<group>"; };
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LLMClientRequestBodyTests.swift; sourceTree = "<group>"; };
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = "<group>"; };
B006C71F68032201FC764C29 /* RetypeModeChunkingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RetypeModeChunkingTests.swift; sourceTree = "<group>"; };
9A1D1884585C2592C6BEA589 /* RetypeModeSettingsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RetypeModeSettingsTests.swift; sourceTree = "<group>"; };
FB05EADEF59C2A8730DBB46D /* RetypeModeFocusSettleTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RetypeModeFocusSettleTests.swift; sourceTree = "<group>"; };
7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFixtureLoader.swift; sourceTree = "<group>"; };
7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = dictation_fixture.wav; sourceTree = "<group>"; };
7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; };
Expand Down Expand Up @@ -110,6 +116,9 @@
7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */,
343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */,
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */,
B006C71F68032201FC764C29 /* RetypeModeChunkingTests.swift */,
9A1D1884585C2592C6BEA589 /* RetypeModeSettingsTests.swift */,
FB05EADEF59C2A8730DBB46D /* RetypeModeFocusSettleTests.swift */,
);
path = FluidDictationIntegrationTests;
sourceTree = "<group>";
Expand Down Expand Up @@ -266,6 +275,9 @@
7C91B0012F42AA0100C0DEF0 /* HotkeyShortcutTests.swift in Sources */,
86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */,
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */,
A151A2638402FB94A8B623B0 /* RetypeModeChunkingTests.swift in Sources */,
4B6FB68A15BCDAE48784EFE4 /* RetypeModeSettingsTests.swift in Sources */,
4736A7DABDC6DC91AC3ED756 /* RetypeModeFocusSettleTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
66 changes: 66 additions & 0 deletions Sources/Fluid/Persistence/SettingsStore.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ final class SettingsStore: ObservableObject {
static let privateAIContextTokenLimitRange: ClosedRange<Int> = 2048...8192
static let privateAIContextTokenLimitStep = 512
static let defaultPrivateAIContextTokenLimit = 4096
static let slowTypeWarmupMsRange: ClosedRange<Double> = 0...2000
static let slowTypeRampCharCountRange: ClosedRange<Int> = 0...30
static let slowTypeRampDelayMsRange: ClosedRange<Double> = 0...200
static let slowTypeSteadyDelayMsRange: ClosedRange<Double> = 0...100
static let privateAIDictationSystemOverheadTokens = 1280
static let privateAIDictationMinimumOutputTokens = 256
static let privateAIDictationRoundTripTokenCost = 2.75
Expand Down Expand Up @@ -4477,6 +4481,10 @@ private extension SettingsStore {
static let experimentalDirectAudioCaptureEnabled = "ExperimentalDirectAudioCaptureEnabled"
static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard"
static let textInsertionMode = "TextInsertionMode"
static let slowTypeWarmupMs = "SlowTypeWarmupMs"
static let slowTypeRampCharCount = "SlowTypeRampCharCount"
static let slowTypeRampDelayMs = "SlowTypeRampDelayMs"
static let slowTypeSteadyDelayMs = "SlowTypeSteadyDelayMs"
static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled"
static let betaReleasesEnabled = "BetaReleasesEnabled"
static let lastUpdateCheckDate = "LastUpdateCheckDate"
Expand Down Expand Up @@ -4601,6 +4609,7 @@ extension SettingsStore {
enum TextInsertionMode: String, CaseIterable, Identifiable, Codable {
case standard
case reliablePaste
case slowType

var id: String {
self.rawValue
Expand All @@ -4612,6 +4621,8 @@ extension SettingsStore {
return "Clipboard Free Insert"
case .reliablePaste:
return "Clipboard Paste"
case .slowType:
return "Retype Mode"
}
}

Expand All @@ -4621,6 +4632,8 @@ extension SettingsStore {
return "Fastest path. Inserts text without changing the clipboard, with paste fallback if direct insertion is unavailable."
case .reliablePaste:
return "Compatibility path. Uses a temporary clipboard paste, so clipboard history apps may briefly record dictated text."
case .slowType:
return "Types one character at a time with a short warm-up and per-character delay. Slower, but more reliable with remote-desktop, VDI, or virtualization apps that garble fast text insertion or block the clipboard."
}
}
}
Expand All @@ -4640,6 +4653,59 @@ extension SettingsStore {
}
}

/// Warm-up delay (ms) before Retype Mode sends its first keystroke, letting a
/// remote session finish syncing. Only used in `.slowType` mode.
var slowTypeWarmupMs: Double {
get {
let stored = self.defaults.object(forKey: Keys.slowTypeWarmupMs) as? NSNumber
return stored?.doubleValue ?? 350.0
}
set {
objectWillChange.send()
let clamped = min(max(newValue, Self.slowTypeWarmupMsRange.lowerBound), Self.slowTypeWarmupMsRange.upperBound)
self.defaults.set(clamped, forKey: Keys.slowTypeWarmupMs)
}
}

/// Number of leading characters typed at the slower ramp pace in Retype Mode.
var slowTypeRampCharCount: Int {
get {
let stored = self.defaults.object(forKey: Keys.slowTypeRampCharCount) as? NSNumber
return stored?.intValue ?? 6
}
set {
objectWillChange.send()
let clamped = min(max(newValue, Self.slowTypeRampCharCountRange.lowerBound), Self.slowTypeRampCharCountRange.upperBound)
self.defaults.set(clamped, forKey: Keys.slowTypeRampCharCount)
}
}

/// Per-character delay (ms) during Retype Mode's ramp phase.
var slowTypeRampDelayMs: Double {
get {
let stored = self.defaults.object(forKey: Keys.slowTypeRampDelayMs) as? NSNumber
return stored?.doubleValue ?? 45.0
}
set {
objectWillChange.send()
let clamped = min(max(newValue, Self.slowTypeRampDelayMsRange.lowerBound), Self.slowTypeRampDelayMsRange.upperBound)
self.defaults.set(clamped, forKey: Keys.slowTypeRampDelayMs)
}
}

/// Per-character delay (ms) after Retype Mode's ramp phase.
var slowTypeSteadyDelayMs: Double {
get {
let stored = self.defaults.object(forKey: Keys.slowTypeSteadyDelayMs) as? NSNumber
return stored?.doubleValue ?? 18.0
}
set {
objectWillChange.send()
let clamped = min(max(newValue, Self.slowTypeSteadyDelayMsRange.lowerBound), Self.slowTypeSteadyDelayMsRange.upperBound)
self.defaults.set(clamped, forKey: Keys.slowTypeSteadyDelayMs)
}
}

var betaReleasesEnabled: Bool {
get {
let value = self.defaults.object(forKey: Keys.betaReleasesEnabled)
Expand Down
119 changes: 108 additions & 11 deletions Sources/Fluid/Services/TypingService.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,22 @@ final class TypingService {
self.typeOutputPlanInstantly(.plain(text), preferredTargetPID: preferredTargetPID, textReadyAt: textReadyAt)
}

/// How long to wait, before attempting insertion, for macOS to finish restoring focus
/// to the real target app. Needed whenever `preferredTargetPID` is nil, since the PID
/// is then resolved from whatever is *currently* focused — if that resolution runs too
/// soon (e.g. right after we hide our own window), it can capture a stale app instead.
/// Retype Mode's own warm-up delay (see `insertTextSlowly`) happens *after* PID
/// resolution, so it does not substitute for this pre-capture settle.
/// Internal (not `private`) so regression tests can exercise it directly — see
/// `RetypeModeChunkingTests`.
static func settleDelayMs(mode: SettingsStore.TextInsertionMode, preferredTargetPID: pid_t?) -> Int {
guard preferredTargetPID == nil else { return 0 }
switch mode {
case .reliablePaste: return 80
case .slowType, .standard: return 200
}
}

func typeOutputPlanInstantly(
_ plan: DictationLiteralOutputPlan,
preferredTargetPID: pid_t?,
Expand All @@ -267,12 +283,7 @@ final class TypingService {
let requestedAt = ProcessInfo.processInfo.systemUptime
let text = plan.plainText
let mode = self.textInsertionMode
let settleDelayMs: Int = {
if mode == .reliablePaste {
return preferredTargetPID == nil ? 80 : 0
}
return preferredTargetPID == nil ? 200 : 0
}()
let settleDelayMs = Self.settleDelayMs(mode: mode, preferredTargetPID: preferredTargetPID)
let textReadyAge = textReadyAt.map { Self.elapsedMs(from: $0, to: requestedAt) }
self.bench(
"request chars=\(text.count) mode=\(mode.rawValue) autocompleteSteps=\(plan.steps.count) preferredPID=\(preferredTargetPID.map { String($0) } ?? "nil") textReadyAgeMs=\(textReadyAge.map { String($0) } ?? "nil")"
Expand Down Expand Up @@ -350,6 +361,15 @@ final class TypingService {
self.log("[TypingService] insertTextInstantly called with \(text.count) characters")
self.log("[TypingService] Attempting to type text: \"\(text.prefix(50))\(text.count > 50 ? "..." : "")\"")

if self.textInsertionMode == .slowType {
self.log("[TypingService] Slow/compatibility typing mode enabled")
if self.insertTextSlowly(text, preferredTargetPID: preferredTargetPID) {
self.log("[TypingService] SUCCESS: Slow typing completed")
return
}
self.log("[TypingService] Slow typing fell through to standard pipeline")
}

if self.textInsertionMode == .reliablePaste {
self.log("[TypingService] Reliable Paste mode enabled")
if self.tryReliablePasteInsertion(text, preferredTargetPID: preferredTargetPID) {
Expand Down Expand Up @@ -684,9 +704,61 @@ final class TypingService {
}
}

// MARK: - Retype Mode (remote-desktop compatibility)

// Remote-desktop keyboard channels need a moment
// to sync session/layout state; sending the whole string instantly garbles the
// leading characters. Warm-up delay and per-character pacing are user-tunable in
// Settings (see `SettingsStore.slowTypeWarmupMs` and friends) since the right values
// depend on the remote session's own latency.

/// Types text one character at a time with a warm-up delay and a slow-then-normal
/// ramp, for apps that garble fast unicode insertion (remote-desktop/VDI targets).
private func insertTextSlowly(_ text: String, preferredTargetPID: pid_t?) -> Bool {
let targetPID = preferredTargetPID.flatMap { $0 > 0 ? $0 : nil }
?? self.getSystemFocusedElementAndPID()?.pid

guard let targetPID, targetPID > 0 else {
self.log("[TypingService] Slow typing: no target PID available, deferring to fallback pipeline")
return false
}

let settings = SettingsStore.shared
if NSWorkspace.shared.frontmostApplication?.processIdentifier != targetPID {
_ = Self.activateApp(pid: targetPID)
usleep(80_000)
}
usleep(useconds_t(settings.slowTypeWarmupMs * 1000))

let utf16Array = Array(text.utf16)
self.log("[TypingService] Starting slow typing (\(utf16Array.count) UTF16 units) to PID \(targetPID)")

let rampEnd = Self.rampEnd(for: utf16Array, rampCharCount: settings.slowTypeRampCharCount)
let rampSucceeded = self.postUnicodeChunks(
Array(utf16Array[0..<rampEnd]),
destinationDescription: "PID \(targetPID) (ramp)",
chunkSize: 1,

@chatgpt-codex-connector chatgpt-codex-connector Bot Jul 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Keep surrogate pairs together in Retype Mode

When Retype Mode handles text containing non-BMP characters such as emoji or some CJK extensions, chunkSize: 1 makes unicodeChunkEnd split the UTF-16 surrogate pair: it backs off from the high surrogate boundary, but then max(end, start + 1) still returns a one-unit chunk. keyboardSetUnicodeString then receives invalid surrogate halves in separate CGEvents, so those characters can be dropped or garbled even though the existing bulk path preserved them. Use scalar-aware chunking or allow the pair to be sent as one chunk.

Useful? React with 👍 / 👎.

@gabornyergesX gabornyergesX Jul 9, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in c86b85a — unicodeChunkEnd now widens forward instead of backing off when the backoff would produce an empty chunk (i.e. at chunkSize: 1), so the surrogate pair is kept together as a single 2-unit chunk instead of splitting into two lone-surrogate CGEvents. Verified with a standalone simulation against "a🎉b": chunks are now ["a"], ["🎉" as one 2-unit chunk], ["b"] at chunkSize 1, and the bulk chunkSize-200 path is unchanged.

interChunkDelayMicros: useconds_t(settings.slowTypeRampDelayMs * 1000)
) { $0.postToPid(targetPID) }

var steadySucceeded = true
if rampEnd < utf16Array.count {
steadySucceeded = self.postUnicodeChunks(
Array(utf16Array[rampEnd...]),
destinationDescription: "PID \(targetPID) (steady)",
chunkSize: 1,
interChunkDelayMicros: useconds_t(settings.slowTypeSteadyDelayMs * 1000)
) { $0.postToPid(targetPID) }
}

return rampSucceeded && steadySucceeded
}

private func postUnicodeChunks(
_ utf16Array: [UInt16],
destinationDescription: String,
chunkSize: Int = TypingService.cgEventUnicodeChunkSize,
interChunkDelayMicros: useconds_t = 0,
post: (CGEvent) -> Void
) -> Bool {
guard utf16Array.isEmpty == false else { return true }
Expand All @@ -697,7 +769,7 @@ final class TypingService {
var chunkStart = 0
var chunkCount = 0
while chunkStart < buffer.count {
let chunkEnd = Self.unicodeChunkEnd(in: utf16Array, start: chunkStart)
let chunkEnd = Self.unicodeChunkEnd(in: utf16Array, start: chunkStart, chunkSize: chunkSize)
let chunkLength = chunkEnd - chunkStart

guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
Expand All @@ -713,6 +785,9 @@ final class TypingService {

post(keyDown)
post(keyUp)
if interChunkDelayMicros > 0 {
usleep(interChunkDelayMicros)
}

chunkStart = chunkEnd
chunkCount += 1
Expand All @@ -722,18 +797,40 @@ final class TypingService {

guard chunkCount >= 0 else { return false }

self.log("[TypingService] Posted \(chunkCount) unicode CGEvent chunk(s) to \(destinationDescription) with chunkSize=\(Self.cgEventUnicodeChunkSize) interChunkDelayMs=0")
self.log("[TypingService] Posted \(chunkCount) unicode CGEvent chunk(s) to \(destinationDescription) with chunkSize=\(chunkSize) interChunkDelayMs=\(Double(interChunkDelayMicros) / 1000)")
return true
}

private static func unicodeChunkEnd(in utf16Array: [UInt16], start: Int) -> Int {
var end = min(start + Self.cgEventUnicodeChunkSize, utf16Array.count)
/// Boundary between Retype Mode's ramp and steady phases. `unicodeChunkEnd` always
/// advances by at least one unit (it's built for chunking loops that must make forward
/// progress), so a ramp length of 0 would still floor to a 1-unit ramp slice instead of
/// skipping the ramp phase entirely — and if that lone unit were a high surrogate, it'd
/// split the pair at this boundary before `postUnicodeChunks` ever sees it. A ramp
/// length of 0 means "no ramp phase," so skip straight to 0 rather than chunking.
/// Internal (not `private`) so regression tests can exercise it directly — see
/// `RetypeModeChunkingTests`.
static func rampEnd(for utf16Array: [UInt16], rampCharCount: Int) -> Int {
rampCharCount > 0 ? Self.unicodeChunkEnd(in: utf16Array, start: 0, chunkSize: rampCharCount) : 0
}

/// Internal (not `private`) so regression tests can exercise the surrogate-pair
/// chunking logic directly — see `RetypeModeChunkingTests`.
static func unicodeChunkEnd(in utf16Array: [UInt16], start: Int, chunkSize: Int = TypingService.cgEventUnicodeChunkSize) -> Int {
var end = min(start + chunkSize, utf16Array.count)
if end < utf16Array.count,
end > start,
Self.isHighSurrogate(utf16Array[end - 1]),
Self.isLowSurrogate(utf16Array[end])
{
end -= 1
// Backing off keeps the pair intact for the *next* chunk, but only if that
// leaves this chunk non-empty. At chunkSize 1 (Retype Mode) the pair's high
// surrogate IS the whole chunk, so back off would produce an empty chunk;
// widen forward instead to keep both surrogate halves together here.
if end - 1 > start {
end -= 1
} else {
end += 1
}
}
return max(end, start + 1)
}
Expand Down
Loading
Loading