diff --git a/AGENTS.md b/AGENTS.md index 870da3a..8ff672d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,7 +137,7 @@ If the output says Windows cannot safely access SQLite through a WSL UNC path: If sync reports `Skipped locked rollout files`: - treat the sync as mostly successful -- explain that the active session still holds one or more rollout files open +- explain that an active session either still holds one or more rollout files open, or appended to one while it was being scanned - tell the user to rerun `codex-provider sync` after that session ends if they want a full rewrite If `switch ` fails because the provider is missing: diff --git a/desktop/CodexProviderSync.App.Tests/MainFormPresentationTests.cs b/desktop/CodexProviderSync.App.Tests/MainFormPresentationTests.cs index c86c95c..2139a85 100644 --- a/desktop/CodexProviderSync.App.Tests/MainFormPresentationTests.cs +++ b/desktop/CodexProviderSync.App.Tests/MainFormPresentationTests.cs @@ -226,6 +226,70 @@ private static void PerformLayoutRecursively(Control control) } } + [Fact] + public void OpenBackupFolder_ReportsNoError_WhenShellReusesAnExistingProcess() + { + string root = Path.Combine(Path.GetTempPath(), $"codex-provider-ui-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + // Explorer commonly satisfies a folder request through a window it + // already owns and then returns no Process handle. The boundary must + // treat that as a successful open, not a failure. + RecordingPlatformBoundary boundary = new(); + using MainForm form = new(new ExecutionLogService(root), platformBoundary: boundary); + SetField(form, "_currentStatus", StatusWithBackupRoot(root)); + + Invoke(form, "OpenBackupFolder"); + + Assert.Equal(root, Assert.Single(boundary.OpenedPaths)); + Assert.DoesNotContain("打开备份目录失败", Field(form, "_logBox").Text); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + // The failure branch of OpenBackupFolder is deliberately not covered here: + // its catch reports through MessageBox.Show, which has no injection seam in + // MainForm, so a test driving it would block the runner on a modal dialog. + + private static StatusSnapshot StatusWithBackupRoot(string backupRoot) => new() + { + CodexHome = @"C:\Users\user\.codex", + SqliteAccess = new SqliteAccessInfo(true, "windows", null), + CurrentProvider = new CurrentProviderInfo("openai", false), + ConfiguredProviders = ["openai"], + RolloutCounts = new ProviderCounts(), + LockedRolloutFiles = [], + UnreadableRolloutFiles = [], + EncryptedContentCounts = new ProviderCounts(), + SqliteCounts = null, + BackupRoot = backupRoot, + BackupSummary = new BackupSummary { Count = 0, TotalBytes = 0 } + }; + + private sealed class RecordingPlatformBoundary : IAppPlatformBoundary + { + public List OpenedPaths { get; } = []; + + public bool UpdatesEnabled => false; + + public string? PickFolder(IWin32Window owner, FolderPickerRequest request) => null; + + public Task CheckForUpdateAsync( + UpdateService updateService, + Version currentVersion, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public void OpenPath(string path) => OpenedPaths.Add(path); + + public void StartUpdate(string downloadedExePath, string targetExePath, string expectedSha256) => + throw new NotSupportedException(); + } + private static T Field(MainForm form, string name) where T : class { return typeof(MainForm) diff --git a/desktop/CodexProviderSync.App/AppPaths.cs b/desktop/CodexProviderSync.App/AppPaths.cs index 8110bb7..120ac73 100644 --- a/desktop/CodexProviderSync.App/AppPaths.cs +++ b/desktop/CodexProviderSync.App/AppPaths.cs @@ -227,11 +227,16 @@ public Task CheckForUpdateAsync( public void OpenPath(string path) { + // With UseShellExecute the shell may satisfy the request through a + // process it already owns - Explorer reusing an open window is the + // common case - and then returns no Process handle even though the + // path opened. A null result therefore carries no failure information; + // a genuine failure such as a missing path surfaces as Win32Exception. _ = Process.Start(new ProcessStartInfo { FileName = path, UseShellExecute = true - }) ?? throw new InvalidOperationException($"Unable to open {path}."); + }); } public void StartUpdate(string downloadedExePath, string targetExePath, string expectedSha256) => diff --git a/desktop/CodexProviderSync.App/MainForm.cs b/desktop/CodexProviderSync.App/MainForm.cs index d0bdf16..5ce74ee 100644 --- a/desktop/CodexProviderSync.App/MainForm.cs +++ b/desktop/CodexProviderSync.App/MainForm.cs @@ -940,6 +940,9 @@ await RunBusyAsync("执行中...", async () => result, request is SwitchProviderRequest ? "已切换并同步" : "已同步", TextFormatter.ChineseSimplified)); + AppendLog(TextFormatter.FormatPerformanceMetrics( + result.PerformanceMetrics, + TextFormatter.ChineseSimplified)); AppendLog(FormatModelSyncOutcome(result.ModelSync)); AppendLog(string.Empty); await RefreshStatusCoreAsync(request.CodexHome, request.SqliteHomeOverride, provider); @@ -1076,8 +1079,16 @@ private void OpenBackupFolder() { string path = _currentStatus?.BackupRoot ?? AppConstants.DefaultBackupRoot(CurrentCodexHome()); EnsureAutomationPath(path, GuiAutomationCatalog.Ids.OpenBackupDirectory); - Directory.CreateDirectory(path); - _platformBoundary.OpenPath(path); + try + { + Directory.CreateDirectory(path); + _platformBoundary.OpenPath(path); + } + catch (Exception error) + { + AppendLog($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss}] 打开备份目录失败: {error}"); + MessageBox.Show(this, $"无法打开备份目录。{Environment.NewLine}{Environment.NewLine}{error.Message}", Text, MessageBoxButtons.OK, MessageBoxIcon.Error); + } } private void OpenLogFolder() @@ -1591,6 +1602,9 @@ private async Task ApplyControllerRefreshAsync(AppSnapshot snapshot) ReloadProviderList(); _codexHomeCombo.Text = _currentStatus.CodexHome; AppendLog($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss}] 已刷新: {_currentStatus.CodexHome}"); + AppendLog(TextFormatter.FormatPerformanceMetrics( + _currentStatus.PerformanceMetrics, + TextFormatter.ChineseSimplified)); } private WindowBoundsState CaptureWindowBounds() diff --git a/desktop/CodexProviderSync.Core.Tests/BackupParityTests.cs b/desktop/CodexProviderSync.Core.Tests/BackupParityTests.cs index b431a96..51cd17f 100644 --- a/desktop/CodexProviderSync.Core.Tests/BackupParityTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/BackupParityTests.cs @@ -36,6 +36,12 @@ await File.WriteAllTextAsync( Assert.False(globalStateFiles.GetProperty(AppConstants.GlobalStateBackupFileBasename).GetBoolean()); Assert.True(metadataRoot.GetProperty("globalStateFilePresent").GetBoolean()); Assert.False(metadataRoot.GetProperty("globalStateBackupFilePresent").GetBoolean()); + Assert.True(metadataRoot.GetProperty("sizeBytes").GetInt64() > 0); + Assert.True(metadataRoot.GetProperty("fileCount").GetInt32() > 0); + Assert.Equal( + Directory.EnumerateFiles(backupDir, "*", SearchOption.AllDirectories) + .Sum(static path => new FileInfo(path).Length), + metadataRoot.GetProperty("sizeBytes").GetInt64()); using JsonDocument manifest = JsonDocument.Parse( await File.ReadAllTextAsync(Path.Combine(backupDir, "session-meta-backup.json"))); @@ -48,6 +54,79 @@ await File.WriteAllTextAsync( entry.GetProperty("originalLastWriteTimeUtcTicks").GetString()); } + [Fact] + public async Task BackupSummary_UsesCachedInventoryAndFallsBackForLegacyMetadata() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + string cached = fixture.BackupPath("20260708T091011111Z"); + string legacy = fixture.BackupPath("20260708T091011110Z"); + Directory.CreateDirectory(cached); + Directory.CreateDirectory(legacy); + await File.WriteAllTextAsync( + Path.Combine(cached, "metadata.json"), + JsonSerializer.Serialize(new + { + version = 2, + @namespace = AppConstants.BackupNamespace, + codexHome = fixture.CodexHome, + targetProvider = "openai", + createdAt = DateTimeOffset.UtcNow, + dbFiles = Array.Empty(), + changedSessionFiles = 0, + sizeBytes = 123L, + fileCount = 1 + })); + await File.WriteAllTextAsync(Path.Combine(cached, "added-later.bin"), new string('x', 4096)); + await File.WriteAllTextAsync( + Path.Combine(legacy, "metadata.json"), + JsonSerializer.Serialize(new + { + version = 1, + @namespace = AppConstants.BackupNamespace, + codexHome = fixture.CodexHome, + targetProvider = "openai", + createdAt = DateTimeOffset.UtcNow, + dbFiles = Array.Empty(), + changedSessionFiles = 0 + })); + await File.WriteAllTextAsync(Path.Combine(legacy, "payload.bin"), new string('y', 17)); + + List fallbacks = []; + BackupService backups = new(new SessionRolloutService(), new SqliteStateService()) + { + DirectoryInventoryFallbackObserver = fallbacks.Add + }; + BackupSummary summary = await backups.GetBackupSummaryAsync(fixture.CodexHome); + + long legacyBytes = Directory.EnumerateFiles(legacy, "*", SearchOption.AllDirectories) + .Sum(static path => new FileInfo(path).Length); + Assert.Equal(2, summary.Count); + Assert.Equal(123L + legacyBytes, summary.TotalBytes); + Assert.Single(fallbacks); + Assert.Equal(Path.GetFullPath(legacy), Path.GetFullPath(fallbacks[0])); + } + + [Fact] + public async Task IncompleteMetadata_IsNotManagedOrPrunedEvenWithProviderSyncNamespace() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + string incomplete = fixture.BackupPath("20260708T091011109Z"); + Directory.CreateDirectory(incomplete); + string sentinel = Path.Combine(incomplete, "keep.txt"); + await File.WriteAllTextAsync( + Path.Combine(incomplete, "metadata.json"), + "{\"namespace\":\"provider-sync\",\"sizeBytes\":1,\"fileCount\":1}"); + await File.WriteAllTextAsync(sentinel, "must remain"); + + BackupService backups = new(new SessionRolloutService(), new SqliteStateService()); + BackupSummary summary = await backups.GetBackupSummaryAsync(fixture.CodexHome); + BackupPruneResult pruned = await backups.PruneBackupsAsync(fixture.CodexHome, 0); + + Assert.Equal(0, summary.Count); + Assert.Equal(0, pruned.DeletedCount); + Assert.True(File.Exists(sentinel)); + } + [Fact] public async Task RestoreBackup_AcceptsNodeStyleV2MetadataAndMillisecondSessionTimestamp() { diff --git a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs index da14a9f..8e21897 100644 --- a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Text.Json.Nodes; using System.Diagnostics; using Microsoft.Data.Sqlite; @@ -45,6 +46,57 @@ await fixture.WriteStateDbAsync([ Assert.Equal("apigather", await ReadProviderAsync(fixture.StateDbPath(), "thread-a")); Assert.Equal("apigather", await ReadProviderAsync(fixture.StateDbPath(), "thread-b")); Assert.Empty(await FileTransactionJournal.FindPendingAsync(fixture.CodexHome)); + + // The rollback appended more journal records after the inventory was + // written, so the retained backup must record its real size. Status and + // pruning read these cached values without revalidating them. + AssertBackupInventoryMatchesDisk(error.BackupDirectory); + } + + [Fact] + public async Task RunSync_RefreshesRetainedBackupInventory_WhenRollbackCompletes() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + await fixture.WriteConfigAsync("model_provider = \"openai\""); + string sessionPath = fixture.RolloutPath("sessions", "rollout-a.jsonl"); + await fixture.WriteRolloutAsync(sessionPath, "thread-a", "apigather"); + await fixture.WriteStateDbAsync([("thread-a", "apigather", false)]); + + // Fail the apply so the rollback path runs and the journal appends its + // terminal records after metadata.json was already written. + CodexSyncService service = new(); + service.FaultInjector = (point, _, _) => + { + if (point == "before_rollout_apply") + { + throw new IOException("injected apply failure"); + } + return Task.CompletedTask; + }; + + SyncTransactionException error = await Assert.ThrowsAsync( + () => service.RunSyncAsync(fixture.CodexHome, provider: "openai")); + + // Pin the path this covers: the rollback succeeded, so the journal wrote + // its rolledBack record into the retained backup after the inventory was + // captured. The recorded size and file count must still match disk. + Assert.Equal("complete", error.RollbackStatus); + Assert.False(error.RecoveryRequired); + AssertBackupInventoryMatchesDisk(error.BackupDirectory); + } + + private static void AssertBackupInventoryMatchesDisk(string backupDirectory) + { + string metadataPath = Path.Combine(backupDirectory, "metadata.json"); + using JsonDocument document = JsonDocument.Parse(File.ReadAllText(metadataPath)); + long recordedSize = document.RootElement.GetProperty("sizeBytes").GetInt64(); + int recordedCount = document.RootElement.GetProperty("fileCount").GetInt32(); + + string[] actualFiles = Directory.GetFiles(backupDirectory, "*", SearchOption.AllDirectories); + long actualSize = actualFiles.Sum(file => new FileInfo(file).Length); + + Assert.Equal(actualFiles.Length, recordedCount); + Assert.Equal(actualSize, recordedSize); } [Fact] @@ -960,9 +1012,21 @@ await fixture.WriteStateDbAsync( Assert.Empty(syncResult.SkippedLockedRolloutFiles); Assert.Empty(syncResult.SkippedUnreadableRolloutFiles); Assert.Equal(2, syncResult.SqliteRowsUpdated); + Assert.Equal(2, syncResult.PerformanceMetrics.RolloutScan.EnumeratedRolloutFiles); + Assert.Equal(2, syncResult.PerformanceMetrics.RolloutScan.ContentScanPasses); + Assert.Equal( + OperatingSystem.IsWindows() ? 2 : 8, + syncResult.PerformanceMetrics.JournalFullValidationCount); BackupMetadataFile backupMetadata = JsonSerializer.Deserialize( await File.ReadAllTextAsync(Path.Combine(syncResult.BackupDir, "metadata.json")), new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase })!; + Assert.Equal( + Directory.EnumerateFiles(syncResult.BackupDir, "*", SearchOption.AllDirectories) + .Sum(static path => new FileInfo(path).Length), + backupMetadata.SizeBytes); + Assert.Equal( + Directory.EnumerateFiles(syncResult.BackupDir, "*", SearchOption.AllDirectories).Count(), + backupMetadata.FileCount); Assert.Equal( [ Path.Combine(AppConstants.SqliteDirBasename, AppConstants.DbFileBasename) @@ -1343,6 +1407,8 @@ await fixture.WriteStateDbAsync( Assert.Equal(fixture.StateDbPath(), status.StateDbLocation.Path); Assert.Equal(2, status.BackupSummary.Count); Assert.Equal(backupOneBytes + backupTwoBytes, status.BackupSummary.TotalBytes); + Assert.Equal(2, status.PerformanceMetrics.RolloutScan.EnumeratedRolloutFiles); + Assert.Equal(2, status.PerformanceMetrics.RolloutScan.ContentScanPasses); Assert.Contains($"database: {fixture.StateDbPath()}", TextFormatter.FormatStatus(status)); } @@ -1580,6 +1646,67 @@ await File.AppendAllTextAsync( Assert.Contains("\"message\":\"later\"", rollout); } + [Fact] + public async Task CollectSessionChanges_ReportsLockedInsteadOfAborting_WhenRolloutChangesDuringScan() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + await fixture.WriteConfigAsync("model_provider = \"openai\""); + string movingPath = fixture.RolloutPath("sessions", "rollout-a.jsonl"); + string stablePath = fixture.RolloutPath("sessions", "rollout-b.jsonl"); + await fixture.WriteRolloutAsync(movingPath, "thread-a", "apigather"); + await fixture.WriteRolloutAsync(stablePath, "thread-b", "apigather"); + + // Emulate an active Codex session appending to its own rollout inside the + // window between the digest fold and the post-scan snapshot. + SessionRolloutService service = new() + { + ScanFaultInjector = async scannedPath => + { + if (string.Equals(scannedPath, movingPath, StringComparison.Ordinal)) + { + await File.AppendAllTextAsync( + scannedPath, + "{\"timestamp\":\"2026-03-19T00:00:01.000Z\",\"type\":\"event_msg\",\"payload\":{\"type\":\"assistant_message\",\"message\":\"live\"}}\n"); + } + } + }; + + SessionChangeCollection collected = await service.CollectSessionChangesAsync( + fixture.CodexHome, + "openai", + skipLockedReads: true); + + Assert.Contains(movingPath, collected.LockedPaths); + Assert.DoesNotContain(movingPath, collected.Changes.Select(static change => change.Path)); + Assert.Contains(stablePath, collected.Changes.Select(static change => change.Path)); + + // The rest of the corpus is still rewritten; only the live rollout waits. + SessionApplyResult applyResult = await service.ApplySessionChangesAsync(collected.Changes); + Assert.Equal(1, applyResult.AppliedCount); + Assert.Empty(applyResult.SkippedPaths); + Assert.Contains("\"model_provider\":\"apigather\"", await File.ReadAllTextAsync(movingPath)); + Assert.Contains("\"model_provider\":\"openai\"", await File.ReadAllTextAsync(stablePath)); + } + + [Fact] + public async Task CollectSessionChanges_Throws_WhenRolloutChangesDuringScanWithoutSkipLockedReads() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + await fixture.WriteConfigAsync("model_provider = \"openai\""); + string movingPath = fixture.RolloutPath("sessions", "rollout-a.jsonl"); + await fixture.WriteRolloutAsync(movingPath, "thread-a", "apigather"); + + SessionRolloutService service = new() + { + ScanFaultInjector = async scannedPath => await File.AppendAllTextAsync( + scannedPath, + "{\"timestamp\":\"2026-03-19T00:00:01.000Z\",\"type\":\"event_msg\",\"payload\":{\"type\":\"assistant_message\",\"message\":\"live\"}}\n") + }; + + await Assert.ThrowsAsync(() => + service.CollectSessionChangesAsync(fixture.CodexHome, "openai", skipLockedReads: false)); + } + [Fact] public async Task ApplySessionChanges_RewritesFile_WhenRolloutIsUnchanged() { @@ -1799,6 +1926,58 @@ await fixture.WriteBackupAsync( Assert.True(string.IsNullOrWhiteSpace(result.AutoPruneWarning)); } + [Fact] + public async Task RunSync_SucceedsWithWarningAndStillPrunes_WhenInventoryRefreshFailsAfterCommit() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + await fixture.WriteConfigAsync("model_provider = \"openai\""); + string sessionPath = fixture.RolloutPath("sessions", "rollout-a.jsonl"); + await fixture.WriteRolloutAsync(sessionPath, "thread-a", "apigather"); + await fixture.WriteStateDbAsync([("thread-a", "apigather", false)]); + + for (int index = 0; index < AppConstants.DefaultBackupRetentionCount; index += 1) + { + await fixture.WriteBackupAsync( + $"20240101T0000{index:00}000Z", + ("note.txt", $"backup-{index}")); + } + + // Break the inventory refresh that runs immediately after the journal + // commit. The transaction is durable by then, so the sync must report + // success with a warning rather than failing. + CodexSyncService service = new(); + service.FaultInjector = (point, _, _) => + { + if (point == "before_transaction_commit") + { + // The backup directory this transaction owns is the only one + // still carrying a journal file. Keep the namespace so it stays a + // managed backup for the prune pass, but make the version + // unreadable so only the inventory refresh fails. + string activeBackupDir = Directory + .GetDirectories(fixture.BackupRoot()) + .Single(dir => File.Exists(Path.Combine(dir, FileTransactionJournal.FileName))); + string metadataPath = Path.Combine(activeBackupDir, "metadata.json"); + JsonNode metadata = JsonNode.Parse(File.ReadAllText(metadataPath))!; + metadata["version"] = 99; + File.WriteAllText(metadataPath, metadata.ToJsonString()); + } + + return Task.CompletedTask; + }; + + SyncResult result = await service.RunSyncAsync(fixture.CodexHome); + + Assert.NotNull(result.AutoPruneResult); + Assert.Equal(1, result.AutoPruneResult!.DeletedCount); + Assert.Contains("Backup inventory refresh failed", result.AutoPruneWarning); + + // The mutation itself still landed. + Assert.Contains("\"model_provider\":\"openai\"", await File.ReadAllTextAsync(sessionPath)); + Assert.Equal("openai", await ReadProviderAsync(fixture.StateDbPath(), "thread-a")); + Assert.Empty(await FileTransactionJournal.FindPendingAsync(fixture.CodexHome)); + } + [Fact] public async Task RunSync_UsesCustomAutomaticBackupRetentionCount() { diff --git a/desktop/CodexProviderSync.Core.Tests/CoreWritePlanningTests.cs b/desktop/CodexProviderSync.Core.Tests/CoreWritePlanningTests.cs index 16c48d4..9e8a3b9 100644 --- a/desktop/CodexProviderSync.Core.Tests/CoreWritePlanningTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/CoreWritePlanningTests.cs @@ -5,6 +5,45 @@ namespace CodexProviderSync.Core.Tests; public sealed class CoreWritePlanningTests { + [Fact] + public async Task ContentFingerprintHint_ReusesExactDigestForPlanPreview() + { + string root = Path.Combine(Path.GetTempPath(), $"codex-provider-hint-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + string path = Path.Combine(root, "rollout.jsonl"); + await File.WriteAllTextAsync(path, "{\"type\":\"session_meta\"}\n"); + try + { + FileInfo info = new(path); + string digest = "sha256:" + Convert.ToHexString( + System.Security.Cryptography.SHA256.HashData(await File.ReadAllBytesAsync(path))) + .ToLowerInvariant() + + $":{info.Length}:{info.LastWriteTimeUtc.Ticks}"; + int hintedReads = 0; + CoreWritePlanSnapshot hinted = await CoreWriteSnapshotBuilder.BuildAsync( + "sync", + "hinted", + [new CoreWriteTargetSpec(path, "replace")], + contentFingerprintHints: + [ + new CoreWriteContentFingerprintHint( + path, + digest, + info.Length, + info.LastWriteTimeUtc.Ticks) + ], + fingerprintObserver: _ => hintedReads += 1); + + CoreWritePlanTarget target = Assert.Single(hinted.Targets); + Assert.Equal(digest, target.Fingerprint); + Assert.Equal(0, hintedReads); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + [Fact] public async Task CheckedSync_RejectsDriftBeforeBackupOrMutation() { @@ -524,6 +563,39 @@ public async Task SqliteWalFingerprint_TreatsMissingAndZeroLengthAsEquivalent() CoreWriteSnapshotBuilder.AssertExactMatch(empty, missingAgain); } + [Fact] + public async Task SnapshotBuilder_ReusesRecursiveInventoryAcrossMainAndAutoPruneTargets() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + string backupRoot = Path.Combine(fixture.Root, "backups"); + string candidate = Path.Combine(backupRoot, "20260807T000000000Z"); + string payload = Path.Combine(candidate, "metadata.json"); + Directory.CreateDirectory(candidate); + await File.WriteAllTextAsync(payload, "{}"); + List fingerprintedPaths = []; + + CoreWritePlanSnapshot snapshot = await CoreWriteSnapshotBuilder.BuildAsync( + "sync", + "shared-recursive-inventory", + [new CoreWriteTargetSpec( + backupRoot, + "create-and-prune", + CoreWriteFingerprintMode.RecursiveInventory)], + [new CoreWriteTargetSpec( + candidate, + "delete", + CoreWriteFingerprintMode.RecursiveInventory)], + fingerprintObserver: fingerprintedPaths.Add); + + Assert.Single(snapshot.Targets); + Assert.Single(snapshot.AutoPruneDeletionTargets); + StringComparer comparer = OperatingSystem.IsWindows() + ? StringComparer.OrdinalIgnoreCase + : StringComparer.Ordinal; + Assert.Equal(1, fingerprintedPaths.Count(path => comparer.Equals(path, Path.GetFullPath(candidate)))); + Assert.Equal(1, fingerprintedPaths.Count(path => comparer.Equals(path, Path.GetFullPath(payload)))); + } + [Fact] public async Task CheckedRestore_RejectsCommittedWalDriftBeforeMutation() { diff --git a/desktop/CodexProviderSync.Core.Tests/SessionRolloutPerformanceTests.cs b/desktop/CodexProviderSync.Core.Tests/SessionRolloutPerformanceTests.cs new file mode 100644 index 0000000..4d08f3e --- /dev/null +++ b/desktop/CodexProviderSync.Core.Tests/SessionRolloutPerformanceTests.cs @@ -0,0 +1,156 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text.Json; +using Xunit.Abstractions; + +namespace CodexProviderSync.Core.Tests; + +public sealed class SessionRolloutPerformanceTests(ITestOutputHelper output) +{ + [Fact] + public async Task ProviderChange_ContentFingerprintMatchesExactRolloutBytes() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + try + { + string path = fixture.RolloutPath("sessions", "rollout-fingerprint.jsonl"); + await fixture.WriteRolloutAsync(path, "thread-fingerprint", "apigather"); + FileInfo snapshot = new(path); + string digest = Convert.ToHexString( + SHA256.HashData(await File.ReadAllBytesAsync(path))) + .ToLowerInvariant(); + + SessionChangeCollection result = await new SessionRolloutService() + .CollectSessionChangesAsync(fixture.CodexHome, "openai"); + + SessionChange change = Assert.Single(result.Changes); + Assert.Equal( + $"sha256:{digest}:{snapshot.Length}:{snapshot.LastWriteTimeUtc.Ticks}", + change.ContentFingerprint); + } + finally + { + Directory.Delete(fixture.Root, recursive: true); + } + } + + [Fact] + public async Task CollectSessionChanges_UsesOneContentPassPerRollout() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + try + { + for (int index = 0; index < 16; index++) + { + string directory = index < 12 ? "sessions" : "archived_sessions"; + string path = fixture.RolloutPath(directory, $"rollout-{index:D2}.jsonl"); + await fixture.WriteRolloutAsync(path, $"thread-{index:D2}", "apigather"); + } + + SessionChangeCollection result = await new SessionRolloutService() + .CollectSessionChangesAsync(fixture.CodexHome, "openai"); + + Assert.Equal(16, result.ScanMetrics.EnumeratedRolloutFiles); + Assert.Equal(16, result.ScanMetrics.ParsedSessionFiles); + Assert.Equal(16, result.ScanMetrics.ContentScanPasses); + Assert.Equal(0, result.ScanMetrics.ModelScanFiles); + Assert.Equal(16, result.Changes.Count); + Assert.Equal(16, result.UserEventThreadIds.Count); + } + finally + { + Directory.Delete(fixture.Root, recursive: true); + } + } + + [Fact] + public async Task CollectSessionChanges_CollectsAllModelBackupsInTheSameContentPass() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + try + { + for (int index = 0; index < 8; index++) + { + string path = fixture.RolloutPath("sessions", $"rollout-model-{index:D2}.jsonl"); + await fixture.WriteRolloutWithTurnContextAsync( + path, + $"thread-model-{index:D2}", + "openai", + "old-model"); + } + + SessionChangeCollection result = await new SessionRolloutService() + .CollectSessionChangesAsync( + fixture.CodexHome, + "openai", + targetModel: "new-model"); + + Assert.Equal(8, result.ScanMetrics.EnumeratedRolloutFiles); + Assert.Equal(8, result.ScanMetrics.ContentScanPasses); + Assert.Equal(8, result.ScanMetrics.ModelScanFiles); + Assert.Equal(8, result.Changes.Count); + Assert.All(result.Changes, change => Assert.Equal(2, change.OriginalTurnContextModels.Count)); + } + finally + { + Directory.Delete(fixture.Root, recursive: true); + } + } + + [Fact] + [Trait("Category", "Performance")] + public async Task EightHundredRollouts_UseExactlyEightHundredContentPasses() + { + if (!string.Equals( + Environment.GetEnvironmentVariable("CODEX_PROVIDER_SYNC_RUN_PERF_TESTS"), + "1", + StringComparison.Ordinal)) + { + return; + } + + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + try + { + string sessionDirectory = Path.GetDirectoryName( + fixture.RolloutPath("sessions", "rollout-placeholder.jsonl"))!; + for (int index = 0; index < 800; index++) + { + string first = JsonSerializer.Serialize(new + { + timestamp = "2026-08-07T00:00:00.000Z", + type = "session_meta", + payload = new + { + id = $"thread-{index:D4}", + cwd = "C:\\AITemp", + model_provider = "apigather" + } + }); + string user = JsonSerializer.Serialize(new + { + type = "event_msg", + payload = new { type = "user_message", message = "hi" } + }); + await File.WriteAllTextAsync( + Path.Combine(sessionDirectory, $"rollout-{index:D4}.jsonl"), + $"{first}\n{user}\n"); + } + + Stopwatch timer = Stopwatch.StartNew(); + SessionChangeCollection result = await new SessionRolloutService() + .CollectSessionChangesAsync(fixture.CodexHome, "openai"); + timer.Stop(); + + Assert.Equal(800, result.ScanMetrics.EnumeratedRolloutFiles); + Assert.Equal(800, result.ScanMetrics.ContentScanPasses); + Assert.Equal(0, result.ScanMetrics.ModelScanFiles); + output.WriteLine( + $"rollouts=800 elapsedMs={timer.ElapsedMilliseconds} contentPasses={result.ScanMetrics.ContentScanPasses}"); + } + finally + { + Directory.Delete(fixture.Root, recursive: true); + } + } +} diff --git a/desktop/CodexProviderSync.Core.Tests/TextFormatterTests.cs b/desktop/CodexProviderSync.Core.Tests/TextFormatterTests.cs index 5d4a540..70e478c 100644 --- a/desktop/CodexProviderSync.Core.Tests/TextFormatterTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/TextFormatterTests.cs @@ -145,6 +145,42 @@ public void ResultFormatters_ChineseCoverSyncRestoreAndBackupCleanup() Assert.Contains("释放空间: 2 KB", pruneText); } + [Fact] + public void PerformanceFormatters_ReportStageDurationsAndStructuralCounts() + { + SyncPerformanceMetrics sync = new() + { + TotalDurationMs = 100, + PreparationDurationMs = 20, + CheckedPlanValidationDurationMs = 15, + BackupDurationMs = 10, + MutationDurationMs = 50, + PruneDurationMs = 5, + JournalFullValidationCount = 2, + RolloutScan = new SessionScanMetrics + { + EnumeratedRolloutFiles = 800, + ContentScanPasses = 800 + } + }; + StatusPerformanceMetrics status = new() + { + TotalDurationMs = 30, + RolloutScanDurationMs = 20, + BackupSummaryDurationMs = 4, + RolloutScan = sync.RolloutScan + }; + + string syncText = TextFormatter.FormatPerformanceMetrics(sync, TextFormatter.ChineseSimplified); + string statusText = TextFormatter.FormatPerformanceMetrics(status, TextFormatter.English); + + Assert.Contains("计划校验=15ms", syncText); + Assert.Contains("内容扫描=800", syncText); + Assert.Contains("journal全量校验=2", syncText); + Assert.Contains("rollout-scan=20ms", statusText); + Assert.Contains("content-scans=800", statusText); + } + private static StatusSnapshot MinimalStatus() => new() { CodexHome = @"C:\Codex", diff --git a/desktop/CodexProviderSync.Core.Tests/TransactionJournalPerformanceTests.cs b/desktop/CodexProviderSync.Core.Tests/TransactionJournalPerformanceTests.cs new file mode 100644 index 0000000..d7990d5 --- /dev/null +++ b/desktop/CodexProviderSync.Core.Tests/TransactionJournalPerformanceTests.cs @@ -0,0 +1,64 @@ +using System.Diagnostics; +using Xunit.Abstractions; + +namespace CodexProviderSync.Core.Tests; + +public sealed class TransactionJournalPerformanceTests(ITestOutputHelper output) +{ + [Fact] + [Trait("Category", "Performance")] + public async Task EightHundredTargets_AppendWithoutGrowingFullJournalReads() + { + if (!OperatingSystem.IsWindows()) + { + return; + } + if (!string.Equals( + Environment.GetEnvironmentVariable("CODEX_PROVIDER_SYNC_RUN_PERF_TESTS"), + "1", + StringComparison.Ordinal)) + { + return; + } + + string root = Path.Combine( + Path.GetTempPath(), + $"codex-provider-journal-perf-{Guid.NewGuid():N}"); + string backupDir = Path.Combine(root, "backup"); + string codexHome = Path.Combine(root, ".codex"); + Directory.CreateDirectory(backupDir); + Directory.CreateDirectory(codexHome); + string[] targets = Enumerable.Range(0, 800) + .Select(index => Path.Combine(codexHome, $"rollout-{index:D4}.jsonl")) + .ToArray(); + + try + { + await using FileTransactionJournal journal = await FileTransactionJournal.CreateOwnedAsync( + backupDir, + codexHome, + "target-provider", + targets); + Stopwatch timer = Stopwatch.StartNew(); + foreach (string target in targets) + { + await journal.ApplyingAsync("rollout", target); + await journal.AppliedAsync("rollout", target); + } + timer.Stop(); + + Assert.Equal(0, journal.AppendFullJournalValidationCount); + await journal.CommittedAsync(); + Assert.Equal(2, journal.AppendFullJournalValidationCount); + output.WriteLine( + $"targets=800 appends=1600 elapsedMs={timer.ElapsedMilliseconds} fullJournalReads={journal.AppendFullJournalValidationCount}"); + } + finally + { + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + } + } +} diff --git a/desktop/CodexProviderSync.Core.Tests/TransactionJournalTests.cs b/desktop/CodexProviderSync.Core.Tests/TransactionJournalTests.cs index 4ab78f0..9a1578d 100644 --- a/desktop/CodexProviderSync.Core.Tests/TransactionJournalTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/TransactionJournalTests.cs @@ -82,6 +82,90 @@ public async Task AppendAsync_ResynchronizesFromValidExternallyAppendedRecord() Assert.Equal(3, applied.LastSequence); Assert.Equal("applied", applied.State); Assert.False(applied.InvalidTail); + Assert.Equal(1, fixture.Journal.AppendFullJournalValidationCount); + } + + [Fact] + public async Task NormalProgressAppends_DoNotReparseGrowingJournal() + { + JournalFixture fixture = await JournalFixture.CreateAsync(32, owned: true); + + foreach (string target in fixture.Targets) + { + await fixture.Journal.ApplyingAsync("rollout", target); + await fixture.Journal.AppliedAsync("rollout", target); + } + + Assert.Equal( + OperatingSystem.IsWindows() ? 0 : 64, + fixture.Journal.AppendFullJournalValidationCount); + await fixture.Journal.CommittedAsync(); + Assert.Equal( + OperatingSystem.IsWindows() ? 2 : 66, + fixture.Journal.AppendFullJournalValidationCount); + + PendingTransactionInfo committed = await fixture.Journal.ReadCurrentInfoAsync(); + Assert.True(committed.Terminal); + Assert.Equal(66, committed.LastSequence); + Assert.All(committed.AffectedTargets, target => Assert.Equal("applied", target.State)); + } + + [Fact] + public async Task AppendAsync_RejectsSameLengthInvalidTailBeforeWriting() + { + JournalFixture fixture = await JournalFixture.CreateAsync(1); + byte[] before = await File.ReadAllBytesAsync(fixture.Journal.FilePath); + Assert.Equal((byte)'\n', before[^1]); + before[^1] = (byte)' '; + await File.WriteAllBytesAsync(fixture.Journal.FilePath, before); + long invalidLength = new FileInfo(fixture.Journal.FilePath).Length; + + InvalidOperationException error = await Assert.ThrowsAsync( + () => fixture.Journal.ApplyingAsync("rollout", fixture.Targets[0])); + + Assert.Contains("invalid", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(invalidLength, new FileInfo(fixture.Journal.FilePath).Length); + Assert.Equal(1, fixture.Journal.AppendFullJournalValidationCount); + Assert.True((await fixture.Journal.ReadCurrentInfoAsync()).InvalidTail); + } + + [Fact] + public async Task AppendAsync_RejectsSameLengthPreparedRewriteBeforeWriting() + { + JournalFixture fixture = await JournalFixture.CreateAsync(1); + string originalTarget = Path.GetFullPath(fixture.Targets[0]); + string before = await File.ReadAllTextAsync(fixture.Journal.FilePath); + string rewritten = before.Replace("rollout-0.jsonl", "rollout-0.jsonx", StringComparison.Ordinal); + Assert.Equal(before.Length, rewritten.Length); + Assert.NotEqual(before, rewritten); + await File.WriteAllTextAsync(fixture.Journal.FilePath, rewritten); + long rewrittenLength = new FileInfo(fixture.Journal.FilePath).Length; + + InvalidOperationException error = await Assert.ThrowsAsync( + () => fixture.Journal.ApplyingAsync("rollout", originalTarget)); + + Assert.Contains("prepared", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(rewrittenLength, new FileInfo(fixture.Journal.FilePath).Length); + } + + [Fact] + public async Task OwnedJournal_BlocksExternalSameLengthRewriteOnWindows() + { + if (!OperatingSystem.IsWindows()) + { + return; + } + JournalFixture fixture = await JournalFixture.CreateAsync(1, owned: true); + await Assert.ThrowsAsync( + async () => + { + await using FileStream _ = new( + fixture.Journal.FilePath, + FileMode.Open, + FileAccess.Write, + FileShare.ReadWrite | FileShare.Delete); + }); + await fixture.Journal.ApplyingAsync("rollout", fixture.Targets[0]); } [Fact] @@ -185,7 +269,7 @@ private sealed record JournalFixture( FileTransactionJournal Journal, IReadOnlyList Targets) { - internal static async Task CreateAsync(int targetCount) + internal static async Task CreateAsync(int targetCount, bool owned = false) { string root = Path.Combine( Path.GetTempPath(), @@ -197,11 +281,17 @@ internal static async Task CreateAsync(int targetCount) string[] targets = Enumerable.Range(0, targetCount) .Select(index => Path.Combine(codexHome, $"rollout-{index}.jsonl")) .ToArray(); - FileTransactionJournal journal = await FileTransactionJournal.CreateAsync( - backupDir, - codexHome, - "target-provider", - targets); + FileTransactionJournal journal = owned + ? await FileTransactionJournal.CreateOwnedAsync( + backupDir, + codexHome, + "target-provider", + targets) + : await FileTransactionJournal.CreateAsync( + backupDir, + codexHome, + "target-provider", + targets); return new JournalFixture(root, journal, targets); } } diff --git a/desktop/CodexProviderSync.Core/BackupService.cs b/desktop/CodexProviderSync.Core/BackupService.cs index a12fd41..8322717 100644 --- a/desktop/CodexProviderSync.Core/BackupService.cs +++ b/desktop/CodexProviderSync.Core/BackupService.cs @@ -1,3 +1,4 @@ +using System.Text; using System.Text.Json; namespace CodexProviderSync.Core; @@ -9,6 +10,8 @@ public sealed class BackupService internal Func? AtomicWriteFaultInjector { get; set; } + internal Action? DirectoryInventoryFallbackObserver { get; set; } + public BackupService(SessionRolloutService sessionRolloutService, SqliteStateService sqliteStateService) { _sessionRolloutService = sessionRolloutService; @@ -136,10 +139,7 @@ await AtomicFile.WriteAllTextAsync( GlobalStateFilePresent = globalStateFilePresent, GlobalStateBackupFilePresent = globalStateBackupFilePresent }; - await AtomicFile.WriteAllTextAsync( - Path.Combine(backupDir, "metadata.json"), - JsonSerializer.Serialize(metadata, JsonOptions()), - faultInjector: AtomicWriteFaultInjector); + await WriteMetadataWithInventoryAsync(backupDir, metadata); return backupDir; } @@ -332,17 +332,16 @@ await File.ReadAllTextAsync(metadataPath), ChangedSessionFiles = sessionChanges.Count, GlobalStateFiles = metadata.GlobalStateFiles, GlobalStateFilePresent = metadata.GlobalStateFilePresent, - GlobalStateBackupFilePresent = metadata.GlobalStateBackupFilePresent + GlobalStateBackupFilePresent = metadata.GlobalStateBackupFilePresent, + SizeBytes = metadata.SizeBytes, + FileCount = metadata.FileCount }; await AtomicFile.WriteAllTextAsync( manifestPath, JsonSerializer.Serialize(sessionManifest, JsonOptions()), faultInjector: AtomicWriteFaultInjector); - await AtomicFile.WriteAllTextAsync( - metadataPath, - JsonSerializer.Serialize(metadata, JsonOptions()), - faultInjector: AtomicWriteFaultInjector); + await WriteMetadataWithInventoryAsync(normalizedBackupDir, metadata); } internal async Task> ReadSessionBackupEntriesAsync( @@ -472,6 +471,22 @@ await File.ReadAllTextAsync(metadataPath), }; } + internal async Task RefreshMetadataInventoryAsync(string backupDir) + { + string normalizedBackupDir = Path.GetFullPath(backupDir); + string metadataPath = Path.Combine(normalizedBackupDir, "metadata.json"); + BackupMetadataFile metadata = JsonSerializer.Deserialize( + await File.ReadAllTextAsync(metadataPath), + JsonOptions()) ?? throw new InvalidOperationException($"Backup metadata is invalid: {backupDir}"); + if (!string.Equals(metadata.Namespace, AppConstants.BackupNamespace, StringComparison.Ordinal) + || metadata.Version is not (1 or 2)) + { + throw new InvalidOperationException($"Unsupported backup metadata in {metadataPath}."); + } + + await WriteMetadataWithInventoryAsync(normalizedBackupDir, metadata); + } + public Task GetBackupSummaryAsync(string codexHome) { string backupRoot = AppConstants.DefaultBackupRoot(codexHome); @@ -487,7 +502,7 @@ public Task GetBackupSummaryAsync(string codexHome) } List entries = GetManagedBackupDirectories(backupRoot); - long totalBytes = entries.Sum(static entry => GetDirectorySize(entry.FullName)); + long totalBytes = entries.Sum(entry => GetBackupDirectorySize(entry.FullName)); return new BackupSummary { @@ -581,7 +596,7 @@ private async Task PruneBackupsCoreAsync( long freedBytes = 0; foreach (DirectoryInfo entry in toDelete) { - freedBytes += GetDirectorySize(entry.FullName); + freedBytes += GetBackupDirectorySize(entry.FullName); entry.Delete(recursive: true); } @@ -912,16 +927,127 @@ private static JsonSerializerOptions JsonOptions() }; } - private static long GetDirectorySize(string directoryPath) + private async Task WriteMetadataWithInventoryAsync(string backupDir, BackupMetadataFile metadata) + { + string metadataPath = Path.Combine(backupDir, "metadata.json"); + (long payloadBytes, int payloadFileCount) = GetDirectoryInventory( + backupDir, + metadataPath); + int fileCount = checked(payloadFileCount + 1); + long sizeBytes = 0; + string serialized = string.Empty; + for (int attempt = 0; attempt < 8; attempt += 1) + { + BackupMetadataFile withInventory = CopyMetadataWithInventory(metadata, sizeBytes, fileCount); + serialized = JsonSerializer.Serialize(withInventory, JsonOptions()); + long nextSizeBytes = checked(payloadBytes + Encoding.UTF8.GetByteCount(serialized)); + if (nextSizeBytes == sizeBytes) + { + break; + } + sizeBytes = nextSizeBytes; + } + + BackupMetadataFile finalMetadata = CopyMetadataWithInventory(metadata, sizeBytes, fileCount); + serialized = JsonSerializer.Serialize(finalMetadata, JsonOptions()); + long verifiedSizeBytes = checked(payloadBytes + Encoding.UTF8.GetByteCount(serialized)); + if (verifiedSizeBytes != sizeBytes) + { + finalMetadata = CopyMetadataWithInventory(metadata, verifiedSizeBytes, fileCount); + serialized = JsonSerializer.Serialize(finalMetadata, JsonOptions()); + } + await AtomicFile.WriteAllTextAsync( + metadataPath, + serialized, + faultInjector: AtomicWriteFaultInjector); + } + + private static BackupMetadataFile CopyMetadataWithInventory( + BackupMetadataFile metadata, + long sizeBytes, + int fileCount) + { + return new BackupMetadataFile + { + Version = metadata.Version, + Namespace = metadata.Namespace, + CodexHome = metadata.CodexHome, + SqliteHome = metadata.SqliteHome, + TargetProvider = metadata.TargetProvider, + CreatedAt = metadata.CreatedAt, + DbFiles = metadata.DbFiles, + SqliteDbFiles = metadata.SqliteDbFiles, + ChangedSessionFiles = metadata.ChangedSessionFiles, + GlobalStateFiles = metadata.GlobalStateFiles, + GlobalStateFilePresent = metadata.GlobalStateFilePresent, + GlobalStateBackupFilePresent = metadata.GlobalStateBackupFilePresent, + SizeBytes = sizeBytes, + FileCount = fileCount + }; + } + + private long GetBackupDirectorySize(string directoryPath) + { + if (TryReadCachedDirectoryInventory(directoryPath, out long sizeBytes)) + { + return sizeBytes; + } + DirectoryInventoryFallbackObserver?.Invoke(directoryPath); + return GetDirectoryInventory(directoryPath).SizeBytes; + } + + private static bool TryReadCachedDirectoryInventory(string directoryPath, out long sizeBytes) + { + sizeBytes = 0; + string metadataPath = Path.Combine(directoryPath, "metadata.json"); + try + { + using JsonDocument document = JsonDocument.Parse(File.ReadAllText(metadataPath)); + JsonElement root = document.RootElement; + if (!root.TryGetProperty("namespace", out JsonElement namespaceValue) + || !string.Equals(namespaceValue.GetString(), AppConstants.BackupNamespace, StringComparison.Ordinal) + || !root.TryGetProperty("sizeBytes", out JsonElement sizeValue) + || !sizeValue.TryGetInt64(out long cachedSize) + || cachedSize < 0 + || !root.TryGetProperty("fileCount", out JsonElement countValue) + || !countValue.TryGetInt32(out int cachedFileCount) + || cachedFileCount < 1) + { + return false; + } + sizeBytes = cachedSize; + return true; + } + catch + { + return false; + } + } + + private static (long SizeBytes, int FileCount) GetDirectoryInventory( + string directoryPath, + string? excludedFilePath = null) { if (!Directory.Exists(directoryPath)) { - return 0; + return (0, 0); } - return Directory - .EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories) - .Sum(static filePath => new FileInfo(filePath).Length); + string? excluded = string.IsNullOrWhiteSpace(excludedFilePath) + ? null + : Path.GetFullPath(excludedFilePath); + long sizeBytes = 0; + int fileCount = 0; + foreach (string filePath in Directory.EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories)) + { + if (excluded is not null && PathsEqual(filePath, excluded)) + { + continue; + } + sizeBytes = checked(sizeBytes + new FileInfo(filePath).Length); + fileCount = checked(fileCount + 1); + } + return (sizeBytes, fileCount); } private static List GetManagedBackupDirectories(string backupRoot) @@ -945,7 +1071,7 @@ private static bool IsManagedBackupDirectory(string backupDirectoryPath) try { - BackupMetadataFile? metadata = JsonSerializer.Deserialize( + BackupMetadataValidationFile? metadata = JsonSerializer.Deserialize( File.ReadAllText(metadataPath), JsonOptions()); return string.Equals(metadata?.Namespace, AppConstants.BackupNamespace, StringComparison.Ordinal); @@ -955,6 +1081,22 @@ private static bool IsManagedBackupDirectory(string backupDirectoryPath) return false; } } + + private sealed class BackupMetadataValidationFile + { + public int Version { get; init; } + public required string Namespace { get; init; } + public required string CodexHome { get; init; } + public string? SqliteHome { get; init; } + public required string TargetProvider { get; init; } + public required DateTimeOffset CreatedAt { get; init; } + public required List DbFiles { get; init; } + public List SqliteDbFiles { get; init; } = []; + public int ChangedSessionFiles { get; init; } + public Dictionary? GlobalStateFiles { get; init; } + public bool? GlobalStateFilePresent { get; init; } + public bool? GlobalStateBackupFilePresent { get; init; } + } } internal sealed record BackupRecoveryCoverage(bool Config, bool Database, bool Sessions); diff --git a/desktop/CodexProviderSync.Core/CodexSyncService.cs b/desktop/CodexProviderSync.Core/CodexSyncService.cs index 8778207..6a7bc2b 100644 --- a/desktop/CodexProviderSync.Core/CodexSyncService.cs +++ b/desktop/CodexProviderSync.Core/CodexSyncService.cs @@ -1,3 +1,5 @@ +using System.Diagnostics; + namespace CodexProviderSync.Core; public sealed class CodexSyncService @@ -50,13 +52,16 @@ public async Task GetStatusAsync( string? explicitCodexHome = null, string? explicitSqliteHome = null) { + long totalStarted = Stopwatch.GetTimestamp(); string codexHome = _codexHomeService.NormalizeCodexHome(explicitCodexHome); await _codexHomeService.EnsureCodexHomeAsync(codexHome); string configText = await _configFileService.ReadConfigTextAsync(_codexHomeService.ConfigPath(codexHome)); CodexStorageLayout storage = await PrepareStorageAsync(codexHome, explicitSqliteHome, configText); CurrentProviderInfo currentProvider = _configFileService.ReadCurrentProviderFromConfigText(configText); IReadOnlyList configuredProviders = _configFileService.ListConfiguredProviderIds(configText); + long rolloutScanStarted = Stopwatch.GetTimestamp(); SessionChangeCollection rolloutInfo = await _sessionRolloutService.CollectSessionChangesAsync(codexHome, "__status_only__", skipLockedReads: true); + long rolloutScanDurationMs = ElapsedMilliseconds(rolloutScanStarted); StateDbLocation? stateDbLocation = storage.StateDbLocation; ProviderCounts? sqliteCounts = storage.SqliteAccess.Supported ? await _sqliteStateService.ReadSqliteProviderCountsAsync(storage) @@ -71,7 +76,9 @@ public async Task GetStatusAsync( || sqliteCounts?.Unreadable == true ? [] : await _globalStateService.ReadProjectThreadVisibilityAsync(storage); + long backupSummaryStarted = Stopwatch.GetTimestamp(); BackupSummary backupSummary = await _backupService.GetBackupSummaryAsync(codexHome); + long backupSummaryDurationMs = ElapsedMilliseconds(backupSummaryStarted); IReadOnlyList pendingTransactions = await FileTransactionJournal.FindPendingAsync(codexHome); return new StatusSnapshot @@ -100,7 +107,14 @@ public async Task GetStatusAsync( item.State, item.BackupDir, item.JournalPath)) - .ToArray() + .ToArray(), + PerformanceMetrics = new StatusPerformanceMetrics + { + TotalDurationMs = ElapsedMilliseconds(totalStarted), + RolloutScanDurationMs = rolloutScanDurationMs, + BackupSummaryDurationMs = backupSummaryDurationMs, + RolloutScan = rolloutInfo.ScanMetrics + } }; } @@ -260,7 +274,8 @@ private async Task PrepareSyncAsync( private async Task BuildSyncPlanSnapshotAsync( SyncPreparation preparation, int keepCount, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool useContentFingerprintHints = true) { string operation = preparation.SwitchPreparation is null ? "sync" : "switch"; List targets = @@ -339,6 +354,15 @@ private async Task BuildSyncPlanSnapshotAsync( "delete", CoreWriteFingerprintMode.RecursiveInventory)), warnings: warnings, + contentFingerprintHints: useContentFingerprintHints + ? preparation.WritableChanges + .Where(static change => change.ContentFingerprint is not null) + .Select(static change => new CoreWriteContentFingerprintHint( + change.Path, + change.ContentFingerprint!, + change.OriginalFileLength, + change.OriginalLastWriteTimeUtcTicks)) + : null, cancellationToken: cancellationToken); } @@ -366,12 +390,14 @@ private async Task RunSyncCoreAsync( DateTimeOffset? snapshotExpiresAtUtc, CancellationToken cancellationToken = default) { + long totalStarted = Stopwatch.GetTimestamp(); ValidateAutomaticRetention(keepCount); string codexHome = _codexHomeService.NormalizeCodexHome(explicitCodexHome); await _codexHomeService.EnsureCodexHomeAsync(codexHome); await using LockHandle _ = await _lockService.AcquireLockAsync(codexHome, "sync"); await FileTransactionJournal.AssertNoPendingAsync(codexHome); + long preparationStarted = Stopwatch.GetTimestamp(); SyncPreparation preparation = await PrepareSyncAsync( codexHome, provider, @@ -380,6 +406,7 @@ private async Task RunSyncCoreAsync( explicitSqliteHome, switchPreparationFactory, cancellationToken); + long preparationDurationMs = ElapsedMilliseconds(preparationStarted); string configPath = preparation.ConfigPath; string configText = preparation.ConfigText; SwitchPreparation? switchPreparation = preparation.SwitchPreparation; @@ -394,16 +421,20 @@ private async Task RunSyncCoreAsync( List skippedRolloutFiles = [.. preparation.SkippedRolloutFiles]; IReadOnlyList skippedUnreadableRolloutFiles = preparation.SkippedUnreadableRolloutFiles; IReadOnlyList? checkedAutoPruneDeletionTargets = null; + long checkedPlanValidationDurationMs = 0; if (expectedSnapshot is not null) { + long checkedPlanValidationStarted = Stopwatch.GetTimestamp(); AssertSnapshotFresh(snapshotExpiresAtUtc); CoreWritePlanSnapshot actualSnapshot = await BuildSyncPlanSnapshotAsync( preparation, keepCount, - cancellationToken); + cancellationToken, + useContentFingerprintHints: false); CoreWriteSnapshotBuilder.AssertExactMatch(expectedSnapshot, actualSnapshot); AssertSnapshotFresh(snapshotExpiresAtUtc); checkedAutoPruneDeletionTargets = expectedSnapshot.AutoPruneDeletionTargets; + checkedPlanValidationDurationMs = ElapsedMilliseconds(checkedPlanValidationStarted); } cancellationToken.ThrowIfCancellationRequested(); if (FaultInjector is not null) @@ -411,12 +442,15 @@ private async Task RunSyncCoreAsync( await FaultInjector("before_backup", null, 0); } string? effectiveConfigBackupText = switchPreparation is null ? configBackupText : configText; + long backupStarted = Stopwatch.GetTimestamp(); string backupDir = await _backupService.CreateBackupAsync( storage, targetProvider, writableChanges, configPath, effectiveConfigBackupText); + long backupDurationMs = ElapsedMilliseconds(backupStarted); + long mutationStarted = Stopwatch.GetTimestamp(); List appliedSessionChanges = []; bool sqliteMutationCommitted = false; bool sqliteCommitAttempted = false; @@ -429,7 +463,7 @@ private async Task RunSyncCoreAsync( .Concat(switchPreparation is null ? [] : [Path.GetFullPath(configPath)]) .Concat(storage.StateDbLocation is null ? [] : [Path.GetFullPath(storage.StateDbLocation.Path)]) .ToArray(); - FileTransactionJournal journal = await FileTransactionJournal.CreateAsync( + await using FileTransactionJournal journal = await FileTransactionJournal.CreateOwnedAsync( backupDir, codexHome, targetProvider, @@ -600,6 +634,22 @@ await FaultInjector( } await journal.CommittedAsync(); transactionCommitted = true; + await journal.DisposeAsync(); + // The transaction is already committed and every target is on disk. + // Refreshing the backup inventory only corrects the recorded size + // and file count in metadata.json, so a failure here must degrade + // to a warning: throwing would report a successful sync as failed + // and skip the automatic backup pruning below. + string? backupInventoryWarning = null; + try + { + await _backupService.RefreshMetadataInventoryAsync(backupDir); + } + catch (Exception error) + { + backupInventoryWarning = $"Backup inventory refresh failed: {error.Message}"; + } + long mutationDurationMs = ElapsedMilliseconds(mutationStarted); if (FaultInjector is not null) { await FaultInjector("after_transaction_commit", null, completedTargets.Count); @@ -607,6 +657,7 @@ await FaultInjector( BackupPruneResult? autoPruneResult = null; string? autoPruneWarning = null; + long pruneStarted = Stopwatch.GetTimestamp(); try { autoPruneResult = await _backupService.PruneAutomaticBackupsAsync( @@ -619,6 +670,8 @@ await FaultInjector( { autoPruneWarning = $"Automatic backup cleanup failed: {error.Message}"; } + long pruneDurationMs = ElapsedMilliseconds(pruneStarted); + autoPruneWarning = JoinBackupWarnings(backupInventoryWarning, autoPruneWarning); SyncResult result = new() { @@ -645,7 +698,18 @@ await FaultInjector( AutoPruneResult = autoPruneResult, AutoPruneWarning = autoPruneWarning, ConfigUpdated = switchPreparation is not null, - ModelSync = switchPreparation?.ModelSync ?? ModelSyncOutcome.NotApplicable() + ModelSync = switchPreparation?.ModelSync ?? ModelSyncOutcome.NotApplicable(), + PerformanceMetrics = new SyncPerformanceMetrics + { + TotalDurationMs = ElapsedMilliseconds(totalStarted), + PreparationDurationMs = preparationDurationMs, + CheckedPlanValidationDurationMs = checkedPlanValidationDurationMs, + BackupDurationMs = backupDurationMs, + MutationDurationMs = mutationDurationMs, + PruneDurationMs = pruneDurationMs, + JournalFullValidationCount = journal.AppendFullJournalValidationCount, + RolloutScan = sessionInfo.ScanMetrics + } }; return result; } @@ -712,6 +776,7 @@ await FaultInjector( // Preserve the original and rollback failures when the // journal itself is no longer writable. } + await TryRefreshBackupInventoryAsync(backupDir); IReadOnlyList reportedCompletedTargets = BuildReportedCompletedTargets( completedTargets, observedMutatedTargets); @@ -729,6 +794,7 @@ await FaultInjector( recoveryRequired: true); } + await TryRefreshBackupInventoryAsync(backupDir); IReadOnlyList completedAfterRollback = BuildReportedCompletedTargets( completedTargets, observedMutatedTargets); @@ -1154,7 +1220,27 @@ await FileTransactionJournal.MarkBackupRolledBackAsync( preparation.BackupDirectory, codexHome, result.TargetProvider); - return result; + // The restore and its journal marker are already durable. Refreshing the + // inventory only corrects metadata.json bookkeeping, so surface a + // failure as a warning instead of reporting a completed restore as + // failed. + try + { + await _backupService.RefreshMetadataInventoryAsync(preparation.BackupDirectory); + return result; + } + catch (Exception error) + { + return new RestoreResult + { + CodexHome = result.CodexHome, + BackupDir = result.BackupDir, + TargetProvider = result.TargetProvider, + CreatedAt = result.CreatedAt, + ChangedSessionFiles = result.ChangedSessionFiles, + BackupInventoryWarning = $"Backup inventory refresh failed: {error.Message}" + }; + } } private async Task PrepareRestoreAsync( @@ -1444,6 +1530,38 @@ public Task GetBackupStorageInfoAsync(string backupDir) return $"Encrypted content warning: {total} rollout file(s) contain encrypted_content from provider(s) {string.Join(", ", riskyProviders)}. Visibility metadata can be synchronized to {targetProvider}, but continuing or compacting those histories may fail with invalid_encrypted_content. Return to the original provider/account or start a new session if you need reliable continuation."; } + private static long ElapsedMilliseconds(long started) => + (long)Math.Round(Stopwatch.GetElapsedTime(started).TotalMilliseconds); + + /// + /// Rewrites the retained backup's recorded size and file count after the + /// journal reached a terminal state, so status and pruning do not trust an + /// inventory captured before those journal records existed. Used on the + /// rollback paths, where the caller is already reporting a failure: a + /// bookkeeping problem here must never replace the original error. + /// + private async Task TryRefreshBackupInventoryAsync(string backupDir) + { + try + { + await _backupService.RefreshMetadataInventoryAsync(backupDir); + } + catch + { + // The original sync failure and its rollback details are the + // authoritative diagnosis and must reach the caller unchanged. + } + } + + private static string? JoinBackupWarnings(string? first, string? second) + { + string[] parts = new[] { first, second } + .Where(static part => !string.IsNullOrWhiteSpace(part)) + .Select(static part => part!.Trim()) + .ToArray(); + return parts.Length == 0 ? null : string.Join(" | ", parts); + } + private async Task PrepareStorageAsync( string codexHome, string? explicitSqliteHome, diff --git a/desktop/CodexProviderSync.Core/CoreWritePlanning.cs b/desktop/CodexProviderSync.Core/CoreWritePlanning.cs index 54e51f9..a2ec0bd 100644 --- a/desktop/CodexProviderSync.Core/CoreWritePlanning.cs +++ b/desktop/CodexProviderSync.Core/CoreWritePlanning.cs @@ -54,6 +54,12 @@ internal sealed record CoreWriteTargetSpec( string Action, CoreWriteFingerprintMode FingerprintMode = CoreWriteFingerprintMode.Content); +internal sealed record CoreWriteContentFingerprintHint( + string Path, + string Fingerprint, + long Length, + long LastWriteTimeUtcTicks); + internal static class CoreWriteSnapshotBuilder { private const string FormatVersion = "core-write-snapshot-v2"; @@ -64,17 +70,36 @@ public static async Task BuildAsync( IEnumerable targets, IEnumerable? autoPruneDeletionTargets = null, IEnumerable? warnings = null, - CancellationToken cancellationToken = default) + IEnumerable? contentFingerprintHints = null, + CancellationToken cancellationToken = default, + Action? fingerprintObserver = null) { ArgumentException.ThrowIfNullOrWhiteSpace(operation); ArgumentNullException.ThrowIfNull(binding); + Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache = []; + foreach (CoreWriteContentFingerprintHint hint in contentFingerprintHints ?? []) + { + string fullPath = Path.GetFullPath(hint.Path); + FileInfo current = new(fullPath); + if (!current.Exists + || current.Length != hint.Length + || current.LastWriteTimeUtc.Ticks != hint.LastWriteTimeUtcTicks) + { + throw new CoreWritePlanStaleException(); + } + fingerprintCache[(fullPath, CoreWriteFingerprintMode.Content)] = hint.Fingerprint; + } IReadOnlyList capturedTargets = await CaptureTargetsAsync( targets, - cancellationToken); + fingerprintCache, + cancellationToken, + fingerprintObserver); IReadOnlyList capturedAutoPruneTargets = await CaptureTargetsAsync( autoPruneDeletionTargets ?? [], - cancellationToken); + fingerprintCache, + cancellationToken, + fingerprintObserver); IReadOnlyList capturedWarnings = (warnings ?? []) .Select(static warning => warning with { }) .OrderBy(static warning => warning.Code, StringComparer.Ordinal) @@ -132,7 +157,9 @@ public static void AssertExactMatch( private static async Task> CaptureTargetsAsync( IEnumerable specs, - CancellationToken cancellationToken) + Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache, + CancellationToken cancellationToken, + Action? fingerprintObserver) { CoreWriteTargetSpec[] normalized = specs .Select(static spec => new CoreWriteTargetSpec( @@ -144,19 +171,15 @@ private static async Task> CaptureTargetsAsyn .ThenBy(static spec => spec.Action, StringComparer.Ordinal) .ToArray(); List result = new(normalized.Length); - Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache = []; foreach (CoreWriteTargetSpec spec in normalized) { cancellationToken.ThrowIfCancellationRequested(); - (string Path, CoreWriteFingerprintMode Mode) cacheKey = (spec.Path, spec.FingerprintMode); - if (!fingerprintCache.TryGetValue(cacheKey, out string? fingerprint)) - { - fingerprint = await FingerprintPathAsync( - spec.Path, - spec.FingerprintMode, - cancellationToken); - fingerprintCache.Add(cacheKey, fingerprint); - } + string fingerprint = await FingerprintPathAsync( + spec.Path, + spec.FingerprintMode, + fingerprintCache, + cancellationToken, + fingerprintObserver); result.Add(new CoreWritePlanTarget(spec.Path, spec.Action, fingerprint)); } return result.AsReadOnly(); @@ -165,26 +188,44 @@ private static async Task> CaptureTargetsAsyn private static async Task FingerprintPathAsync( string fullPath, CoreWriteFingerprintMode mode, - CancellationToken cancellationToken) + Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache, + CancellationToken cancellationToken, + Action? fingerprintObserver) { + fullPath = Path.GetFullPath(fullPath); + (string Path, CoreWriteFingerprintMode Mode) cacheKey = (fullPath, mode); + if (fingerprintCache.TryGetValue(cacheKey, out string? cached)) + { + return cached; + } + cancellationToken.ThrowIfCancellationRequested(); + fingerprintObserver?.Invoke(fullPath); + string fingerprint; if (!File.Exists(fullPath) && !Directory.Exists(fullPath)) { - if (mode == CoreWriteFingerprintMode.SqliteWalContent) - { - return FingerprintEmptySqliteWal(fullPath); - } - return Sha256($"missing\n{fullPath}"); + fingerprint = mode == CoreWriteFingerprintMode.SqliteWalContent + ? FingerprintEmptySqliteWal(fullPath) + : Sha256($"missing\n{fullPath}"); + fingerprintCache.Add(cacheKey, fingerprint); + return fingerprint; } FileAttributes attributes = File.GetAttributes(fullPath); if ((attributes & FileAttributes.ReparsePoint) != 0) { DateTime lastWrite = File.GetLastWriteTimeUtc(fullPath); - return Sha256($"reparse\n{fullPath}\n{(int)attributes}\n{lastWrite.Ticks}"); + fingerprint = Sha256($"reparse\n{fullPath}\n{(int)attributes}\n{lastWrite.Ticks}"); + fingerprintCache.Add(cacheKey, fingerprint); + return fingerprint; } - return (attributes & FileAttributes.Directory) != 0 - ? await FingerprintDirectoryAsync(fullPath, mode, cancellationToken) + fingerprint = (attributes & FileAttributes.Directory) != 0 + ? await FingerprintDirectoryAsync( + fullPath, + mode, + fingerprintCache, + cancellationToken, + fingerprintObserver) : mode switch { CoreWriteFingerprintMode.RecursiveInventory => FingerprintFileInventory(fullPath, attributes), @@ -198,12 +239,16 @@ private static async Task FingerprintPathAsync( cancellationToken), _ => await FingerprintFileAsync(fullPath, cancellationToken) }; + fingerprintCache.Add(cacheKey, fingerprint); + return fingerprint; } private static async Task FingerprintDirectoryAsync( string directoryPath, CoreWriteFingerprintMode mode, - CancellationToken cancellationToken) + Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache, + CancellationToken cancellationToken, + Action? fingerprintObserver) { StringBuilder canonical = new(); Append(canonical, "type", "directory"); @@ -231,7 +276,12 @@ private static async Task FingerprintDirectoryAsync( Append( canonical, "entry.fingerprint", - await FingerprintPathAsync(entry, mode, cancellationToken)); + await FingerprintPathAsync( + entry, + mode, + fingerprintCache, + cancellationToken, + fingerprintObserver)); } return Sha256(canonical.ToString()); } diff --git a/desktop/CodexProviderSync.Core/Models.cs b/desktop/CodexProviderSync.Core/Models.cs index 4f744c8..e7158b7 100644 --- a/desktop/CodexProviderSync.Core/Models.cs +++ b/desktop/CodexProviderSync.Core/Models.cs @@ -37,6 +37,16 @@ public sealed class StatusSnapshot public required string BackupRoot { get; init; } public required BackupSummary BackupSummary { get; init; } public IReadOnlyList PendingTransactions { get; init; } = []; + [JsonIgnore] + public StatusPerformanceMetrics PerformanceMetrics { get; init; } = new(); +} + +public sealed class StatusPerformanceMetrics +{ + public long TotalDurationMs { get; init; } + public long RolloutScanDurationMs { get; init; } + public long BackupSummaryDurationMs { get; init; } + public SessionScanMetrics RolloutScan { get; init; } = new(); } public sealed record TransactionRecoveryInfo( @@ -119,6 +129,8 @@ public sealed class SessionChange public required string UpdatedFirstLine { get; init; } public bool ModelOnlyChange { get; init; } public IReadOnlyList OriginalTurnContextModels { get; set; } = []; + [JsonIgnore] + public string? ContentFingerprint { get; init; } } public sealed class TurnContextModelBackup @@ -137,6 +149,16 @@ public sealed class SessionChangeCollection public required ProviderCounts EncryptedContentCounts { get; init; } public required IReadOnlyCollection UserEventThreadIds { get; init; } public required IReadOnlyDictionary ThreadCwdsById { get; init; } + public SessionScanMetrics ScanMetrics { get; init; } = new(); +} + +public sealed class SessionScanMetrics +{ + public int EnumeratedRolloutFiles { get; init; } + public int ParsedSessionFiles { get; init; } + public int ContentScanPasses { get; init; } + public int ModelScanFiles { get; init; } + public long DurationMs { get; init; } } public sealed class SyncResult @@ -165,6 +187,20 @@ public sealed class SyncResult public ModelSyncOutcome ModelSync { get; init; } = ModelSyncOutcome.NotApplicable(); public BackupPruneResult? AutoPruneResult { get; init; } public string? AutoPruneWarning { get; init; } + [JsonIgnore] + public SyncPerformanceMetrics PerformanceMetrics { get; init; } = new(); +} + +public sealed class SyncPerformanceMetrics +{ + public long TotalDurationMs { get; init; } + public long PreparationDurationMs { get; init; } + public long CheckedPlanValidationDurationMs { get; init; } + public long BackupDurationMs { get; init; } + public long MutationDurationMs { get; init; } + public long PruneDurationMs { get; init; } + public int JournalFullValidationCount { get; init; } + public SessionScanMetrics RolloutScan { get; init; } = new(); } public sealed class ModelSyncOutcome @@ -209,6 +245,14 @@ public sealed class RestoreResult public required string TargetProvider { get; init; } public DateTimeOffset? CreatedAt { get; init; } public int ChangedSessionFiles { get; init; } + + /// + /// Set when the restore itself succeeded but refreshing the backup + /// directory inventory afterwards did not. The restored state is already + /// authoritative; only the recorded size and file count in metadata.json + /// may be stale. + /// + public string? BackupInventoryWarning { get; init; } } public sealed class BackupStorageInfo @@ -281,6 +325,8 @@ internal sealed class BackupMetadataFile public Dictionary? GlobalStateFiles { get; init; } public bool? GlobalStateFilePresent { get; init; } public bool? GlobalStateBackupFilePresent { get; init; } + public long? SizeBytes { get; init; } + public int? FileCount { get; init; } } internal sealed class SessionBackupManifest diff --git a/desktop/CodexProviderSync.Core/SessionRolloutService.cs b/desktop/CodexProviderSync.Core/SessionRolloutService.cs index bad86b9..2396166 100644 --- a/desktop/CodexProviderSync.Core/SessionRolloutService.cs +++ b/desktop/CodexProviderSync.Core/SessionRolloutService.cs @@ -1,4 +1,6 @@ using System.Buffers; +using System.Diagnostics; +using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; @@ -9,16 +11,26 @@ namespace CodexProviderSync.Core; public sealed class SessionRolloutService { private const string StatusOnlyProvider = "__status_only__"; - private const int ScanBufferSize = 1024 * 1024; - internal Func? ApplyFaultInjector { get; set; } + /// + /// Test seam invoked after a rollout's content digest is folded but before + /// its post-scan snapshot is taken, so a test can deterministically mutate + /// the file inside that window. + /// + internal Func? ScanFaultInjector { get; set; } + public async Task CollectSessionChangesAsync( string codexHome, string targetProvider, bool skipLockedReads = false, string? targetModel = null) { + long scanStarted = Stopwatch.GetTimestamp(); + int enumeratedRolloutFiles = 0; + int parsedSessionFiles = 0; + int contentScanPasses = 0; + int modelScanFiles = 0; List changes = []; List lockedPaths = []; List unreadablePaths = []; @@ -41,9 +53,12 @@ public async Task CollectSessionChangesAsync( .EnumerateFiles(rootDir, "rollout-*.jsonl", SearchOption.AllDirectories) .Order(StringComparer.Ordinal)) { + enumeratedRolloutFiles += 1; FirstLineRecord record; + FileSnapshot scanStart; try { + scanStart = GetFileSnapshot(rolloutPath); record = await ReadFirstLineRecordAsync(rolloutPath); } catch (Exception error) when (skipLockedReads && IsRolloutFileBusyError(error)) @@ -61,6 +76,7 @@ record = await ReadFirstLineRecordAsync(rolloutPath); { continue; } + parsedSessionFiles += 1; string currentProvider = payload!["model_provider"]?.GetValue() ?? "(missing)"; Dictionary bucket = dirName == "archived_sessions" ? archivedCounts : sessionCounts; @@ -72,14 +88,22 @@ record = await ReadFirstLineRecordAsync(rolloutPath); { threadCwdsById[metadataThreadId] = ToDesktopWorkspacePath(metadataCwd); } - bool hasEncryptedContent; + RolloutContentScan contentScan; try { - hasEncryptedContent = await FileHasEncryptedContentAsync(rolloutPath, record.FirstLine, record.Offset); - if (payload["id"]?.GetValue() is string threadId - && await FileHasUserEventAsync(rolloutPath, record.FirstLine, record.Offset)) + bool collectModels = !string.IsNullOrEmpty(targetModel); + bool providerMayChange = !string.Equals(targetProvider, StatusOnlyProvider, StringComparison.Ordinal) + && !string.Equals(currentProvider, targetProvider, StringComparison.Ordinal); + contentScan = await ScanRolloutContentAsync( + rolloutPath, + record, + collectModels, + collectFingerprint: providerMayChange); + contentScanPasses += 1; + modelScanFiles += collectModels ? 1 : 0; + if (ScanFaultInjector is not null) { - userEventThreadIds.Add(threadId); + await ScanFaultInjector(rolloutPath); } } catch (Exception error) when (skipLockedReads && IsRolloutFileBusyError(error)) @@ -93,7 +117,13 @@ record = await ReadFirstLineRecordAsync(rolloutPath); continue; } - if (hasEncryptedContent) + if (payload["id"]?.GetValue() is string threadId + && contentScan.HasUserEvent) + { + userEventThreadIds.Add(threadId); + } + + if (contentScan.HasEncryptedContent) { Dictionary encryptedBucket = dirName == "archived_sessions" ? encryptedArchivedCounts : encryptedSessionCounts; encryptedBucket[currentProvider] = encryptedBucket.TryGetValue(currentProvider, out int encryptedCount) ? encryptedCount + 1 : 1; @@ -106,31 +136,35 @@ record = await ReadFirstLineRecordAsync(rolloutPath); bool modelChanged = false; if (!string.IsNullOrEmpty(targetModel)) { - try - { - currentModelBackups = await ReadTurnContextModelBackupsAsync(rolloutPath, record); - currentModels = currentModelBackups - .SelectMany(static backup => backup.OriginalModels.Count > 0 - ? backup.OriginalModels - : [backup.OriginalModel]) - .ToArray(); - modelChanged = currentModels.Any(model => !string.Equals(model, targetModel, StringComparison.Ordinal)); - } - catch (Exception error) when (skipLockedReads && IsRolloutFileBusyError(error)) - { - lockedPaths.Add(rolloutPath); - continue; - } - catch (Exception error) when (skipLockedReads && IsRolloutFileUnreadableError(error)) - { - unreadablePaths.Add(rolloutPath); - continue; - } + currentModelBackups = contentScan.TurnContextModels; + currentModels = currentModelBackups + .SelectMany(static backup => backup.OriginalModels.Count > 0 + ? backup.OriginalModels + : [backup.OriginalModel]) + .ToArray(); + modelChanged = currentModels.Any(model => !string.Equals(model, targetModel, StringComparison.Ordinal)); } if (providerChanged || modelChanged) { FileSnapshot snapshot = GetFileSnapshot(rolloutPath); + if (contentScan.ContentFingerprint is not null && snapshot != scanStart) + { + // The rollout grew or was touched while we were folding + // its content digest, so the fingerprint describes a + // state that no longer exists and must not be cached as + // a plan hint. An active Codex session appending to its + // own rollout is the ordinary cause, so report it the + // same way a busy file is reported and keep rewriting + // the rest instead of aborting the whole sync. + if (skipLockedReads) + { + lockedPaths.Add(rolloutPath); + continue; + } + + throw new CoreWritePlanStaleException(); + } if (providerChanged) { payload["model_provider"] = targetProvider; @@ -148,7 +182,10 @@ record = await ReadFirstLineRecordAsync(rolloutPath); OriginalProvider = currentProvider, UpdatedFirstLine = providerChanged ? root!.ToJsonString() : record.FirstLine, ModelOnlyChange = !providerChanged && modelChanged, - OriginalTurnContextModels = currentModelBackups + OriginalTurnContextModels = currentModelBackups, + ContentFingerprint = contentScan.ContentFingerprint is null + ? null + : $"sha256:{contentScan.ContentFingerprint}:{snapshot.Length}:{snapshot.LastWriteTimeUtcTicks}" }); } } @@ -170,7 +207,15 @@ record = await ReadFirstLineRecordAsync(rolloutPath); ArchivedSessions = encryptedArchivedCounts }, UserEventThreadIds = userEventThreadIds, - ThreadCwdsById = threadCwdsById + ThreadCwdsById = threadCwdsById, + ScanMetrics = new SessionScanMetrics + { + EnumeratedRolloutFiles = enumeratedRolloutFiles, + ParsedSessionFiles = parsedSessionFiles, + ContentScanPasses = contentScanPasses, + ModelScanFiles = modelScanFiles, + DurationMs = (long)Math.Round(Stopwatch.GetElapsedTime(scanStarted).TotalMilliseconds) + } }; } @@ -463,12 +508,20 @@ private static async Task ReadFirstLineRecordAsync(FileStream s bool crlf = newlineIndex > 0 && current[newlineIndex - 1] == '\r'; int lineLength = crlf ? newlineIndex - 1 : newlineIndex; string firstLine = Encoding.UTF8.GetString(current[..lineLength]); - return new FirstLineRecord(firstLine, crlf ? "\r\n" : "\n", newlineIndex + 1); + return new FirstLineRecord( + firstLine, + crlf ? "\r\n" : "\n", + newlineIndex + 1, + current[..(newlineIndex + 1)].ToArray()); } } string text = Encoding.UTF8.GetString(collected.GetBuffer(), 0, (int)collected.Length); - return new FirstLineRecord(text, string.Empty, (int)collected.Length); + return new FirstLineRecord( + text, + string.Empty, + (int)collected.Length, + collected.GetBuffer().AsSpan(0, (int)collected.Length).ToArray()); } finally { @@ -499,27 +552,72 @@ private static async Task ReadFirstLineRecordAsync(FileStream s "\"model\"\\s*:\\s*\"((?:[^\"\\\\]|\\\\.)*)\"", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static async Task> ReadTurnContextModelBackupsAsync( + private static async Task ScanRolloutContentAsync( string rolloutPath, - FirstLineRecord record) + FirstLineRecord record, + bool collectModels, + bool collectFingerprint) { List backups = []; + bool hasEncryptedContent = record.FirstLine.Contains("encrypted_content", StringComparison.Ordinal); + bool hasUserEvent = false; + try + { + hasUserEvent = RecordHasUserEvent(JsonNode.Parse(record.FirstLine)); + } + catch + { + // Keep scanning the rest of the rollout below. + } try { await using FileStream stream = new( rolloutPath, FileMode.Open, FileAccess.Read, - FileShare.ReadWrite | FileShare.Delete); + FileShare.Read, + 64 * 1024, + FileOptions.Asynchronous | FileOptions.SequentialScan); stream.Seek(record.Offset, SeekOrigin.Begin); - using StreamReader reader = new(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: false, bufferSize: 4096, leaveOpen: true); + using IncrementalHash? contentHash = collectFingerprint + ? IncrementalHash.CreateHash(HashAlgorithmName.SHA256) + : null; + contentHash?.AppendData(record.PrefixBytes); + using HashingReadStream hashingStream = new(stream, contentHash); + using StreamReader reader = new( + hashingStream, + Encoding.UTF8, + detectEncodingFromByteOrderMarks: false, + bufferSize: 64 * 1024, + leaveOpen: true); int lineIndex = 1; string? line; while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null) { - if (!TurnContextTypeRegex.IsMatch(line)) + if (!hasEncryptedContent + && line.Contains("encrypted_content", StringComparison.Ordinal)) + { + hasEncryptedContent = true; + } + if (!hasUserEvent && !string.IsNullOrWhiteSpace(line)) + { + try + { + hasUserEvent = RecordHasUserEvent(JsonNode.Parse(line)); + } + catch + { + // Ignore malformed non-metadata lines; positive evidence is sufficient. + } + } + + if (!collectModels || !TurnContextTypeRegex.IsMatch(line)) { lineIndex += 1; + if (!collectModels && !collectFingerprint && hasEncryptedContent && hasUserEvent) + { + break; + } continue; } @@ -550,7 +648,14 @@ private static async Task> ReadTurnContext } lineIndex += 1; } - return backups; + string? contentFingerprint = contentHash is null + ? null + : Convert.ToHexString(contentHash.GetHashAndReset()).ToLowerInvariant(); + return new RolloutContentScan( + hasEncryptedContent, + hasUserEvent, + backups, + contentFingerprint); } catch (Exception error) when (IsRolloutFileBusyError(error)) { @@ -927,188 +1032,6 @@ private static FileSnapshot GetFileSnapshot(string filePath) return new FileSnapshot(fileInfo.Length, fileInfo.LastWriteTimeUtc.Ticks); } - private static async Task FileContainsTextAsync(string filePath, string text, int startOffset) - { - byte[] needle = Encoding.UTF8.GetBytes(text); - byte[] buffer = ArrayPool.Shared.Rent(ScanBufferSize); - byte[] tail = []; - - try - { - await using FileStream stream = new( - filePath, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - ScanBufferSize, - FileOptions.Asynchronous | FileOptions.SequentialScan); - - if (startOffset > 0) - { - stream.Seek(startOffset, SeekOrigin.Begin); - } - - while (true) - { - int bytesRead = await stream.ReadAsync(buffer.AsMemory(0, ScanBufferSize)); - if (bytesRead == 0) - { - return false; - } - - byte[] haystack = buffer; - int haystackLength = bytesRead; - if (tail.Length > 0) - { - haystackLength = tail.Length + bytesRead; - haystack = ArrayPool.Shared.Rent(haystackLength); - Buffer.BlockCopy(tail, 0, haystack, 0, tail.Length); - Buffer.BlockCopy(buffer, 0, haystack, tail.Length, bytesRead); - } - - try - { - if (ContainsNeedle(haystack, haystackLength, needle)) - { - return true; - } - - int keepBytes = Math.Min(Math.Max(0, needle.Length - 1), haystackLength); - if (keepBytes == 0) - { - tail = []; - } - else - { - tail = new byte[keepBytes]; - Buffer.BlockCopy(haystack, haystackLength - keepBytes, tail, 0, keepBytes); - } - } - finally - { - if (!ReferenceEquals(haystack, buffer)) - { - ArrayPool.Shared.Return(haystack); - } - } - } - } - catch (Exception error) - { - throw WrapRolloutFileBusyError(error, filePath, "scan"); - } - finally - { - ArrayPool.Shared.Return(buffer); - } - } - - private static bool ContainsNeedle(byte[] haystack, int haystackLength, byte[] needle) - { - if (needle.Length == 0) - { - return true; - } - - if (haystackLength < needle.Length) - { - return false; - } - - int lastStart = haystackLength - needle.Length; - for (int index = 0; index <= lastStart; index += 1) - { - bool match = true; - for (int needleIndex = 0; needleIndex < needle.Length; needleIndex += 1) - { - if (haystack[index + needleIndex] != needle[needleIndex]) - { - match = false; - break; - } - } - - if (match) - { - return true; - } - } - - return false; - } - - private static async Task FileHasEncryptedContentAsync(string filePath, string firstLine, int startOffset) - { - if (firstLine.Contains("encrypted_content", StringComparison.Ordinal)) - { - return true; - } - - return await FileContainsTextAsync(filePath, "encrypted_content", startOffset); - } - - private static async Task FileHasUserEventAsync(string filePath, string firstLine, int startOffset) - { - try - { - if (RecordHasUserEvent(JsonNode.Parse(firstLine))) - { - return true; - } - } - catch - { - // Keep scanning the rest of the rollout below. - } - - try - { - await using FileStream stream = new( - filePath, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - 64 * 1024, - FileOptions.Asynchronous | FileOptions.SequentialScan); - if (startOffset > 0) - { - stream.Seek(startOffset, SeekOrigin.Begin); - } - - using StreamReader reader = new( - stream, - Encoding.UTF8, - detectEncodingFromByteOrderMarks: true, - bufferSize: 64 * 1024, - leaveOpen: false); - while (await reader.ReadLineAsync() is string rawLine) - { - if (string.IsNullOrWhiteSpace(rawLine)) - { - continue; - } - - try - { - if (RecordHasUserEvent(JsonNode.Parse(rawLine))) - { - return true; - } - } - catch - { - // Ignore malformed non-metadata lines; provider sync only needs positive evidence. - } - } - - return false; - } - catch (Exception error) - { - throw WrapRolloutFileBusyError(error, filePath, "scan"); - } - } - private static bool RecordHasUserEvent(JsonNode? record) { if (record is not JsonObject root) @@ -1250,8 +1173,77 @@ private static Exception WrapRolloutFileBusyError(Exception error, string filePa error); } - private readonly record struct FirstLineRecord(string FirstLine, string Separator, int Offset); + private readonly record struct FirstLineRecord( + string FirstLine, + string Separator, + int Offset, + byte[] PrefixBytes); private readonly record struct FileSnapshot(long Length, long LastWriteTimeUtcTicks); + private readonly record struct RolloutContentScan( + bool HasEncryptedContent, + bool HasUserEvent, + IReadOnlyList TurnContextModels, + string? ContentFingerprint); + + private sealed class HashingReadStream(Stream inner, IncrementalHash? hash) : Stream + { + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => false; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + public override void Flush() => inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) + { + int read = inner.Read(buffer, offset, count); + if (read > 0) + { + hash?.AppendData(buffer, offset, read); + } + return read; + } + public override int Read(Span buffer) + { + int read = inner.Read(buffer); + if (read > 0) + { + hash?.AppendData(buffer[..read]); + } + return read; + } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken) + { + int read = await inner.ReadAsync(buffer.AsMemory(offset, count), cancellationToken); + if (read > 0) + { + hash?.AppendData(buffer, offset, read); + } + return read; + } + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default) + { + int read = await inner.ReadAsync(buffer, cancellationToken); + if (read > 0) + { + hash?.AppendData(buffer.Span[..read]); + } + return read; + } + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + protected override void Dispose(bool disposing) + { + // The caller owns the underlying rollout stream. + base.Dispose(disposing); + } + } private readonly record struct ModelLineRewrite( string Line, bool Replaced, diff --git a/desktop/CodexProviderSync.Core/TextFormatter.cs b/desktop/CodexProviderSync.Core/TextFormatter.cs index 291d9d9..b58da7c 100644 --- a/desktop/CodexProviderSync.Core/TextFormatter.cs +++ b/desktop/CodexProviderSync.Core/TextFormatter.cs @@ -27,6 +27,34 @@ public static string FormatSyncResult(SyncResult result, string label, string la : FormatSyncResultEnglish(result, label); } + public static string FormatPerformanceMetrics(SyncPerformanceMetrics metrics, string language) + { + bool chinese = IsChinese(language); + return chinese + ? $"性能: 总计={metrics.TotalDurationMs}ms; 准备={metrics.PreparationDurationMs}ms; " + + $"计划校验={metrics.CheckedPlanValidationDurationMs}ms; 备份={metrics.BackupDurationMs}ms; " + + $"写入={metrics.MutationDurationMs}ms; 清理={metrics.PruneDurationMs}ms; " + + $"rollout={metrics.RolloutScan.EnumeratedRolloutFiles}; 内容扫描={metrics.RolloutScan.ContentScanPasses}; " + + $"journal全量校验={metrics.JournalFullValidationCount}" + : $"Performance: total={metrics.TotalDurationMs}ms; prepare={metrics.PreparationDurationMs}ms; " + + $"plan-check={metrics.CheckedPlanValidationDurationMs}ms; backup={metrics.BackupDurationMs}ms; " + + $"mutation={metrics.MutationDurationMs}ms; prune={metrics.PruneDurationMs}ms; " + + $"rollouts={metrics.RolloutScan.EnumeratedRolloutFiles}; content-scans={metrics.RolloutScan.ContentScanPasses}; " + + $"journal-full-validations={metrics.JournalFullValidationCount}"; + } + + public static string FormatPerformanceMetrics(StatusPerformanceMetrics metrics, string language) + { + bool chinese = IsChinese(language); + return chinese + ? $"刷新性能: 总计={metrics.TotalDurationMs}ms; rollout扫描={metrics.RolloutScanDurationMs}ms; " + + $"备份统计={metrics.BackupSummaryDurationMs}ms; rollout={metrics.RolloutScan.EnumeratedRolloutFiles}; " + + $"内容扫描={metrics.RolloutScan.ContentScanPasses}" + : $"Refresh performance: total={metrics.TotalDurationMs}ms; rollout-scan={metrics.RolloutScanDurationMs}ms; " + + $"backup-summary={metrics.BackupSummaryDurationMs}ms; rollouts={metrics.RolloutScan.EnumeratedRolloutFiles}; " + + $"content-scans={metrics.RolloutScan.ContentScanPasses}"; + } + public static string FormatRestoreResult(RestoreResult result) => FormatRestoreResult(result, English); @@ -55,6 +83,13 @@ public static string FormatRestoreResult(RestoreResult result, string language) : $"Backup created at: {result.CreatedAt:O}"); } + if (!string.IsNullOrWhiteSpace(result.BackupInventoryWarning)) + { + lines.Add(IsChinese(language) + ? $"备份清单警告: {result.BackupInventoryWarning}" + : $"Backup inventory warning: {result.BackupInventoryWarning}"); + } + return string.Join(Environment.NewLine, lines); } diff --git a/desktop/CodexProviderSync.Core/TransactionJournalService.cs b/desktop/CodexProviderSync.Core/TransactionJournalService.cs index 6f2a94c..2c4e608 100644 --- a/desktop/CodexProviderSync.Core/TransactionJournalService.cs +++ b/desktop/CodexProviderSync.Core/TransactionJournalService.cs @@ -3,7 +3,7 @@ namespace CodexProviderSync.Core; -internal sealed class FileTransactionJournal +internal sealed class FileTransactionJournal : IAsyncDisposable { internal const string FileName = "transaction-journal.jsonl"; private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); @@ -11,20 +11,46 @@ internal sealed class FileTransactionJournal private readonly string _filePath; private readonly string _operationId; private readonly SemaphoreSlim _appendGate = new(1, 1); + private FileStream? _writerLease; private int _sequence; + private PendingTransactionInfo? _current; + private long _expectedLength; + private byte[] _expectedTailRecord = []; + private bool _disposed; - private FileTransactionJournal(string filePath, string operationId, int sequence = 0) + private FileTransactionJournal( + string filePath, + string operationId, + PendingTransactionInfo? current = null, + FileStream? writerLease = null) { _filePath = filePath; _operationId = operationId; - _sequence = sequence; + _current = current; + _writerLease = writerLease; + _sequence = current?.LastSequence ?? 0; + _expectedLength = current is null ? 0 : -1; } internal string FilePath => _filePath; + internal int AppendFullJournalValidationCount { get; private set; } + internal Func? AppendFaultInjector { get; set; } - internal Task ReadCurrentInfoAsync() => ReadInfoAsync(_filePath); + internal async Task ReadCurrentInfoAsync() + { + await _appendGate.WaitAsync(); + try + { + ObjectDisposedException.ThrowIf(_disposed, this); + return (await ReadJournalForCurrentInstanceAsync()).Info; + } + finally + { + _appendGate.Release(); + } + } internal static async Task CreateAsync( string backupDir, @@ -32,23 +58,62 @@ internal static async Task CreateAsync( string targetProvider, IEnumerable potentialTargets) { + return await CreateCoreAsync( + backupDir, + codexHome, + targetProvider, + potentialTargets, + acquireWriterLease: false); + } + + internal static async Task CreateOwnedAsync( + string backupDir, + string codexHome, + string targetProvider, + IEnumerable potentialTargets) + { + return await CreateCoreAsync( + backupDir, + codexHome, + targetProvider, + potentialTargets, + acquireWriterLease: OperatingSystem.IsWindows()); + } + + private static async Task CreateCoreAsync( + string backupDir, + string codexHome, + string targetProvider, + IEnumerable potentialTargets, + bool acquireWriterLease) + { + string filePath = Path.Combine(backupDir, FileName); string operationId = Guid.NewGuid().ToString("D"); - FileTransactionJournal journal = new( - Path.Combine(backupDir, FileName), - operationId); - await journal.AppendAsync("prepared", new Dictionary - { - ["protocolVersion"] = 1, - ["backupDir"] = Path.GetFullPath(backupDir), - ["codexHome"] = Path.GetFullPath(codexHome), - ["targetProvider"] = targetProvider, - ["potentialTargets"] = potentialTargets - .Select(Path.GetFullPath) - .Distinct(PathComparer) - .Order(PathComparer) - .ToArray() - }); - return journal; + FileStream? writerLease = acquireWriterLease + ? OpenWriterLease(filePath, FileMode.CreateNew) + : null; + FileTransactionJournal journal = new(filePath, operationId, writerLease: writerLease); + try + { + await journal.AppendAsync("prepared", new Dictionary + { + ["protocolVersion"] = 1, + ["backupDir"] = Path.GetFullPath(backupDir), + ["codexHome"] = Path.GetFullPath(codexHome), + ["targetProvider"] = targetProvider, + ["potentialTargets"] = potentialTargets + .Select(Path.GetFullPath) + .Distinct(PathComparer) + .Order(PathComparer) + .ToArray() + }); + return journal; + } + catch + { + await journal.DisposeAsync(); + throw; + } } internal Task ApplyingAsync(string kind, string targetPath) => AppendAsync( @@ -96,29 +161,9 @@ private async Task AppendAsync(string state, IReadOnlyDictionary 0) - { - before = await ReadInfoAsync(_filePath); - if (before.InvalidTail) - { - throw new InvalidOperationException( - state == "committed" - ? $"Transaction journal is invalid and cannot commit until recovery: {_filePath}" - : $"Transaction journal is invalid and requires recovery before append: {_filePath}"); - } - if (!string.Equals(before.OperationId, _operationId, StringComparison.Ordinal)) - { - throw new InvalidOperationException( - $"Transaction journal operationId changed before append: {_filePath}"); - } - _sequence = before.LastSequence; - } - else if (_sequence != 0) - { - throw new InvalidOperationException( - $"Transaction journal disappeared after it was created: {_filePath}"); - } + ObjectDisposedException.ThrowIf(_disposed, this); + bool terminal = state is "committed" or "rolledBack"; + PendingTransactionInfo? before = await EnsureJournalFrontierAsync(forceFullValidation: terminal); ValidateAppendTransition(before, state, details); int nextSequence = _sequence + 1; @@ -141,20 +186,38 @@ private async Task AppendAsync(string state, IReadOnlyDictionary EnsureJournalFrontierAsync(bool forceFullValidation) + { + if (!File.Exists(_filePath)) + { + if (_sequence != 0 || _current is not null) + { + throw new InvalidOperationException( + $"Transaction journal disappeared after it was created: {_filePath}"); + } + _expectedLength = 0; + _expectedTailRecord = []; + return null; + } + + long actualLength = new FileInfo(_filePath).Length; + if (actualLength == 0 && _sequence == 0 && _current is null) + { + _expectedLength = 0; + _expectedTailRecord = []; + return null; + } + bool frontierMatches = _writerLease is not null + && OperatingSystem.IsWindows() + && !forceFullValidation + && actualLength == _expectedLength + && await TailMatchesAsync(actualLength, _expectedTailRecord); + if (!frontierMatches) + { + JournalReadResult read = await ReadJournalForAppendAsync(); + AdoptJournalState(read); + } + + if (_current is null) + { + return null; + } + if (_current.InvalidTail) + { + throw new InvalidOperationException( + forceFullValidation + ? $"Transaction journal is invalid and cannot commit until recovery: {_filePath}" + : $"Transaction journal is invalid and requires recovery before append: {_filePath}"); + } + if (!string.Equals(_current.OperationId, _operationId, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Transaction journal operationId changed before append: {_filePath}"); + } + _sequence = _current.LastSequence; + return _current; + } + + private async Task ReadJournalForAppendAsync() + { + AppendFullJournalValidationCount += 1; + return await ReadJournalForCurrentInstanceAsync(); + } + + private Task ReadJournalForCurrentInstanceAsync() + { + return _writerLease is null + ? ReadJournalAsync(_filePath) + : ReadJournalAsync(_filePath, _writerLease); + } + + private void AdoptJournalState(JournalReadResult read) + { + _current = read.Info; + _sequence = read.Info.LastSequence; + _expectedLength = _writerLease?.Length ?? new FileInfo(_filePath).Length; + _expectedTailRecord = read.ValidLines.Count == 0 + ? [] + : Encoding.UTF8.GetBytes(read.ValidLines[^1] + "\n"); + } + + private async Task VerifyAppendedRecordAsync(long appendOffset, byte[] expectedRecord) + { + if (_writerLease is not null) + { + long leasedExpectedLength = appendOffset + expectedRecord.Length; + if (_writerLease.Length != leasedExpectedLength) + { + throw new InvalidOperationException( + $"Transaction journal changed while appending: {_filePath}"); + } + _writerLease.Seek(appendOffset, SeekOrigin.Begin); + byte[] leasedActual = new byte[expectedRecord.Length]; + await _writerLease.ReadExactlyAsync(leasedActual); + if (!leasedActual.AsSpan().SequenceEqual(expectedRecord)) + { + throw new InvalidOperationException( + $"Transaction journal append bytes could not be verified: {_filePath}"); + } + return; + } + + await using FileStream stream = new( + _filePath, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + 4096, + FileOptions.Asynchronous | FileOptions.SequentialScan); + long expectedLength = appendOffset + expectedRecord.Length; + if (stream.Length != expectedLength) + { + throw new InvalidOperationException( + $"Transaction journal changed while appending: {_filePath}"); + } + + stream.Seek(appendOffset, SeekOrigin.Begin); + byte[] actual = new byte[expectedRecord.Length]; + await stream.ReadExactlyAsync(actual); + if (!actual.AsSpan().SequenceEqual(expectedRecord)) + { + throw new InvalidOperationException( + $"Transaction journal append bytes could not be verified: {_filePath}"); + } + } + + private async Task TailMatchesAsync(long actualLength, byte[] expectedTail) + { + if (actualLength == 0) + { + return expectedTail.Length == 0; + } + if (expectedTail.Length == 0 || actualLength < expectedTail.Length) + { + return false; + } + + if (_writerLease is not null) + { + if (_writerLease.Length != actualLength) + { + return false; + } + _writerLease.Seek(actualLength - expectedTail.Length, SeekOrigin.Begin); + byte[] leasedTail = new byte[expectedTail.Length]; + await _writerLease.ReadExactlyAsync(leasedTail); + return leasedTail.AsSpan().SequenceEqual(expectedTail); + } + + await using FileStream stream = new( + _filePath, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + 4096, + FileOptions.Asynchronous | FileOptions.SequentialScan); + if (stream.Length != actualLength) + { + return false; + } + stream.Seek(actualLength - expectedTail.Length, SeekOrigin.Begin); + byte[] actualTail = new byte[expectedTail.Length]; + await stream.ReadExactlyAsync(actualTail); + return actualTail.AsSpan().SequenceEqual(expectedTail); + } + + public async ValueTask DisposeAsync() + { + await _appendGate.WaitAsync(); + try + { + if (_disposed) + { + return; + } + _disposed = true; + FileStream? writerLease = Interlocked.Exchange(ref _writerLease, null); + if (writerLease is not null) + { + await writerLease.DisposeAsync(); + } + } + finally + { + _appendGate.Release(); + } + } + + private static FileStream OpenWriterLease(string filePath, FileMode mode) + { + return new FileStream( + filePath, + mode, + FileAccess.ReadWrite, + FileShare.Read, + 4096, + FileOptions.Asynchronous | FileOptions.WriteThrough | FileOptions.SequentialScan); + } + + private PendingTransactionInfo AdvanceJournalState( + PendingTransactionInfo? current, + string nextState, + IReadOnlyDictionary? details, + int nextSequence) + { + IReadOnlyList potentialTargets = current?.PotentialTargets + ?? ReadPreparedPotentialTargets(details); + Dictionary affected = new(PathComparer); + if (current is not null) + { + foreach (TransactionTargetInfo target in current.AffectedTargets) + { + affected[target.Kind + "\0" + Path.GetFullPath(target.TargetPath)] = target; + } + } + + if (nextState is "applying" or "applied" or "skipped") + { + string kind = ReadRequiredDetail(details, "kind"); + string targetPath = Path.GetFullPath(ReadRequiredDetail(details, "targetPath")); + string key = kind + "\0" + targetPath; + if (nextState == "skipped") + { + affected.Remove(key); + } + else + { + affected[key] = new TransactionTargetInfo(kind, targetPath, nextState); + } + } + + bool terminal = nextState is "committed" or "rolledBack"; + string journalPath = current?.JournalPath ?? _filePath; + return new PendingTransactionInfo( + journalPath, + current?.BackupDir ?? Path.GetDirectoryName(_filePath)!, + current?.OperationId ?? _operationId, + nextSequence, + nextState, + terminal, + InvalidTail: false, + LastValidState: nextState, + potentialTargets, + affected.Values.ToArray()); + } + + private static IReadOnlyList ReadPreparedPotentialTargets( + IReadOnlyDictionary? details) + { + if (details is not null + && details.TryGetValue("potentialTargets", out object? value) + && value is IEnumerable targets) + { + return targets + .Select(Path.GetFullPath) + .Distinct(PathComparer) + .Order(PathComparer) + .ToArray(); + } + return []; + } + private static void ValidateAppendTransition( PendingTransactionInfo? current, string nextState, @@ -436,7 +786,7 @@ await replacement.RollingBackAsync( FileTransactionJournal journal = new( journalPath, info.OperationId!, - info.LastSequence); + info); if (info.State is not ("rollingBack" or "recoveryRequired")) { await journal.RollingBackAsync(new InvalidOperationException("Explicit managed-backup restore")); @@ -465,6 +815,23 @@ internal static async Task ReadInfoAsync(string journalP private static async Task ReadJournalAsync(string journalPath) { byte[] journalBytes = await File.ReadAllBytesAsync(journalPath); + return ParseJournal(journalPath, journalBytes); + } + + private static async Task ReadJournalAsync(string journalPath, FileStream stream) + { + if (stream.Length > int.MaxValue) + { + throw new InvalidOperationException($"Transaction journal is too large to validate: {journalPath}"); + } + byte[] journalBytes = new byte[(int)stream.Length]; + stream.Seek(0, SeekOrigin.Begin); + await stream.ReadExactlyAsync(journalBytes); + return ParseJournal(journalPath, journalBytes); + } + + private static JournalReadResult ParseJournal(string journalPath, byte[] journalBytes) + { bool missingTerminalLf = journalBytes.Length > 0 && journalBytes[^1] != (byte)'\n'; string journalText = Encoding.UTF8.GetString(journalBytes); string? operationId = null; diff --git a/src/backup.js b/src/backup.js index 8d0ca4c..2fa3606 100644 --- a/src/backup.js +++ b/src/backup.js @@ -322,26 +322,18 @@ export async function createBackup({ "utf8" ); - await writeFileAtomic( - path.join(backupDir, "metadata.json"), - JSON.stringify( - { - version: 2, - namespace: BACKUP_NAMESPACE, - codexHome, - sqliteHome: actualSqliteHome, - targetProvider, - createdAt: sessionManifest.createdAt, - dbFiles: copiedDbFiles, - sqliteDbFiles: copiedSqliteDbFiles, - globalStateFiles, - changedSessionFiles: sessionChanges.length - }, - null, - 2 - ), - "utf8" - ); + await writeMetadataWithInventory(backupDir, { + version: 2, + namespace: BACKUP_NAMESPACE, + codexHome, + sqliteHome: actualSqliteHome, + targetProvider, + createdAt: sessionManifest.createdAt, + dbFiles: copiedDbFiles, + sqliteDbFiles: copiedSqliteDbFiles, + globalStateFiles, + changedSessionFiles: sessionChanges.length + }); return backupDir; } @@ -382,12 +374,19 @@ export async function updateSessionBackupManifest(backupDir, sessionChanges, opt "utf8", { faultInjector: options.faultInjector } ); - await writeFileAtomic( - metadataPath, - JSON.stringify(metadata, null, 2), - "utf8", - { faultInjector: options.faultInjector } - ); + await writeMetadataWithInventory(backupDir, metadata, { + faultInjector: options.faultInjector + }); +} + +export async function refreshBackupInventory(backupDir, options = {}) { + const normalizedBackupDir = path.resolve(backupDir); + const metadataPath = path.join(normalizedBackupDir, "metadata.json"); + const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); + if (metadata?.namespace !== BACKUP_NAMESPACE || !new Set([1, 2]).has(metadata.version)) { + throw new Error(`Unsupported backup metadata in ${metadataPath}.`); + } + await writeMetadataWithInventory(normalizedBackupDir, metadata, options); } export async function getBackupSummary(codexHome) { @@ -395,7 +394,7 @@ export async function getBackupSummary(codexHome) { const backupDirs = await listManagedBackupDirectories(backupRoot); let totalBytes = 0; for (const entry of backupDirs) { - totalBytes += await getDirectorySize(entry.fullPath); + totalBytes += await getBackupDirectorySize(entry.fullPath); } return { @@ -420,7 +419,7 @@ export async function pruneBackups(codexHome, keepCount = DEFAULT_BACKUP_RETENTI .filter((entry) => !protectedBackups.has(pathComparisonKey(entry.fullPath))); let freedBytes = 0; for (const entry of toDelete) { - freedBytes += await getDirectorySize(entry.fullPath); + freedBytes += await getBackupDirectorySize(entry.fullPath); await fs.rm(entry.fullPath, { recursive: true, force: true }); } @@ -715,29 +714,77 @@ async function isManagedBackupDirectory(backupDir) { } } +async function writeMetadataWithInventory(backupDir, metadata, options = {}) { + const metadataPath = path.join(backupDir, "metadata.json"); + const payload = await getDirectoryInventory(backupDir, metadataPath); + const fileCount = payload.fileCount + 1; + let sizeBytes = 0; + let serialized = ""; + for (let attempt = 0; attempt < 8; attempt += 1) { + serialized = JSON.stringify({ ...metadata, sizeBytes, fileCount }, null, 2); + const nextSizeBytes = payload.sizeBytes + Buffer.byteLength(serialized, "utf8"); + if (nextSizeBytes === sizeBytes) { + await writeFileAtomic(metadataPath, serialized, "utf8", options); + return; + } + sizeBytes = nextSizeBytes; + } + throw new Error(`Backup metadata inventory did not converge: ${metadataPath}`); +} + +async function getBackupDirectorySize(backupDir) { + try { + const metadata = JSON.parse( + await fs.readFile(path.join(backupDir, "metadata.json"), "utf8") + ); + if (metadata?.namespace === BACKUP_NAMESPACE + && Number.isSafeInteger(metadata.sizeBytes) + && metadata.sizeBytes >= 0 + && Number.isSafeInteger(metadata.fileCount) + && metadata.fileCount >= 1) { + return metadata.sizeBytes; + } + } catch { + // Older or damaged inventory fields fall back to the recursive scan below. + } + return getDirectorySize(backupDir); +} + async function getDirectorySize(directoryPath) { + return (await getDirectoryInventory(directoryPath)).sizeBytes; +} + +async function getDirectoryInventory(directoryPath, excludedFilePath = null) { let entries; try { entries = await fs.readdir(directoryPath, { withFileTypes: true }); } catch (error) { if (error?.code === "ENOENT") { - return 0; + return { sizeBytes: 0, fileCount: 0 }; } throw error; } - let total = 0; + let sizeBytes = 0; + let fileCount = 0; + const excluded = excludedFilePath === null ? null : path.resolve(excludedFilePath); for (const entry of entries) { const fullPath = path.join(directoryPath, entry.name); if (entry.isDirectory()) { - total += await getDirectorySize(fullPath); + const child = await getDirectoryInventory(fullPath, excluded); + sizeBytes += child.sizeBytes; + fileCount += child.fileCount; continue; } if (entry.isFile()) { + if (excluded !== null && path.resolve(fullPath) === excluded) { + continue; + } const stat = await fs.stat(fullPath); - total += stat.size; + sizeBytes += stat.size; + fileCount += 1; } } - return total; + return { sizeBytes, fileCount }; } diff --git a/src/cli.js b/src/cli.js index 836253a..8052edb 100644 --- a/src/cli.js +++ b/src/cli.js @@ -328,6 +328,9 @@ async function main() { console.log(`Restored backup from ${path.resolve(backupDir)}`); console.log(`Codex home: ${result.codexHome}`); console.log(`Provider at backup time: ${result.targetProvider}`); + if (result.backupInventoryWarning) { + console.log(`Backup inventory warning: ${result.backupInventoryWarning}`); + } return; } diff --git a/src/service.js b/src/service.js index eb1c7e4..82bc191 100644 --- a/src/service.js +++ b/src/service.js @@ -22,6 +22,7 @@ import { getBackupRecoveryCoverage, getBackupSummary, pruneBackups, + refreshBackupInventory, restoreBackup, restoreGlobalStateFilesFromBackup } from "./backup.js"; @@ -200,6 +201,20 @@ async function commitJournalWithReconciliation(journal, faultInjector) { throw new Error(`Transaction journal did not persist a valid committed terminal state: ${journal.filePath}`); } +// Rewrites the retained backup's recorded size and file count after the journal +// reached a terminal state, so status and pruning do not trust an inventory +// captured before those journal records existed. Used on the rollback paths, +// where the caller is already reporting a failure: a bookkeeping problem here +// must never replace the original error. +async function tryRefreshBackupInventory(backupDir) { + try { + await refreshBackupInventory(backupDir); + } catch { + // The original sync failure and its rollback details are the authoritative + // diagnosis and must reach the caller unchanged. + } +} + async function rollbackJournalWithReconciliation(journal, faultInjector) { let acknowledgementError = null; try { @@ -643,6 +658,16 @@ async function runSyncCore({ await faultInjector?.({ point: "before_transaction_commit", completedCount: completedTargets.length }); await commitJournalWithReconciliation(journal, faultInjector); transactionCommitted = true; + // The transaction is committed and every target is on disk. Refreshing the + // inventory only corrects the recorded size and file count in + // metadata.json, so a failure must degrade to a warning: throwing would + // report a successful sync as failed and skip the pruning below. + let backupInventoryWarning = null; + try { + await refreshBackupInventory(backupDir); + } catch (inventoryError) { + backupInventoryWarning = `Backup inventory refresh failed: ${inventoryError instanceof Error ? inventoryError.message : String(inventoryError)}`; + } await faultInjector?.({ point: "after_transaction_commit", completedCount: completedTargets.length }); let autoPruneResult = null; let autoPruneWarning = null; @@ -662,6 +687,10 @@ async function runSyncCore({ deletedCount: autoPruneResult?.deletedCount ?? 0, warning: autoPruneWarning }); + autoPruneWarning = [backupInventoryWarning, autoPruneWarning] + .filter((part) => typeof part === "string" && part.trim().length > 0) + .map((part) => part.trim()) + .join(" | ") || null; const result = { codexHome, sqliteHome: storage.sqliteHome, @@ -795,6 +824,7 @@ async function runSyncCore({ // Preserve the original and rollback errors even if the journal is // no longer writable. } + await tryRefreshBackupInventory(backupDir); const persistedCompletedTargets = journalSnapshot ? uniqueResolvedPaths([...getAppliedJournalTargets(journalSnapshot), ...completedTargets]) : uniqueResolvedPaths(completedTargets); @@ -813,6 +843,7 @@ async function runSyncCore({ { rollbackStatus: "incomplete", recoveryRequired: true } ); } + await tryRefreshBackupInventory(backupDir); const persistedCompletedTargets = journalSnapshot ? uniqueResolvedPaths([...getAppliedJournalTargets(journalSnapshot), ...completedTargets]) : uniqueResolvedPaths(completedTargets); @@ -937,7 +968,8 @@ export async function runRestore({ restoreDatabase = true, restoreSessions = true, allowSqliteHomeRelocation = false, - platform + platform, + faultInjector }) { if (!backupDir) { throw new Error("Missing backup path. Usage: codex-provider restore "); @@ -1011,6 +1043,17 @@ export async function runRestore({ allowSqliteHomeRelocation }); await markBackupTransactionRolledBack(normalizedBackupDir); + // The restore and its journal marker are already durable. Refreshing the + // inventory only corrects metadata.json bookkeeping, so surface a failure as + // a warning instead of reporting a completed restore as failed. + try { + await refreshBackupInventory(normalizedBackupDir, { faultInjector }); + } catch (inventoryError) { + return { + ...result, + backupInventoryWarning: `Backup inventory refresh failed: ${inventoryError instanceof Error ? inventoryError.message : String(inventoryError)}` + }; + } return result; } finally { await releaseLock(); diff --git a/test/sync-service.test.js b/test/sync-service.test.js index 2f7d2b5..d69163a 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -492,6 +492,37 @@ test("unfinished journal blocks writes until the bound backup is restored", asyn assert.equal(result.targetProvider, "openai"); }); +test("runRestore reports a warning instead of failing when the inventory refresh fails", async () => { + const { codexHome } = await makeTempCodexHome(); + await writeConfig(codexHome, 'model_provider = "openai"'); + const sessionPath = path.join(codexHome, "sessions", "2026", "03", "19", "rollout-a.jsonl"); + await writeRollout(sessionPath, "thread-a", "apigather"); + await writeStateDb(codexHome, [ + { id: "thread-a", model_provider: "apigather", archived: false } + ]); + + const synced = await runSync({ codexHome }); + const backupDir = synced.backupDir; + + // Fail only the write the post-restore inventory refresh performs. The + // injected seam is deterministic regardless of the user the suite runs as, + // which file permission bits are not: root bypasses them entirely. + const result = await runRestore({ + backupDir, + codexHome, + faultInjector: ({ point }) => { + if (point === "before_atomic_replace") { + throw new Error("injected inventory write failure"); + } + } + }); + + // The restore itself is authoritative and must still be reported as done. + assert.equal(result.targetProvider, "openai"); + assert.match(result.backupInventoryWarning, /Backup inventory refresh failed/); + assert.match(await fs.readFile(sessionPath, "utf8"), /"model_provider":"apigather"/); +}); + test("crash recovery restores actually mutated rollout and database from a pending journal", async () => { const { codexHome } = await makeTempCodexHome(); await writeConfig(codexHome, 'model_provider = "openai"'); @@ -724,7 +755,7 @@ test("repeated sync is idempotent for rollout and SQLite state", async () => { assert.deepEqual(await findPendingTransactions(codexHome), []); }); -test("runSync leaves the backup manifest and metadata unchanged after creation", async () => { +test("runSync leaves the backup manifest and metadata payload unchanged after creation", async () => { const { codexHome } = await makeTempCodexHome(); await writeConfig(codexHome, 'model_provider = "openai"'); const sessionPath = path.join(codexHome, "sessions", "2026", "03", "19", "rollout-immutable-backup.jsonl"); @@ -753,7 +784,27 @@ test("runSync leaves the backup manifest and metadata unchanged after creation", assert.ok(manifestBefore); assert.ok(metadataBefore); assert.deepEqual(await fs.readFile(path.join(backupDir, "session-meta-backup.json")), manifestBefore); - assert.deepEqual(await fs.readFile(path.join(backupDir, "metadata.json")), metadataBefore); + const metadataBeforeValue = JSON.parse(metadataBefore.toString("utf8")); + const metadataAfterValue = JSON.parse( + await fs.readFile(path.join(backupDir, "metadata.json"), "utf8") + ); + const { + sizeBytes: sizeBytesBefore, + fileCount: fileCountBefore, + ...metadataPayloadBefore + } = metadataBeforeValue; + const { + sizeBytes: sizeBytesAfter, + fileCount: fileCountAfter, + ...metadataPayloadAfter + } = metadataAfterValue; + assert.deepEqual(metadataPayloadAfter, metadataPayloadBefore); + assert.ok(sizeBytesAfter > sizeBytesBefore); + assert.equal(fileCountAfter, fileCountBefore + 1); + assert.deepEqual( + { sizeBytes: sizeBytesAfter, fileCount: fileCountAfter }, + await getDirectoryInventory(backupDir) + ); const manifest = JSON.parse(manifestBefore.toString("utf8")); assert.equal(manifest.appliedPaths, null); }); @@ -1416,6 +1467,24 @@ async function writeBackup(codexHome, directoryName, files) { return totalBytes; } +async function getDirectoryInventory(directoryPath) { + const entries = await fs.readdir(directoryPath, { withFileTypes: true }); + let sizeBytes = 0; + let fileCount = 0; + for (const entry of entries) { + const fullPath = path.join(directoryPath, entry.name); + if (entry.isDirectory()) { + const child = await getDirectoryInventory(fullPath); + sizeBytes += child.sizeBytes; + fileCount += child.fileCount; + } else if (entry.isFile()) { + sizeBytes += (await fs.stat(fullPath)).size; + fileCount += 1; + } + } + return { sizeBytes, fileCount }; +} + async function writeConfig(codexHome, modelProviderLine = "") { const config = `${modelProviderLine}${modelProviderLine ? "\n" : ""}sandbox_mode = "danger-full-access"\n\n[model_providers.apigather]\nbase_url = "https://example.com"\n`; await fs.writeFile(path.join(codexHome, "config.toml"), config, "utf8"); @@ -1637,6 +1706,13 @@ test("runSync rewrites rollout files and sqlite, then restore reverts both", asy assert.equal(backupMetadata.version, 2); assert.equal(backupMetadata.sqliteHome, path.join(codexHome, SQLITE_DIR_BASENAME)); assert.deepEqual(backupMetadata.sqliteDbFiles, [DB_FILE_BASENAME]); + assert.ok(Number.isSafeInteger(backupMetadata.sizeBytes)); + assert.ok(backupMetadata.sizeBytes > 0); + assert.ok(Number.isSafeInteger(backupMetadata.fileCount)); + assert.ok(backupMetadata.fileCount > 0); + const backupInventory = await getDirectoryInventory(syncResult.backupDir); + assert.equal(backupMetadata.sizeBytes, backupInventory.sizeBytes); + assert.equal(backupMetadata.fileCount, backupInventory.fileCount); assert.deepEqual( backupMetadata.dbFiles.map((fileName) => fileName.replaceAll("\\", "/")), ["sqlite/state_5.sqlite"] @@ -3609,6 +3685,33 @@ test("pruneBackups removes the oldest backup directories", async () => { await fs.access(path.join(backupRoot(codexHome), "20260321T000000000Z")); }); +test("backup summary and prune use cached inventory with legacy fallback", async () => { + const { codexHome } = await makeTempCodexHome(); + const cachedDir = path.join(backupRoot(codexHome), "20260319T000000000Z"); + const legacyDir = path.join(backupRoot(codexHome), "20260320T000000000Z"); + await fs.mkdir(cachedDir, { recursive: true }); + await fs.mkdir(legacyDir, { recursive: true }); + await fs.writeFile(path.join(cachedDir, "metadata.json"), JSON.stringify({ + namespace: "provider-sync", + sizeBytes: 123, + fileCount: 1 + }), "utf8"); + await fs.writeFile(path.join(cachedDir, "added-later.bin"), "x".repeat(4096), "utf8"); + await fs.writeFile(path.join(legacyDir, "metadata.json"), JSON.stringify({ + namespace: "provider-sync" + }), "utf8"); + await fs.writeFile(path.join(legacyDir, "payload.bin"), "legacy", "utf8"); + const legacyBytes = (await fs.stat(path.join(legacyDir, "metadata.json"))).size + + (await fs.stat(path.join(legacyDir, "payload.bin"))).size; + + const summary = await getBackupSummary(codexHome); + assert.deepEqual(summary, { count: 2, totalBytes: 123 + legacyBytes }); + + const pruned = await pruneBackups(codexHome, 1); + assert.equal(pruned.deletedCount, 1); + assert.equal(pruned.freedBytes, 123); +}); + test("pruneBackups ignores directories without managed backup metadata", async () => { const { codexHome } = await makeTempCodexHome(); await writeBackup(codexHome, "20260320T000000000Z", [ @@ -3669,6 +3772,55 @@ test("runSync auto-prunes backups to the default retention count", async () => { assert.equal(result.autoPruneWarning, null); }); +test("runSync succeeds with a warning and still prunes when the inventory refresh fails after commit", async () => { + const { codexHome } = await makeTempCodexHome(); + await writeConfig(codexHome, 'model_provider = "openai"'); + const sessionPath = path.join(codexHome, "sessions", "2026", "03", "19", "rollout-a.jsonl"); + await writeRollout(sessionPath, "thread-a", "apigather"); + await writeStateDb(codexHome, [ + { id: "thread-a", model_provider: "apigather", archived: false } + ]); + + for (let index = 0; index < DEFAULT_BACKUP_RETENTION_COUNT; index += 1) { + await writeBackup(codexHome, `20240101T0000${String(index).padStart(2, "0")}000Z`, [ + ["note.txt", `backup-${index}`] + ]); + } + + const result = await runSync({ + codexHome, + // Break the inventory refresh that runs right after the journal commit. The + // transaction is durable by then, so the sync must report success with a + // warning instead of failing and skipping the prune below. + async faultInjector({ point }) { + if (point !== "before_transaction_commit") { + return; + } + const dirs = await fs.readdir(backupRoot(codexHome)); + for (const dir of dirs) { + const candidate = path.join(backupRoot(codexHome), dir); + try { + await fs.access(path.join(candidate, "transaction-journal.jsonl")); + } catch { + continue; + } + // Keep the namespace so the directory is still a managed backup for the + // prune pass, but make the version unreadable so only the inventory + // refresh fails. + const metadataPath = path.join(candidate, "metadata.json"); + const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); + await fs.writeFile(metadataPath, JSON.stringify({ ...metadata, version: 99 }, null, 2)); + } + } + }); + + assert.equal(result.autoPruneResult.deletedCount, 1); + assert.match(result.autoPruneWarning, /Backup inventory refresh failed/); + assert.match(await fs.readFile(sessionPath, "utf8"), /"model_provider":"openai"/); + assert.equal(await readProvider(codexHome, "thread-a"), "openai"); + assert.deepEqual(await findPendingTransactions(codexHome), []); +}); + test("runSync uses a custom automatic backup retention count", async () => { const { codexHome } = await makeTempCodexHome(); await writeConfig(codexHome, 'model_provider = "openai"');