From c2e0646fc7a48d4402c2f5221f993f79bfd34310 Mon Sep 17 00:00:00 2001 From: "DAL\\Administrator" <3452720699@qq.com> Date: Fri, 7 Aug 2026 18:28:00 +0800 Subject: [PATCH 1/5] perf: harden sync planning and journal throughput --- desktop/CodexProviderSync.App/MainForm.cs | 6 + .../BackupParityTests.cs | 79 +++ .../CoreIntegrationTests.cs | 5 + .../CoreWritePlanningTests.cs | 72 +++ .../SessionRolloutPerformanceTests.cs | 156 ++++++ .../TextFormatterTests.cs | 36 ++ .../TransactionJournalPerformanceTests.cs | 64 +++ .../TransactionJournalTests.cs | 102 +++- .../CodexProviderSync.Core/BackupService.cs | 160 +++++- .../CodexSyncService.cs | 62 ++- .../CoreWritePlanning.cs | 100 +++- desktop/CodexProviderSync.Core/Models.cs | 38 ++ .../SessionRolloutService.cs | 414 +++++++------- .../CodexProviderSync.Core/TextFormatter.cs | 28 + .../TransactionJournalService.cs | 503 +++++++++++++++--- src/backup.js | 103 ++-- test/sync-service.test.js | 31 ++ 17 files changed, 1582 insertions(+), 377 deletions(-) create mode 100644 desktop/CodexProviderSync.Core.Tests/SessionRolloutPerformanceTests.cs create mode 100644 desktop/CodexProviderSync.Core.Tests/TransactionJournalPerformanceTests.cs diff --git a/desktop/CodexProviderSync.App/MainForm.cs b/desktop/CodexProviderSync.App/MainForm.cs index d0bdf16..40d0f42 100644 --- a/desktop/CodexProviderSync.App/MainForm.cs +++ b/desktop/CodexProviderSync.App/MainForm.cs @@ -940,6 +940,9 @@ await RunBusyAsync("执行中...", async () => result, request is SwitchProviderRequest ? "已切换并同步" : "已同步", TextFormatter.ChineseSimplified)); + AppendLog(TextFormatter.FormatPerformanceMetrics( + result.PerformanceMetrics, + TextFormatter.ChineseSimplified)); AppendLog(FormatModelSyncOutcome(result.ModelSync)); AppendLog(string.Empty); await RefreshStatusCoreAsync(request.CodexHome, request.SqliteHomeOverride, provider); @@ -1591,6 +1594,9 @@ private async Task ApplyControllerRefreshAsync(AppSnapshot snapshot) ReloadProviderList(); _codexHomeCombo.Text = _currentStatus.CodexHome; AppendLog($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss}] 已刷新: {_currentStatus.CodexHome}"); + AppendLog(TextFormatter.FormatPerformanceMetrics( + _currentStatus.PerformanceMetrics, + TextFormatter.ChineseSimplified)); } private WindowBoundsState CaptureWindowBounds() diff --git a/desktop/CodexProviderSync.Core.Tests/BackupParityTests.cs b/desktop/CodexProviderSync.Core.Tests/BackupParityTests.cs index b431a96..51cd17f 100644 --- a/desktop/CodexProviderSync.Core.Tests/BackupParityTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/BackupParityTests.cs @@ -36,6 +36,12 @@ await File.WriteAllTextAsync( Assert.False(globalStateFiles.GetProperty(AppConstants.GlobalStateBackupFileBasename).GetBoolean()); Assert.True(metadataRoot.GetProperty("globalStateFilePresent").GetBoolean()); Assert.False(metadataRoot.GetProperty("globalStateBackupFilePresent").GetBoolean()); + Assert.True(metadataRoot.GetProperty("sizeBytes").GetInt64() > 0); + Assert.True(metadataRoot.GetProperty("fileCount").GetInt32() > 0); + Assert.Equal( + Directory.EnumerateFiles(backupDir, "*", SearchOption.AllDirectories) + .Sum(static path => new FileInfo(path).Length), + metadataRoot.GetProperty("sizeBytes").GetInt64()); using JsonDocument manifest = JsonDocument.Parse( await File.ReadAllTextAsync(Path.Combine(backupDir, "session-meta-backup.json"))); @@ -48,6 +54,79 @@ await File.WriteAllTextAsync( entry.GetProperty("originalLastWriteTimeUtcTicks").GetString()); } + [Fact] + public async Task BackupSummary_UsesCachedInventoryAndFallsBackForLegacyMetadata() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + string cached = fixture.BackupPath("20260708T091011111Z"); + string legacy = fixture.BackupPath("20260708T091011110Z"); + Directory.CreateDirectory(cached); + Directory.CreateDirectory(legacy); + await File.WriteAllTextAsync( + Path.Combine(cached, "metadata.json"), + JsonSerializer.Serialize(new + { + version = 2, + @namespace = AppConstants.BackupNamespace, + codexHome = fixture.CodexHome, + targetProvider = "openai", + createdAt = DateTimeOffset.UtcNow, + dbFiles = Array.Empty(), + changedSessionFiles = 0, + sizeBytes = 123L, + fileCount = 1 + })); + await File.WriteAllTextAsync(Path.Combine(cached, "added-later.bin"), new string('x', 4096)); + await File.WriteAllTextAsync( + Path.Combine(legacy, "metadata.json"), + JsonSerializer.Serialize(new + { + version = 1, + @namespace = AppConstants.BackupNamespace, + codexHome = fixture.CodexHome, + targetProvider = "openai", + createdAt = DateTimeOffset.UtcNow, + dbFiles = Array.Empty(), + changedSessionFiles = 0 + })); + await File.WriteAllTextAsync(Path.Combine(legacy, "payload.bin"), new string('y', 17)); + + List fallbacks = []; + BackupService backups = new(new SessionRolloutService(), new SqliteStateService()) + { + DirectoryInventoryFallbackObserver = fallbacks.Add + }; + BackupSummary summary = await backups.GetBackupSummaryAsync(fixture.CodexHome); + + long legacyBytes = Directory.EnumerateFiles(legacy, "*", SearchOption.AllDirectories) + .Sum(static path => new FileInfo(path).Length); + Assert.Equal(2, summary.Count); + Assert.Equal(123L + legacyBytes, summary.TotalBytes); + Assert.Single(fallbacks); + Assert.Equal(Path.GetFullPath(legacy), Path.GetFullPath(fallbacks[0])); + } + + [Fact] + public async Task IncompleteMetadata_IsNotManagedOrPrunedEvenWithProviderSyncNamespace() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + string incomplete = fixture.BackupPath("20260708T091011109Z"); + Directory.CreateDirectory(incomplete); + string sentinel = Path.Combine(incomplete, "keep.txt"); + await File.WriteAllTextAsync( + Path.Combine(incomplete, "metadata.json"), + "{\"namespace\":\"provider-sync\",\"sizeBytes\":1,\"fileCount\":1}"); + await File.WriteAllTextAsync(sentinel, "must remain"); + + BackupService backups = new(new SessionRolloutService(), new SqliteStateService()); + BackupSummary summary = await backups.GetBackupSummaryAsync(fixture.CodexHome); + BackupPruneResult pruned = await backups.PruneBackupsAsync(fixture.CodexHome, 0); + + Assert.Equal(0, summary.Count); + Assert.Equal(0, pruned.DeletedCount); + Assert.True(File.Exists(sentinel)); + } + [Fact] public async Task RestoreBackup_AcceptsNodeStyleV2MetadataAndMillisecondSessionTimestamp() { diff --git a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs index da14a9f..2ed819f 100644 --- a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs @@ -960,6 +960,9 @@ await fixture.WriteStateDbAsync( Assert.Empty(syncResult.SkippedLockedRolloutFiles); Assert.Empty(syncResult.SkippedUnreadableRolloutFiles); Assert.Equal(2, syncResult.SqliteRowsUpdated); + Assert.Equal(2, syncResult.PerformanceMetrics.RolloutScan.EnumeratedRolloutFiles); + Assert.Equal(2, syncResult.PerformanceMetrics.RolloutScan.ContentScanPasses); + Assert.Equal(2, syncResult.PerformanceMetrics.JournalFullValidationCount); BackupMetadataFile backupMetadata = JsonSerializer.Deserialize( await File.ReadAllTextAsync(Path.Combine(syncResult.BackupDir, "metadata.json")), new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase })!; @@ -1343,6 +1346,8 @@ await fixture.WriteStateDbAsync( Assert.Equal(fixture.StateDbPath(), status.StateDbLocation.Path); Assert.Equal(2, status.BackupSummary.Count); Assert.Equal(backupOneBytes + backupTwoBytes, status.BackupSummary.TotalBytes); + Assert.Equal(2, status.PerformanceMetrics.RolloutScan.EnumeratedRolloutFiles); + Assert.Equal(2, status.PerformanceMetrics.RolloutScan.ContentScanPasses); Assert.Contains($"database: {fixture.StateDbPath()}", TextFormatter.FormatStatus(status)); } diff --git a/desktop/CodexProviderSync.Core.Tests/CoreWritePlanningTests.cs b/desktop/CodexProviderSync.Core.Tests/CoreWritePlanningTests.cs index 16c48d4..9e8a3b9 100644 --- a/desktop/CodexProviderSync.Core.Tests/CoreWritePlanningTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/CoreWritePlanningTests.cs @@ -5,6 +5,45 @@ namespace CodexProviderSync.Core.Tests; public sealed class CoreWritePlanningTests { + [Fact] + public async Task ContentFingerprintHint_ReusesExactDigestForPlanPreview() + { + string root = Path.Combine(Path.GetTempPath(), $"codex-provider-hint-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + string path = Path.Combine(root, "rollout.jsonl"); + await File.WriteAllTextAsync(path, "{\"type\":\"session_meta\"}\n"); + try + { + FileInfo info = new(path); + string digest = "sha256:" + Convert.ToHexString( + System.Security.Cryptography.SHA256.HashData(await File.ReadAllBytesAsync(path))) + .ToLowerInvariant() + + $":{info.Length}:{info.LastWriteTimeUtc.Ticks}"; + int hintedReads = 0; + CoreWritePlanSnapshot hinted = await CoreWriteSnapshotBuilder.BuildAsync( + "sync", + "hinted", + [new CoreWriteTargetSpec(path, "replace")], + contentFingerprintHints: + [ + new CoreWriteContentFingerprintHint( + path, + digest, + info.Length, + info.LastWriteTimeUtc.Ticks) + ], + fingerprintObserver: _ => hintedReads += 1); + + CoreWritePlanTarget target = Assert.Single(hinted.Targets); + Assert.Equal(digest, target.Fingerprint); + Assert.Equal(0, hintedReads); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + [Fact] public async Task CheckedSync_RejectsDriftBeforeBackupOrMutation() { @@ -524,6 +563,39 @@ public async Task SqliteWalFingerprint_TreatsMissingAndZeroLengthAsEquivalent() CoreWriteSnapshotBuilder.AssertExactMatch(empty, missingAgain); } + [Fact] + public async Task SnapshotBuilder_ReusesRecursiveInventoryAcrossMainAndAutoPruneTargets() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + string backupRoot = Path.Combine(fixture.Root, "backups"); + string candidate = Path.Combine(backupRoot, "20260807T000000000Z"); + string payload = Path.Combine(candidate, "metadata.json"); + Directory.CreateDirectory(candidate); + await File.WriteAllTextAsync(payload, "{}"); + List fingerprintedPaths = []; + + CoreWritePlanSnapshot snapshot = await CoreWriteSnapshotBuilder.BuildAsync( + "sync", + "shared-recursive-inventory", + [new CoreWriteTargetSpec( + backupRoot, + "create-and-prune", + CoreWriteFingerprintMode.RecursiveInventory)], + [new CoreWriteTargetSpec( + candidate, + "delete", + CoreWriteFingerprintMode.RecursiveInventory)], + fingerprintObserver: fingerprintedPaths.Add); + + Assert.Single(snapshot.Targets); + Assert.Single(snapshot.AutoPruneDeletionTargets); + StringComparer comparer = OperatingSystem.IsWindows() + ? StringComparer.OrdinalIgnoreCase + : StringComparer.Ordinal; + Assert.Equal(1, fingerprintedPaths.Count(path => comparer.Equals(path, Path.GetFullPath(candidate)))); + Assert.Equal(1, fingerprintedPaths.Count(path => comparer.Equals(path, Path.GetFullPath(payload)))); + } + [Fact] public async Task CheckedRestore_RejectsCommittedWalDriftBeforeMutation() { diff --git a/desktop/CodexProviderSync.Core.Tests/SessionRolloutPerformanceTests.cs b/desktop/CodexProviderSync.Core.Tests/SessionRolloutPerformanceTests.cs new file mode 100644 index 0000000..4d08f3e --- /dev/null +++ b/desktop/CodexProviderSync.Core.Tests/SessionRolloutPerformanceTests.cs @@ -0,0 +1,156 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text.Json; +using Xunit.Abstractions; + +namespace CodexProviderSync.Core.Tests; + +public sealed class SessionRolloutPerformanceTests(ITestOutputHelper output) +{ + [Fact] + public async Task ProviderChange_ContentFingerprintMatchesExactRolloutBytes() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + try + { + string path = fixture.RolloutPath("sessions", "rollout-fingerprint.jsonl"); + await fixture.WriteRolloutAsync(path, "thread-fingerprint", "apigather"); + FileInfo snapshot = new(path); + string digest = Convert.ToHexString( + SHA256.HashData(await File.ReadAllBytesAsync(path))) + .ToLowerInvariant(); + + SessionChangeCollection result = await new SessionRolloutService() + .CollectSessionChangesAsync(fixture.CodexHome, "openai"); + + SessionChange change = Assert.Single(result.Changes); + Assert.Equal( + $"sha256:{digest}:{snapshot.Length}:{snapshot.LastWriteTimeUtc.Ticks}", + change.ContentFingerprint); + } + finally + { + Directory.Delete(fixture.Root, recursive: true); + } + } + + [Fact] + public async Task CollectSessionChanges_UsesOneContentPassPerRollout() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + try + { + for (int index = 0; index < 16; index++) + { + string directory = index < 12 ? "sessions" : "archived_sessions"; + string path = fixture.RolloutPath(directory, $"rollout-{index:D2}.jsonl"); + await fixture.WriteRolloutAsync(path, $"thread-{index:D2}", "apigather"); + } + + SessionChangeCollection result = await new SessionRolloutService() + .CollectSessionChangesAsync(fixture.CodexHome, "openai"); + + Assert.Equal(16, result.ScanMetrics.EnumeratedRolloutFiles); + Assert.Equal(16, result.ScanMetrics.ParsedSessionFiles); + Assert.Equal(16, result.ScanMetrics.ContentScanPasses); + Assert.Equal(0, result.ScanMetrics.ModelScanFiles); + Assert.Equal(16, result.Changes.Count); + Assert.Equal(16, result.UserEventThreadIds.Count); + } + finally + { + Directory.Delete(fixture.Root, recursive: true); + } + } + + [Fact] + public async Task CollectSessionChanges_CollectsAllModelBackupsInTheSameContentPass() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + try + { + for (int index = 0; index < 8; index++) + { + string path = fixture.RolloutPath("sessions", $"rollout-model-{index:D2}.jsonl"); + await fixture.WriteRolloutWithTurnContextAsync( + path, + $"thread-model-{index:D2}", + "openai", + "old-model"); + } + + SessionChangeCollection result = await new SessionRolloutService() + .CollectSessionChangesAsync( + fixture.CodexHome, + "openai", + targetModel: "new-model"); + + Assert.Equal(8, result.ScanMetrics.EnumeratedRolloutFiles); + Assert.Equal(8, result.ScanMetrics.ContentScanPasses); + Assert.Equal(8, result.ScanMetrics.ModelScanFiles); + Assert.Equal(8, result.Changes.Count); + Assert.All(result.Changes, change => Assert.Equal(2, change.OriginalTurnContextModels.Count)); + } + finally + { + Directory.Delete(fixture.Root, recursive: true); + } + } + + [Fact] + [Trait("Category", "Performance")] + public async Task EightHundredRollouts_UseExactlyEightHundredContentPasses() + { + if (!string.Equals( + Environment.GetEnvironmentVariable("CODEX_PROVIDER_SYNC_RUN_PERF_TESTS"), + "1", + StringComparison.Ordinal)) + { + return; + } + + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + try + { + string sessionDirectory = Path.GetDirectoryName( + fixture.RolloutPath("sessions", "rollout-placeholder.jsonl"))!; + for (int index = 0; index < 800; index++) + { + string first = JsonSerializer.Serialize(new + { + timestamp = "2026-08-07T00:00:00.000Z", + type = "session_meta", + payload = new + { + id = $"thread-{index:D4}", + cwd = "C:\\AITemp", + model_provider = "apigather" + } + }); + string user = JsonSerializer.Serialize(new + { + type = "event_msg", + payload = new { type = "user_message", message = "hi" } + }); + await File.WriteAllTextAsync( + Path.Combine(sessionDirectory, $"rollout-{index:D4}.jsonl"), + $"{first}\n{user}\n"); + } + + Stopwatch timer = Stopwatch.StartNew(); + SessionChangeCollection result = await new SessionRolloutService() + .CollectSessionChangesAsync(fixture.CodexHome, "openai"); + timer.Stop(); + + Assert.Equal(800, result.ScanMetrics.EnumeratedRolloutFiles); + Assert.Equal(800, result.ScanMetrics.ContentScanPasses); + Assert.Equal(0, result.ScanMetrics.ModelScanFiles); + output.WriteLine( + $"rollouts=800 elapsedMs={timer.ElapsedMilliseconds} contentPasses={result.ScanMetrics.ContentScanPasses}"); + } + finally + { + Directory.Delete(fixture.Root, recursive: true); + } + } +} diff --git a/desktop/CodexProviderSync.Core.Tests/TextFormatterTests.cs b/desktop/CodexProviderSync.Core.Tests/TextFormatterTests.cs index 5d4a540..70e478c 100644 --- a/desktop/CodexProviderSync.Core.Tests/TextFormatterTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/TextFormatterTests.cs @@ -145,6 +145,42 @@ public void ResultFormatters_ChineseCoverSyncRestoreAndBackupCleanup() Assert.Contains("释放空间: 2 KB", pruneText); } + [Fact] + public void PerformanceFormatters_ReportStageDurationsAndStructuralCounts() + { + SyncPerformanceMetrics sync = new() + { + TotalDurationMs = 100, + PreparationDurationMs = 20, + CheckedPlanValidationDurationMs = 15, + BackupDurationMs = 10, + MutationDurationMs = 50, + PruneDurationMs = 5, + JournalFullValidationCount = 2, + RolloutScan = new SessionScanMetrics + { + EnumeratedRolloutFiles = 800, + ContentScanPasses = 800 + } + }; + StatusPerformanceMetrics status = new() + { + TotalDurationMs = 30, + RolloutScanDurationMs = 20, + BackupSummaryDurationMs = 4, + RolloutScan = sync.RolloutScan + }; + + string syncText = TextFormatter.FormatPerformanceMetrics(sync, TextFormatter.ChineseSimplified); + string statusText = TextFormatter.FormatPerformanceMetrics(status, TextFormatter.English); + + Assert.Contains("计划校验=15ms", syncText); + Assert.Contains("内容扫描=800", syncText); + Assert.Contains("journal全量校验=2", syncText); + Assert.Contains("rollout-scan=20ms", statusText); + Assert.Contains("content-scans=800", statusText); + } + private static StatusSnapshot MinimalStatus() => new() { CodexHome = @"C:\Codex", diff --git a/desktop/CodexProviderSync.Core.Tests/TransactionJournalPerformanceTests.cs b/desktop/CodexProviderSync.Core.Tests/TransactionJournalPerformanceTests.cs new file mode 100644 index 0000000..d7990d5 --- /dev/null +++ b/desktop/CodexProviderSync.Core.Tests/TransactionJournalPerformanceTests.cs @@ -0,0 +1,64 @@ +using System.Diagnostics; +using Xunit.Abstractions; + +namespace CodexProviderSync.Core.Tests; + +public sealed class TransactionJournalPerformanceTests(ITestOutputHelper output) +{ + [Fact] + [Trait("Category", "Performance")] + public async Task EightHundredTargets_AppendWithoutGrowingFullJournalReads() + { + if (!OperatingSystem.IsWindows()) + { + return; + } + if (!string.Equals( + Environment.GetEnvironmentVariable("CODEX_PROVIDER_SYNC_RUN_PERF_TESTS"), + "1", + StringComparison.Ordinal)) + { + return; + } + + string root = Path.Combine( + Path.GetTempPath(), + $"codex-provider-journal-perf-{Guid.NewGuid():N}"); + string backupDir = Path.Combine(root, "backup"); + string codexHome = Path.Combine(root, ".codex"); + Directory.CreateDirectory(backupDir); + Directory.CreateDirectory(codexHome); + string[] targets = Enumerable.Range(0, 800) + .Select(index => Path.Combine(codexHome, $"rollout-{index:D4}.jsonl")) + .ToArray(); + + try + { + await using FileTransactionJournal journal = await FileTransactionJournal.CreateOwnedAsync( + backupDir, + codexHome, + "target-provider", + targets); + Stopwatch timer = Stopwatch.StartNew(); + foreach (string target in targets) + { + await journal.ApplyingAsync("rollout", target); + await journal.AppliedAsync("rollout", target); + } + timer.Stop(); + + Assert.Equal(0, journal.AppendFullJournalValidationCount); + await journal.CommittedAsync(); + Assert.Equal(2, journal.AppendFullJournalValidationCount); + output.WriteLine( + $"targets=800 appends=1600 elapsedMs={timer.ElapsedMilliseconds} fullJournalReads={journal.AppendFullJournalValidationCount}"); + } + finally + { + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + } + } +} diff --git a/desktop/CodexProviderSync.Core.Tests/TransactionJournalTests.cs b/desktop/CodexProviderSync.Core.Tests/TransactionJournalTests.cs index 4ab78f0..9a1578d 100644 --- a/desktop/CodexProviderSync.Core.Tests/TransactionJournalTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/TransactionJournalTests.cs @@ -82,6 +82,90 @@ public async Task AppendAsync_ResynchronizesFromValidExternallyAppendedRecord() Assert.Equal(3, applied.LastSequence); Assert.Equal("applied", applied.State); Assert.False(applied.InvalidTail); + Assert.Equal(1, fixture.Journal.AppendFullJournalValidationCount); + } + + [Fact] + public async Task NormalProgressAppends_DoNotReparseGrowingJournal() + { + JournalFixture fixture = await JournalFixture.CreateAsync(32, owned: true); + + foreach (string target in fixture.Targets) + { + await fixture.Journal.ApplyingAsync("rollout", target); + await fixture.Journal.AppliedAsync("rollout", target); + } + + Assert.Equal( + OperatingSystem.IsWindows() ? 0 : 64, + fixture.Journal.AppendFullJournalValidationCount); + await fixture.Journal.CommittedAsync(); + Assert.Equal( + OperatingSystem.IsWindows() ? 2 : 66, + fixture.Journal.AppendFullJournalValidationCount); + + PendingTransactionInfo committed = await fixture.Journal.ReadCurrentInfoAsync(); + Assert.True(committed.Terminal); + Assert.Equal(66, committed.LastSequence); + Assert.All(committed.AffectedTargets, target => Assert.Equal("applied", target.State)); + } + + [Fact] + public async Task AppendAsync_RejectsSameLengthInvalidTailBeforeWriting() + { + JournalFixture fixture = await JournalFixture.CreateAsync(1); + byte[] before = await File.ReadAllBytesAsync(fixture.Journal.FilePath); + Assert.Equal((byte)'\n', before[^1]); + before[^1] = (byte)' '; + await File.WriteAllBytesAsync(fixture.Journal.FilePath, before); + long invalidLength = new FileInfo(fixture.Journal.FilePath).Length; + + InvalidOperationException error = await Assert.ThrowsAsync( + () => fixture.Journal.ApplyingAsync("rollout", fixture.Targets[0])); + + Assert.Contains("invalid", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(invalidLength, new FileInfo(fixture.Journal.FilePath).Length); + Assert.Equal(1, fixture.Journal.AppendFullJournalValidationCount); + Assert.True((await fixture.Journal.ReadCurrentInfoAsync()).InvalidTail); + } + + [Fact] + public async Task AppendAsync_RejectsSameLengthPreparedRewriteBeforeWriting() + { + JournalFixture fixture = await JournalFixture.CreateAsync(1); + string originalTarget = Path.GetFullPath(fixture.Targets[0]); + string before = await File.ReadAllTextAsync(fixture.Journal.FilePath); + string rewritten = before.Replace("rollout-0.jsonl", "rollout-0.jsonx", StringComparison.Ordinal); + Assert.Equal(before.Length, rewritten.Length); + Assert.NotEqual(before, rewritten); + await File.WriteAllTextAsync(fixture.Journal.FilePath, rewritten); + long rewrittenLength = new FileInfo(fixture.Journal.FilePath).Length; + + InvalidOperationException error = await Assert.ThrowsAsync( + () => fixture.Journal.ApplyingAsync("rollout", originalTarget)); + + Assert.Contains("prepared", error.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(rewrittenLength, new FileInfo(fixture.Journal.FilePath).Length); + } + + [Fact] + public async Task OwnedJournal_BlocksExternalSameLengthRewriteOnWindows() + { + if (!OperatingSystem.IsWindows()) + { + return; + } + JournalFixture fixture = await JournalFixture.CreateAsync(1, owned: true); + await Assert.ThrowsAsync( + async () => + { + await using FileStream _ = new( + fixture.Journal.FilePath, + FileMode.Open, + FileAccess.Write, + FileShare.ReadWrite | FileShare.Delete); + }); + await fixture.Journal.ApplyingAsync("rollout", fixture.Targets[0]); } [Fact] @@ -185,7 +269,7 @@ private sealed record JournalFixture( FileTransactionJournal Journal, IReadOnlyList Targets) { - internal static async Task CreateAsync(int targetCount) + internal static async Task CreateAsync(int targetCount, bool owned = false) { string root = Path.Combine( Path.GetTempPath(), @@ -197,11 +281,17 @@ internal static async Task CreateAsync(int targetCount) string[] targets = Enumerable.Range(0, targetCount) .Select(index => Path.Combine(codexHome, $"rollout-{index}.jsonl")) .ToArray(); - FileTransactionJournal journal = await FileTransactionJournal.CreateAsync( - backupDir, - codexHome, - "target-provider", - targets); + FileTransactionJournal journal = owned + ? await FileTransactionJournal.CreateOwnedAsync( + backupDir, + codexHome, + "target-provider", + targets) + : await FileTransactionJournal.CreateAsync( + backupDir, + codexHome, + "target-provider", + targets); return new JournalFixture(root, journal, targets); } } diff --git a/desktop/CodexProviderSync.Core/BackupService.cs b/desktop/CodexProviderSync.Core/BackupService.cs index a12fd41..0cfdc35 100644 --- a/desktop/CodexProviderSync.Core/BackupService.cs +++ b/desktop/CodexProviderSync.Core/BackupService.cs @@ -1,3 +1,4 @@ +using System.Text; using System.Text.Json; namespace CodexProviderSync.Core; @@ -9,6 +10,8 @@ public sealed class BackupService internal Func? AtomicWriteFaultInjector { get; set; } + internal Action? DirectoryInventoryFallbackObserver { get; set; } + public BackupService(SessionRolloutService sessionRolloutService, SqliteStateService sqliteStateService) { _sessionRolloutService = sessionRolloutService; @@ -136,10 +139,7 @@ await AtomicFile.WriteAllTextAsync( GlobalStateFilePresent = globalStateFilePresent, GlobalStateBackupFilePresent = globalStateBackupFilePresent }; - await AtomicFile.WriteAllTextAsync( - Path.Combine(backupDir, "metadata.json"), - JsonSerializer.Serialize(metadata, JsonOptions()), - faultInjector: AtomicWriteFaultInjector); + await WriteMetadataWithInventoryAsync(backupDir, metadata); return backupDir; } @@ -332,17 +332,16 @@ await File.ReadAllTextAsync(metadataPath), ChangedSessionFiles = sessionChanges.Count, GlobalStateFiles = metadata.GlobalStateFiles, GlobalStateFilePresent = metadata.GlobalStateFilePresent, - GlobalStateBackupFilePresent = metadata.GlobalStateBackupFilePresent + GlobalStateBackupFilePresent = metadata.GlobalStateBackupFilePresent, + SizeBytes = metadata.SizeBytes, + FileCount = metadata.FileCount }; await AtomicFile.WriteAllTextAsync( manifestPath, JsonSerializer.Serialize(sessionManifest, JsonOptions()), faultInjector: AtomicWriteFaultInjector); - await AtomicFile.WriteAllTextAsync( - metadataPath, - JsonSerializer.Serialize(metadata, JsonOptions()), - faultInjector: AtomicWriteFaultInjector); + await WriteMetadataWithInventoryAsync(normalizedBackupDir, metadata); } internal async Task> ReadSessionBackupEntriesAsync( @@ -487,7 +486,7 @@ public Task GetBackupSummaryAsync(string codexHome) } List entries = GetManagedBackupDirectories(backupRoot); - long totalBytes = entries.Sum(static entry => GetDirectorySize(entry.FullName)); + long totalBytes = entries.Sum(entry => GetBackupDirectorySize(entry.FullName)); return new BackupSummary { @@ -581,7 +580,7 @@ private async Task PruneBackupsCoreAsync( long freedBytes = 0; foreach (DirectoryInfo entry in toDelete) { - freedBytes += GetDirectorySize(entry.FullName); + freedBytes += GetBackupDirectorySize(entry.FullName); entry.Delete(recursive: true); } @@ -912,16 +911,127 @@ private static JsonSerializerOptions JsonOptions() }; } - private static long GetDirectorySize(string directoryPath) + private async Task WriteMetadataWithInventoryAsync(string backupDir, BackupMetadataFile metadata) + { + string metadataPath = Path.Combine(backupDir, "metadata.json"); + (long payloadBytes, int payloadFileCount) = GetDirectoryInventory( + backupDir, + metadataPath); + int fileCount = checked(payloadFileCount + 1); + long sizeBytes = 0; + string serialized = string.Empty; + for (int attempt = 0; attempt < 8; attempt += 1) + { + BackupMetadataFile withInventory = CopyMetadataWithInventory(metadata, sizeBytes, fileCount); + serialized = JsonSerializer.Serialize(withInventory, JsonOptions()); + long nextSizeBytes = checked(payloadBytes + Encoding.UTF8.GetByteCount(serialized)); + if (nextSizeBytes == sizeBytes) + { + break; + } + sizeBytes = nextSizeBytes; + } + + BackupMetadataFile finalMetadata = CopyMetadataWithInventory(metadata, sizeBytes, fileCount); + serialized = JsonSerializer.Serialize(finalMetadata, JsonOptions()); + long verifiedSizeBytes = checked(payloadBytes + Encoding.UTF8.GetByteCount(serialized)); + if (verifiedSizeBytes != sizeBytes) + { + finalMetadata = CopyMetadataWithInventory(metadata, verifiedSizeBytes, fileCount); + serialized = JsonSerializer.Serialize(finalMetadata, JsonOptions()); + } + await AtomicFile.WriteAllTextAsync( + metadataPath, + serialized, + faultInjector: AtomicWriteFaultInjector); + } + + private static BackupMetadataFile CopyMetadataWithInventory( + BackupMetadataFile metadata, + long sizeBytes, + int fileCount) + { + return new BackupMetadataFile + { + Version = metadata.Version, + Namespace = metadata.Namespace, + CodexHome = metadata.CodexHome, + SqliteHome = metadata.SqliteHome, + TargetProvider = metadata.TargetProvider, + CreatedAt = metadata.CreatedAt, + DbFiles = metadata.DbFiles, + SqliteDbFiles = metadata.SqliteDbFiles, + ChangedSessionFiles = metadata.ChangedSessionFiles, + GlobalStateFiles = metadata.GlobalStateFiles, + GlobalStateFilePresent = metadata.GlobalStateFilePresent, + GlobalStateBackupFilePresent = metadata.GlobalStateBackupFilePresent, + SizeBytes = sizeBytes, + FileCount = fileCount + }; + } + + private long GetBackupDirectorySize(string directoryPath) + { + if (TryReadCachedDirectoryInventory(directoryPath, out long sizeBytes)) + { + return sizeBytes; + } + DirectoryInventoryFallbackObserver?.Invoke(directoryPath); + return GetDirectoryInventory(directoryPath).SizeBytes; + } + + private static bool TryReadCachedDirectoryInventory(string directoryPath, out long sizeBytes) + { + sizeBytes = 0; + string metadataPath = Path.Combine(directoryPath, "metadata.json"); + try + { + using JsonDocument document = JsonDocument.Parse(File.ReadAllText(metadataPath)); + JsonElement root = document.RootElement; + if (!root.TryGetProperty("namespace", out JsonElement namespaceValue) + || !string.Equals(namespaceValue.GetString(), AppConstants.BackupNamespace, StringComparison.Ordinal) + || !root.TryGetProperty("sizeBytes", out JsonElement sizeValue) + || !sizeValue.TryGetInt64(out long cachedSize) + || cachedSize < 0 + || !root.TryGetProperty("fileCount", out JsonElement countValue) + || !countValue.TryGetInt32(out int cachedFileCount) + || cachedFileCount < 1) + { + return false; + } + sizeBytes = cachedSize; + return true; + } + catch + { + return false; + } + } + + private static (long SizeBytes, int FileCount) GetDirectoryInventory( + string directoryPath, + string? excludedFilePath = null) { if (!Directory.Exists(directoryPath)) { - return 0; + return (0, 0); } - return Directory - .EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories) - .Sum(static filePath => new FileInfo(filePath).Length); + string? excluded = string.IsNullOrWhiteSpace(excludedFilePath) + ? null + : Path.GetFullPath(excludedFilePath); + long sizeBytes = 0; + int fileCount = 0; + foreach (string filePath in Directory.EnumerateFiles(directoryPath, "*", SearchOption.AllDirectories)) + { + if (excluded is not null && PathsEqual(filePath, excluded)) + { + continue; + } + sizeBytes = checked(sizeBytes + new FileInfo(filePath).Length); + fileCount = checked(fileCount + 1); + } + return (sizeBytes, fileCount); } private static List GetManagedBackupDirectories(string backupRoot) @@ -945,7 +1055,7 @@ private static bool IsManagedBackupDirectory(string backupDirectoryPath) try { - BackupMetadataFile? metadata = JsonSerializer.Deserialize( + BackupMetadataValidationFile? metadata = JsonSerializer.Deserialize( File.ReadAllText(metadataPath), JsonOptions()); return string.Equals(metadata?.Namespace, AppConstants.BackupNamespace, StringComparison.Ordinal); @@ -955,6 +1065,22 @@ private static bool IsManagedBackupDirectory(string backupDirectoryPath) return false; } } + + private sealed class BackupMetadataValidationFile + { + public int Version { get; init; } + public required string Namespace { get; init; } + public required string CodexHome { get; init; } + public string? SqliteHome { get; init; } + public required string TargetProvider { get; init; } + public required DateTimeOffset CreatedAt { get; init; } + public required List DbFiles { get; init; } + public List SqliteDbFiles { get; init; } = []; + public int ChangedSessionFiles { get; init; } + public Dictionary? GlobalStateFiles { get; init; } + public bool? GlobalStateFilePresent { get; init; } + public bool? GlobalStateBackupFilePresent { get; init; } + } } internal sealed record BackupRecoveryCoverage(bool Config, bool Database, bool Sessions); diff --git a/desktop/CodexProviderSync.Core/CodexSyncService.cs b/desktop/CodexProviderSync.Core/CodexSyncService.cs index 8778207..b8d0aa1 100644 --- a/desktop/CodexProviderSync.Core/CodexSyncService.cs +++ b/desktop/CodexProviderSync.Core/CodexSyncService.cs @@ -1,3 +1,5 @@ +using System.Diagnostics; + namespace CodexProviderSync.Core; public sealed class CodexSyncService @@ -50,13 +52,16 @@ public async Task GetStatusAsync( string? explicitCodexHome = null, string? explicitSqliteHome = null) { + long totalStarted = Stopwatch.GetTimestamp(); string codexHome = _codexHomeService.NormalizeCodexHome(explicitCodexHome); await _codexHomeService.EnsureCodexHomeAsync(codexHome); string configText = await _configFileService.ReadConfigTextAsync(_codexHomeService.ConfigPath(codexHome)); CodexStorageLayout storage = await PrepareStorageAsync(codexHome, explicitSqliteHome, configText); CurrentProviderInfo currentProvider = _configFileService.ReadCurrentProviderFromConfigText(configText); IReadOnlyList configuredProviders = _configFileService.ListConfiguredProviderIds(configText); + long rolloutScanStarted = Stopwatch.GetTimestamp(); SessionChangeCollection rolloutInfo = await _sessionRolloutService.CollectSessionChangesAsync(codexHome, "__status_only__", skipLockedReads: true); + long rolloutScanDurationMs = ElapsedMilliseconds(rolloutScanStarted); StateDbLocation? stateDbLocation = storage.StateDbLocation; ProviderCounts? sqliteCounts = storage.SqliteAccess.Supported ? await _sqliteStateService.ReadSqliteProviderCountsAsync(storage) @@ -71,7 +76,9 @@ public async Task GetStatusAsync( || sqliteCounts?.Unreadable == true ? [] : await _globalStateService.ReadProjectThreadVisibilityAsync(storage); + long backupSummaryStarted = Stopwatch.GetTimestamp(); BackupSummary backupSummary = await _backupService.GetBackupSummaryAsync(codexHome); + long backupSummaryDurationMs = ElapsedMilliseconds(backupSummaryStarted); IReadOnlyList pendingTransactions = await FileTransactionJournal.FindPendingAsync(codexHome); return new StatusSnapshot @@ -100,7 +107,14 @@ public async Task GetStatusAsync( item.State, item.BackupDir, item.JournalPath)) - .ToArray() + .ToArray(), + PerformanceMetrics = new StatusPerformanceMetrics + { + TotalDurationMs = ElapsedMilliseconds(totalStarted), + RolloutScanDurationMs = rolloutScanDurationMs, + BackupSummaryDurationMs = backupSummaryDurationMs, + RolloutScan = rolloutInfo.ScanMetrics + } }; } @@ -260,7 +274,8 @@ private async Task PrepareSyncAsync( private async Task BuildSyncPlanSnapshotAsync( SyncPreparation preparation, int keepCount, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool useContentFingerprintHints = true) { string operation = preparation.SwitchPreparation is null ? "sync" : "switch"; List targets = @@ -339,6 +354,15 @@ private async Task BuildSyncPlanSnapshotAsync( "delete", CoreWriteFingerprintMode.RecursiveInventory)), warnings: warnings, + contentFingerprintHints: useContentFingerprintHints + ? preparation.WritableChanges + .Where(static change => change.ContentFingerprint is not null) + .Select(static change => new CoreWriteContentFingerprintHint( + change.Path, + change.ContentFingerprint!, + change.OriginalFileLength, + change.OriginalLastWriteTimeUtcTicks)) + : null, cancellationToken: cancellationToken); } @@ -366,12 +390,14 @@ private async Task RunSyncCoreAsync( DateTimeOffset? snapshotExpiresAtUtc, CancellationToken cancellationToken = default) { + long totalStarted = Stopwatch.GetTimestamp(); ValidateAutomaticRetention(keepCount); string codexHome = _codexHomeService.NormalizeCodexHome(explicitCodexHome); await _codexHomeService.EnsureCodexHomeAsync(codexHome); await using LockHandle _ = await _lockService.AcquireLockAsync(codexHome, "sync"); await FileTransactionJournal.AssertNoPendingAsync(codexHome); + long preparationStarted = Stopwatch.GetTimestamp(); SyncPreparation preparation = await PrepareSyncAsync( codexHome, provider, @@ -380,6 +406,7 @@ private async Task RunSyncCoreAsync( explicitSqliteHome, switchPreparationFactory, cancellationToken); + long preparationDurationMs = ElapsedMilliseconds(preparationStarted); string configPath = preparation.ConfigPath; string configText = preparation.ConfigText; SwitchPreparation? switchPreparation = preparation.SwitchPreparation; @@ -394,16 +421,20 @@ private async Task RunSyncCoreAsync( List skippedRolloutFiles = [.. preparation.SkippedRolloutFiles]; IReadOnlyList skippedUnreadableRolloutFiles = preparation.SkippedUnreadableRolloutFiles; IReadOnlyList? checkedAutoPruneDeletionTargets = null; + long checkedPlanValidationDurationMs = 0; if (expectedSnapshot is not null) { + long checkedPlanValidationStarted = Stopwatch.GetTimestamp(); AssertSnapshotFresh(snapshotExpiresAtUtc); CoreWritePlanSnapshot actualSnapshot = await BuildSyncPlanSnapshotAsync( preparation, keepCount, - cancellationToken); + cancellationToken, + useContentFingerprintHints: false); CoreWriteSnapshotBuilder.AssertExactMatch(expectedSnapshot, actualSnapshot); AssertSnapshotFresh(snapshotExpiresAtUtc); checkedAutoPruneDeletionTargets = expectedSnapshot.AutoPruneDeletionTargets; + checkedPlanValidationDurationMs = ElapsedMilliseconds(checkedPlanValidationStarted); } cancellationToken.ThrowIfCancellationRequested(); if (FaultInjector is not null) @@ -411,12 +442,15 @@ private async Task RunSyncCoreAsync( await FaultInjector("before_backup", null, 0); } string? effectiveConfigBackupText = switchPreparation is null ? configBackupText : configText; + long backupStarted = Stopwatch.GetTimestamp(); string backupDir = await _backupService.CreateBackupAsync( storage, targetProvider, writableChanges, configPath, effectiveConfigBackupText); + long backupDurationMs = ElapsedMilliseconds(backupStarted); + long mutationStarted = Stopwatch.GetTimestamp(); List appliedSessionChanges = []; bool sqliteMutationCommitted = false; bool sqliteCommitAttempted = false; @@ -429,7 +463,7 @@ private async Task RunSyncCoreAsync( .Concat(switchPreparation is null ? [] : [Path.GetFullPath(configPath)]) .Concat(storage.StateDbLocation is null ? [] : [Path.GetFullPath(storage.StateDbLocation.Path)]) .ToArray(); - FileTransactionJournal journal = await FileTransactionJournal.CreateAsync( + await using FileTransactionJournal journal = await FileTransactionJournal.CreateOwnedAsync( backupDir, codexHome, targetProvider, @@ -600,6 +634,8 @@ await FaultInjector( } await journal.CommittedAsync(); transactionCommitted = true; + await journal.DisposeAsync(); + long mutationDurationMs = ElapsedMilliseconds(mutationStarted); if (FaultInjector is not null) { await FaultInjector("after_transaction_commit", null, completedTargets.Count); @@ -607,6 +643,7 @@ await FaultInjector( BackupPruneResult? autoPruneResult = null; string? autoPruneWarning = null; + long pruneStarted = Stopwatch.GetTimestamp(); try { autoPruneResult = await _backupService.PruneAutomaticBackupsAsync( @@ -619,6 +656,7 @@ await FaultInjector( { autoPruneWarning = $"Automatic backup cleanup failed: {error.Message}"; } + long pruneDurationMs = ElapsedMilliseconds(pruneStarted); SyncResult result = new() { @@ -645,7 +683,18 @@ await FaultInjector( AutoPruneResult = autoPruneResult, AutoPruneWarning = autoPruneWarning, ConfigUpdated = switchPreparation is not null, - ModelSync = switchPreparation?.ModelSync ?? ModelSyncOutcome.NotApplicable() + ModelSync = switchPreparation?.ModelSync ?? ModelSyncOutcome.NotApplicable(), + PerformanceMetrics = new SyncPerformanceMetrics + { + TotalDurationMs = ElapsedMilliseconds(totalStarted), + PreparationDurationMs = preparationDurationMs, + CheckedPlanValidationDurationMs = checkedPlanValidationDurationMs, + BackupDurationMs = backupDurationMs, + MutationDurationMs = mutationDurationMs, + PruneDurationMs = pruneDurationMs, + JournalFullValidationCount = journal.AppendFullJournalValidationCount, + RolloutScan = sessionInfo.ScanMetrics + } }; return result; } @@ -1444,6 +1493,9 @@ public Task GetBackupStorageInfoAsync(string backupDir) return $"Encrypted content warning: {total} rollout file(s) contain encrypted_content from provider(s) {string.Join(", ", riskyProviders)}. Visibility metadata can be synchronized to {targetProvider}, but continuing or compacting those histories may fail with invalid_encrypted_content. Return to the original provider/account or start a new session if you need reliable continuation."; } + private static long ElapsedMilliseconds(long started) => + (long)Math.Round(Stopwatch.GetElapsedTime(started).TotalMilliseconds); + private async Task PrepareStorageAsync( string codexHome, string? explicitSqliteHome, diff --git a/desktop/CodexProviderSync.Core/CoreWritePlanning.cs b/desktop/CodexProviderSync.Core/CoreWritePlanning.cs index 54e51f9..a2ec0bd 100644 --- a/desktop/CodexProviderSync.Core/CoreWritePlanning.cs +++ b/desktop/CodexProviderSync.Core/CoreWritePlanning.cs @@ -54,6 +54,12 @@ internal sealed record CoreWriteTargetSpec( string Action, CoreWriteFingerprintMode FingerprintMode = CoreWriteFingerprintMode.Content); +internal sealed record CoreWriteContentFingerprintHint( + string Path, + string Fingerprint, + long Length, + long LastWriteTimeUtcTicks); + internal static class CoreWriteSnapshotBuilder { private const string FormatVersion = "core-write-snapshot-v2"; @@ -64,17 +70,36 @@ public static async Task BuildAsync( IEnumerable targets, IEnumerable? autoPruneDeletionTargets = null, IEnumerable? warnings = null, - CancellationToken cancellationToken = default) + IEnumerable? contentFingerprintHints = null, + CancellationToken cancellationToken = default, + Action? fingerprintObserver = null) { ArgumentException.ThrowIfNullOrWhiteSpace(operation); ArgumentNullException.ThrowIfNull(binding); + Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache = []; + foreach (CoreWriteContentFingerprintHint hint in contentFingerprintHints ?? []) + { + string fullPath = Path.GetFullPath(hint.Path); + FileInfo current = new(fullPath); + if (!current.Exists + || current.Length != hint.Length + || current.LastWriteTimeUtc.Ticks != hint.LastWriteTimeUtcTicks) + { + throw new CoreWritePlanStaleException(); + } + fingerprintCache[(fullPath, CoreWriteFingerprintMode.Content)] = hint.Fingerprint; + } IReadOnlyList capturedTargets = await CaptureTargetsAsync( targets, - cancellationToken); + fingerprintCache, + cancellationToken, + fingerprintObserver); IReadOnlyList capturedAutoPruneTargets = await CaptureTargetsAsync( autoPruneDeletionTargets ?? [], - cancellationToken); + fingerprintCache, + cancellationToken, + fingerprintObserver); IReadOnlyList capturedWarnings = (warnings ?? []) .Select(static warning => warning with { }) .OrderBy(static warning => warning.Code, StringComparer.Ordinal) @@ -132,7 +157,9 @@ public static void AssertExactMatch( private static async Task> CaptureTargetsAsync( IEnumerable specs, - CancellationToken cancellationToken) + Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache, + CancellationToken cancellationToken, + Action? fingerprintObserver) { CoreWriteTargetSpec[] normalized = specs .Select(static spec => new CoreWriteTargetSpec( @@ -144,19 +171,15 @@ private static async Task> CaptureTargetsAsyn .ThenBy(static spec => spec.Action, StringComparer.Ordinal) .ToArray(); List result = new(normalized.Length); - Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache = []; foreach (CoreWriteTargetSpec spec in normalized) { cancellationToken.ThrowIfCancellationRequested(); - (string Path, CoreWriteFingerprintMode Mode) cacheKey = (spec.Path, spec.FingerprintMode); - if (!fingerprintCache.TryGetValue(cacheKey, out string? fingerprint)) - { - fingerprint = await FingerprintPathAsync( - spec.Path, - spec.FingerprintMode, - cancellationToken); - fingerprintCache.Add(cacheKey, fingerprint); - } + string fingerprint = await FingerprintPathAsync( + spec.Path, + spec.FingerprintMode, + fingerprintCache, + cancellationToken, + fingerprintObserver); result.Add(new CoreWritePlanTarget(spec.Path, spec.Action, fingerprint)); } return result.AsReadOnly(); @@ -165,26 +188,44 @@ private static async Task> CaptureTargetsAsyn private static async Task FingerprintPathAsync( string fullPath, CoreWriteFingerprintMode mode, - CancellationToken cancellationToken) + Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache, + CancellationToken cancellationToken, + Action? fingerprintObserver) { + fullPath = Path.GetFullPath(fullPath); + (string Path, CoreWriteFingerprintMode Mode) cacheKey = (fullPath, mode); + if (fingerprintCache.TryGetValue(cacheKey, out string? cached)) + { + return cached; + } + cancellationToken.ThrowIfCancellationRequested(); + fingerprintObserver?.Invoke(fullPath); + string fingerprint; if (!File.Exists(fullPath) && !Directory.Exists(fullPath)) { - if (mode == CoreWriteFingerprintMode.SqliteWalContent) - { - return FingerprintEmptySqliteWal(fullPath); - } - return Sha256($"missing\n{fullPath}"); + fingerprint = mode == CoreWriteFingerprintMode.SqliteWalContent + ? FingerprintEmptySqliteWal(fullPath) + : Sha256($"missing\n{fullPath}"); + fingerprintCache.Add(cacheKey, fingerprint); + return fingerprint; } FileAttributes attributes = File.GetAttributes(fullPath); if ((attributes & FileAttributes.ReparsePoint) != 0) { DateTime lastWrite = File.GetLastWriteTimeUtc(fullPath); - return Sha256($"reparse\n{fullPath}\n{(int)attributes}\n{lastWrite.Ticks}"); + fingerprint = Sha256($"reparse\n{fullPath}\n{(int)attributes}\n{lastWrite.Ticks}"); + fingerprintCache.Add(cacheKey, fingerprint); + return fingerprint; } - return (attributes & FileAttributes.Directory) != 0 - ? await FingerprintDirectoryAsync(fullPath, mode, cancellationToken) + fingerprint = (attributes & FileAttributes.Directory) != 0 + ? await FingerprintDirectoryAsync( + fullPath, + mode, + fingerprintCache, + cancellationToken, + fingerprintObserver) : mode switch { CoreWriteFingerprintMode.RecursiveInventory => FingerprintFileInventory(fullPath, attributes), @@ -198,12 +239,16 @@ private static async Task FingerprintPathAsync( cancellationToken), _ => await FingerprintFileAsync(fullPath, cancellationToken) }; + fingerprintCache.Add(cacheKey, fingerprint); + return fingerprint; } private static async Task FingerprintDirectoryAsync( string directoryPath, CoreWriteFingerprintMode mode, - CancellationToken cancellationToken) + Dictionary<(string Path, CoreWriteFingerprintMode Mode), string> fingerprintCache, + CancellationToken cancellationToken, + Action? fingerprintObserver) { StringBuilder canonical = new(); Append(canonical, "type", "directory"); @@ -231,7 +276,12 @@ private static async Task FingerprintDirectoryAsync( Append( canonical, "entry.fingerprint", - await FingerprintPathAsync(entry, mode, cancellationToken)); + await FingerprintPathAsync( + entry, + mode, + fingerprintCache, + cancellationToken, + fingerprintObserver)); } return Sha256(canonical.ToString()); } diff --git a/desktop/CodexProviderSync.Core/Models.cs b/desktop/CodexProviderSync.Core/Models.cs index 4f744c8..66888c0 100644 --- a/desktop/CodexProviderSync.Core/Models.cs +++ b/desktop/CodexProviderSync.Core/Models.cs @@ -37,6 +37,16 @@ public sealed class StatusSnapshot public required string BackupRoot { get; init; } public required BackupSummary BackupSummary { get; init; } public IReadOnlyList PendingTransactions { get; init; } = []; + [JsonIgnore] + public StatusPerformanceMetrics PerformanceMetrics { get; init; } = new(); +} + +public sealed class StatusPerformanceMetrics +{ + public long TotalDurationMs { get; init; } + public long RolloutScanDurationMs { get; init; } + public long BackupSummaryDurationMs { get; init; } + public SessionScanMetrics RolloutScan { get; init; } = new(); } public sealed record TransactionRecoveryInfo( @@ -119,6 +129,8 @@ public sealed class SessionChange public required string UpdatedFirstLine { get; init; } public bool ModelOnlyChange { get; init; } public IReadOnlyList OriginalTurnContextModels { get; set; } = []; + [JsonIgnore] + public string? ContentFingerprint { get; init; } } public sealed class TurnContextModelBackup @@ -137,6 +149,16 @@ public sealed class SessionChangeCollection public required ProviderCounts EncryptedContentCounts { get; init; } public required IReadOnlyCollection UserEventThreadIds { get; init; } public required IReadOnlyDictionary ThreadCwdsById { get; init; } + public SessionScanMetrics ScanMetrics { get; init; } = new(); +} + +public sealed class SessionScanMetrics +{ + public int EnumeratedRolloutFiles { get; init; } + public int ParsedSessionFiles { get; init; } + public int ContentScanPasses { get; init; } + public int ModelScanFiles { get; init; } + public long DurationMs { get; init; } } public sealed class SyncResult @@ -165,6 +187,20 @@ public sealed class SyncResult public ModelSyncOutcome ModelSync { get; init; } = ModelSyncOutcome.NotApplicable(); public BackupPruneResult? AutoPruneResult { get; init; } public string? AutoPruneWarning { get; init; } + [JsonIgnore] + public SyncPerformanceMetrics PerformanceMetrics { get; init; } = new(); +} + +public sealed class SyncPerformanceMetrics +{ + public long TotalDurationMs { get; init; } + public long PreparationDurationMs { get; init; } + public long CheckedPlanValidationDurationMs { get; init; } + public long BackupDurationMs { get; init; } + public long MutationDurationMs { get; init; } + public long PruneDurationMs { get; init; } + public int JournalFullValidationCount { get; init; } + public SessionScanMetrics RolloutScan { get; init; } = new(); } public sealed class ModelSyncOutcome @@ -281,6 +317,8 @@ internal sealed class BackupMetadataFile public Dictionary? GlobalStateFiles { get; init; } public bool? GlobalStateFilePresent { get; init; } public bool? GlobalStateBackupFilePresent { get; init; } + public long? SizeBytes { get; init; } + public int? FileCount { get; init; } } internal sealed class SessionBackupManifest diff --git a/desktop/CodexProviderSync.Core/SessionRolloutService.cs b/desktop/CodexProviderSync.Core/SessionRolloutService.cs index bad86b9..8d857d0 100644 --- a/desktop/CodexProviderSync.Core/SessionRolloutService.cs +++ b/desktop/CodexProviderSync.Core/SessionRolloutService.cs @@ -1,4 +1,6 @@ using System.Buffers; +using System.Diagnostics; +using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Text.Json.Nodes; @@ -9,8 +11,6 @@ namespace CodexProviderSync.Core; public sealed class SessionRolloutService { private const string StatusOnlyProvider = "__status_only__"; - private const int ScanBufferSize = 1024 * 1024; - internal Func? ApplyFaultInjector { get; set; } public async Task CollectSessionChangesAsync( @@ -19,6 +19,11 @@ public async Task CollectSessionChangesAsync( bool skipLockedReads = false, string? targetModel = null) { + long scanStarted = Stopwatch.GetTimestamp(); + int enumeratedRolloutFiles = 0; + int parsedSessionFiles = 0; + int contentScanPasses = 0; + int modelScanFiles = 0; List changes = []; List lockedPaths = []; List unreadablePaths = []; @@ -41,9 +46,12 @@ public async Task CollectSessionChangesAsync( .EnumerateFiles(rootDir, "rollout-*.jsonl", SearchOption.AllDirectories) .Order(StringComparer.Ordinal)) { + enumeratedRolloutFiles += 1; FirstLineRecord record; + FileSnapshot scanStart; try { + scanStart = GetFileSnapshot(rolloutPath); record = await ReadFirstLineRecordAsync(rolloutPath); } catch (Exception error) when (skipLockedReads && IsRolloutFileBusyError(error)) @@ -61,6 +69,7 @@ record = await ReadFirstLineRecordAsync(rolloutPath); { continue; } + parsedSessionFiles += 1; string currentProvider = payload!["model_provider"]?.GetValue() ?? "(missing)"; Dictionary bucket = dirName == "archived_sessions" ? archivedCounts : sessionCounts; @@ -72,15 +81,19 @@ record = await ReadFirstLineRecordAsync(rolloutPath); { threadCwdsById[metadataThreadId] = ToDesktopWorkspacePath(metadataCwd); } - bool hasEncryptedContent; + RolloutContentScan contentScan; try { - hasEncryptedContent = await FileHasEncryptedContentAsync(rolloutPath, record.FirstLine, record.Offset); - if (payload["id"]?.GetValue() is string threadId - && await FileHasUserEventAsync(rolloutPath, record.FirstLine, record.Offset)) - { - userEventThreadIds.Add(threadId); - } + bool collectModels = !string.IsNullOrEmpty(targetModel); + bool providerMayChange = !string.Equals(targetProvider, StatusOnlyProvider, StringComparison.Ordinal) + && !string.Equals(currentProvider, targetProvider, StringComparison.Ordinal); + contentScan = await ScanRolloutContentAsync( + rolloutPath, + record, + collectModels, + collectFingerprint: providerMayChange); + contentScanPasses += 1; + modelScanFiles += collectModels ? 1 : 0; } catch (Exception error) when (skipLockedReads && IsRolloutFileBusyError(error)) { @@ -93,7 +106,13 @@ record = await ReadFirstLineRecordAsync(rolloutPath); continue; } - if (hasEncryptedContent) + if (payload["id"]?.GetValue() is string threadId + && contentScan.HasUserEvent) + { + userEventThreadIds.Add(threadId); + } + + if (contentScan.HasEncryptedContent) { Dictionary encryptedBucket = dirName == "archived_sessions" ? encryptedArchivedCounts : encryptedSessionCounts; encryptedBucket[currentProvider] = encryptedBucket.TryGetValue(currentProvider, out int encryptedCount) ? encryptedCount + 1 : 1; @@ -106,31 +125,22 @@ record = await ReadFirstLineRecordAsync(rolloutPath); bool modelChanged = false; if (!string.IsNullOrEmpty(targetModel)) { - try - { - currentModelBackups = await ReadTurnContextModelBackupsAsync(rolloutPath, record); - currentModels = currentModelBackups - .SelectMany(static backup => backup.OriginalModels.Count > 0 - ? backup.OriginalModels - : [backup.OriginalModel]) - .ToArray(); - modelChanged = currentModels.Any(model => !string.Equals(model, targetModel, StringComparison.Ordinal)); - } - catch (Exception error) when (skipLockedReads && IsRolloutFileBusyError(error)) - { - lockedPaths.Add(rolloutPath); - continue; - } - catch (Exception error) when (skipLockedReads && IsRolloutFileUnreadableError(error)) - { - unreadablePaths.Add(rolloutPath); - continue; - } + currentModelBackups = contentScan.TurnContextModels; + currentModels = currentModelBackups + .SelectMany(static backup => backup.OriginalModels.Count > 0 + ? backup.OriginalModels + : [backup.OriginalModel]) + .ToArray(); + modelChanged = currentModels.Any(model => !string.Equals(model, targetModel, StringComparison.Ordinal)); } if (providerChanged || modelChanged) { FileSnapshot snapshot = GetFileSnapshot(rolloutPath); + if (contentScan.ContentFingerprint is not null && snapshot != scanStart) + { + throw new CoreWritePlanStaleException(); + } if (providerChanged) { payload["model_provider"] = targetProvider; @@ -148,7 +158,10 @@ record = await ReadFirstLineRecordAsync(rolloutPath); OriginalProvider = currentProvider, UpdatedFirstLine = providerChanged ? root!.ToJsonString() : record.FirstLine, ModelOnlyChange = !providerChanged && modelChanged, - OriginalTurnContextModels = currentModelBackups + OriginalTurnContextModels = currentModelBackups, + ContentFingerprint = contentScan.ContentFingerprint is null + ? null + : $"sha256:{contentScan.ContentFingerprint}:{snapshot.Length}:{snapshot.LastWriteTimeUtcTicks}" }); } } @@ -170,7 +183,15 @@ record = await ReadFirstLineRecordAsync(rolloutPath); ArchivedSessions = encryptedArchivedCounts }, UserEventThreadIds = userEventThreadIds, - ThreadCwdsById = threadCwdsById + ThreadCwdsById = threadCwdsById, + ScanMetrics = new SessionScanMetrics + { + EnumeratedRolloutFiles = enumeratedRolloutFiles, + ParsedSessionFiles = parsedSessionFiles, + ContentScanPasses = contentScanPasses, + ModelScanFiles = modelScanFiles, + DurationMs = (long)Math.Round(Stopwatch.GetElapsedTime(scanStarted).TotalMilliseconds) + } }; } @@ -463,12 +484,20 @@ private static async Task ReadFirstLineRecordAsync(FileStream s bool crlf = newlineIndex > 0 && current[newlineIndex - 1] == '\r'; int lineLength = crlf ? newlineIndex - 1 : newlineIndex; string firstLine = Encoding.UTF8.GetString(current[..lineLength]); - return new FirstLineRecord(firstLine, crlf ? "\r\n" : "\n", newlineIndex + 1); + return new FirstLineRecord( + firstLine, + crlf ? "\r\n" : "\n", + newlineIndex + 1, + current[..(newlineIndex + 1)].ToArray()); } } string text = Encoding.UTF8.GetString(collected.GetBuffer(), 0, (int)collected.Length); - return new FirstLineRecord(text, string.Empty, (int)collected.Length); + return new FirstLineRecord( + text, + string.Empty, + (int)collected.Length, + collected.GetBuffer().AsSpan(0, (int)collected.Length).ToArray()); } finally { @@ -499,27 +528,72 @@ private static async Task ReadFirstLineRecordAsync(FileStream s "\"model\"\\s*:\\s*\"((?:[^\"\\\\]|\\\\.)*)\"", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static async Task> ReadTurnContextModelBackupsAsync( + private static async Task ScanRolloutContentAsync( string rolloutPath, - FirstLineRecord record) + FirstLineRecord record, + bool collectModels, + bool collectFingerprint) { List backups = []; + bool hasEncryptedContent = record.FirstLine.Contains("encrypted_content", StringComparison.Ordinal); + bool hasUserEvent = false; + try + { + hasUserEvent = RecordHasUserEvent(JsonNode.Parse(record.FirstLine)); + } + catch + { + // Keep scanning the rest of the rollout below. + } try { await using FileStream stream = new( rolloutPath, FileMode.Open, FileAccess.Read, - FileShare.ReadWrite | FileShare.Delete); + FileShare.Read, + 64 * 1024, + FileOptions.Asynchronous | FileOptions.SequentialScan); stream.Seek(record.Offset, SeekOrigin.Begin); - using StreamReader reader = new(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: false, bufferSize: 4096, leaveOpen: true); + using IncrementalHash? contentHash = collectFingerprint + ? IncrementalHash.CreateHash(HashAlgorithmName.SHA256) + : null; + contentHash?.AppendData(record.PrefixBytes); + using HashingReadStream hashingStream = new(stream, contentHash); + using StreamReader reader = new( + hashingStream, + Encoding.UTF8, + detectEncodingFromByteOrderMarks: false, + bufferSize: 64 * 1024, + leaveOpen: true); int lineIndex = 1; string? line; while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null) { - if (!TurnContextTypeRegex.IsMatch(line)) + if (!hasEncryptedContent + && line.Contains("encrypted_content", StringComparison.Ordinal)) + { + hasEncryptedContent = true; + } + if (!hasUserEvent && !string.IsNullOrWhiteSpace(line)) + { + try + { + hasUserEvent = RecordHasUserEvent(JsonNode.Parse(line)); + } + catch + { + // Ignore malformed non-metadata lines; positive evidence is sufficient. + } + } + + if (!collectModels || !TurnContextTypeRegex.IsMatch(line)) { lineIndex += 1; + if (!collectModels && !collectFingerprint && hasEncryptedContent && hasUserEvent) + { + break; + } continue; } @@ -550,7 +624,14 @@ private static async Task> ReadTurnContext } lineIndex += 1; } - return backups; + string? contentFingerprint = contentHash is null + ? null + : Convert.ToHexString(contentHash.GetHashAndReset()).ToLowerInvariant(); + return new RolloutContentScan( + hasEncryptedContent, + hasUserEvent, + backups, + contentFingerprint); } catch (Exception error) when (IsRolloutFileBusyError(error)) { @@ -927,188 +1008,6 @@ private static FileSnapshot GetFileSnapshot(string filePath) return new FileSnapshot(fileInfo.Length, fileInfo.LastWriteTimeUtc.Ticks); } - private static async Task FileContainsTextAsync(string filePath, string text, int startOffset) - { - byte[] needle = Encoding.UTF8.GetBytes(text); - byte[] buffer = ArrayPool.Shared.Rent(ScanBufferSize); - byte[] tail = []; - - try - { - await using FileStream stream = new( - filePath, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - ScanBufferSize, - FileOptions.Asynchronous | FileOptions.SequentialScan); - - if (startOffset > 0) - { - stream.Seek(startOffset, SeekOrigin.Begin); - } - - while (true) - { - int bytesRead = await stream.ReadAsync(buffer.AsMemory(0, ScanBufferSize)); - if (bytesRead == 0) - { - return false; - } - - byte[] haystack = buffer; - int haystackLength = bytesRead; - if (tail.Length > 0) - { - haystackLength = tail.Length + bytesRead; - haystack = ArrayPool.Shared.Rent(haystackLength); - Buffer.BlockCopy(tail, 0, haystack, 0, tail.Length); - Buffer.BlockCopy(buffer, 0, haystack, tail.Length, bytesRead); - } - - try - { - if (ContainsNeedle(haystack, haystackLength, needle)) - { - return true; - } - - int keepBytes = Math.Min(Math.Max(0, needle.Length - 1), haystackLength); - if (keepBytes == 0) - { - tail = []; - } - else - { - tail = new byte[keepBytes]; - Buffer.BlockCopy(haystack, haystackLength - keepBytes, tail, 0, keepBytes); - } - } - finally - { - if (!ReferenceEquals(haystack, buffer)) - { - ArrayPool.Shared.Return(haystack); - } - } - } - } - catch (Exception error) - { - throw WrapRolloutFileBusyError(error, filePath, "scan"); - } - finally - { - ArrayPool.Shared.Return(buffer); - } - } - - private static bool ContainsNeedle(byte[] haystack, int haystackLength, byte[] needle) - { - if (needle.Length == 0) - { - return true; - } - - if (haystackLength < needle.Length) - { - return false; - } - - int lastStart = haystackLength - needle.Length; - for (int index = 0; index <= lastStart; index += 1) - { - bool match = true; - for (int needleIndex = 0; needleIndex < needle.Length; needleIndex += 1) - { - if (haystack[index + needleIndex] != needle[needleIndex]) - { - match = false; - break; - } - } - - if (match) - { - return true; - } - } - - return false; - } - - private static async Task FileHasEncryptedContentAsync(string filePath, string firstLine, int startOffset) - { - if (firstLine.Contains("encrypted_content", StringComparison.Ordinal)) - { - return true; - } - - return await FileContainsTextAsync(filePath, "encrypted_content", startOffset); - } - - private static async Task FileHasUserEventAsync(string filePath, string firstLine, int startOffset) - { - try - { - if (RecordHasUserEvent(JsonNode.Parse(firstLine))) - { - return true; - } - } - catch - { - // Keep scanning the rest of the rollout below. - } - - try - { - await using FileStream stream = new( - filePath, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - 64 * 1024, - FileOptions.Asynchronous | FileOptions.SequentialScan); - if (startOffset > 0) - { - stream.Seek(startOffset, SeekOrigin.Begin); - } - - using StreamReader reader = new( - stream, - Encoding.UTF8, - detectEncodingFromByteOrderMarks: true, - bufferSize: 64 * 1024, - leaveOpen: false); - while (await reader.ReadLineAsync() is string rawLine) - { - if (string.IsNullOrWhiteSpace(rawLine)) - { - continue; - } - - try - { - if (RecordHasUserEvent(JsonNode.Parse(rawLine))) - { - return true; - } - } - catch - { - // Ignore malformed non-metadata lines; provider sync only needs positive evidence. - } - } - - return false; - } - catch (Exception error) - { - throw WrapRolloutFileBusyError(error, filePath, "scan"); - } - } - private static bool RecordHasUserEvent(JsonNode? record) { if (record is not JsonObject root) @@ -1250,8 +1149,77 @@ private static Exception WrapRolloutFileBusyError(Exception error, string filePa error); } - private readonly record struct FirstLineRecord(string FirstLine, string Separator, int Offset); + private readonly record struct FirstLineRecord( + string FirstLine, + string Separator, + int Offset, + byte[] PrefixBytes); private readonly record struct FileSnapshot(long Length, long LastWriteTimeUtcTicks); + private readonly record struct RolloutContentScan( + bool HasEncryptedContent, + bool HasUserEvent, + IReadOnlyList TurnContextModels, + string? ContentFingerprint); + + private sealed class HashingReadStream(Stream inner, IncrementalHash? hash) : Stream + { + public override bool CanRead => inner.CanRead; + public override bool CanSeek => inner.CanSeek; + public override bool CanWrite => false; + public override long Length => inner.Length; + public override long Position { get => inner.Position; set => inner.Position = value; } + public override void Flush() => inner.Flush(); + public override int Read(byte[] buffer, int offset, int count) + { + int read = inner.Read(buffer, offset, count); + if (read > 0) + { + hash?.AppendData(buffer, offset, read); + } + return read; + } + public override int Read(Span buffer) + { + int read = inner.Read(buffer); + if (read > 0) + { + hash?.AppendData(buffer[..read]); + } + return read; + } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken) + { + int read = await inner.ReadAsync(buffer.AsMemory(offset, count), cancellationToken); + if (read > 0) + { + hash?.AppendData(buffer, offset, read); + } + return read; + } + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default) + { + int read = await inner.ReadAsync(buffer, cancellationToken); + if (read > 0) + { + hash?.AppendData(buffer.Span[..read]); + } + return read; + } + public override long Seek(long offset, SeekOrigin origin) => inner.Seek(offset, origin); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + protected override void Dispose(bool disposing) + { + // The caller owns the underlying rollout stream. + base.Dispose(disposing); + } + } private readonly record struct ModelLineRewrite( string Line, bool Replaced, diff --git a/desktop/CodexProviderSync.Core/TextFormatter.cs b/desktop/CodexProviderSync.Core/TextFormatter.cs index 291d9d9..83f5d41 100644 --- a/desktop/CodexProviderSync.Core/TextFormatter.cs +++ b/desktop/CodexProviderSync.Core/TextFormatter.cs @@ -27,6 +27,34 @@ public static string FormatSyncResult(SyncResult result, string label, string la : FormatSyncResultEnglish(result, label); } + public static string FormatPerformanceMetrics(SyncPerformanceMetrics metrics, string language) + { + bool chinese = IsChinese(language); + return chinese + ? $"性能: 总计={metrics.TotalDurationMs}ms; 准备={metrics.PreparationDurationMs}ms; " + + $"计划校验={metrics.CheckedPlanValidationDurationMs}ms; 备份={metrics.BackupDurationMs}ms; " + + $"写入={metrics.MutationDurationMs}ms; 清理={metrics.PruneDurationMs}ms; " + + $"rollout={metrics.RolloutScan.EnumeratedRolloutFiles}; 内容扫描={metrics.RolloutScan.ContentScanPasses}; " + + $"journal全量校验={metrics.JournalFullValidationCount}" + : $"Performance: total={metrics.TotalDurationMs}ms; prepare={metrics.PreparationDurationMs}ms; " + + $"plan-check={metrics.CheckedPlanValidationDurationMs}ms; backup={metrics.BackupDurationMs}ms; " + + $"mutation={metrics.MutationDurationMs}ms; prune={metrics.PruneDurationMs}ms; " + + $"rollouts={metrics.RolloutScan.EnumeratedRolloutFiles}; content-scans={metrics.RolloutScan.ContentScanPasses}; " + + $"journal-full-validations={metrics.JournalFullValidationCount}"; + } + + public static string FormatPerformanceMetrics(StatusPerformanceMetrics metrics, string language) + { + bool chinese = IsChinese(language); + return chinese + ? $"刷新性能: 总计={metrics.TotalDurationMs}ms; rollout扫描={metrics.RolloutScanDurationMs}ms; " + + $"备份统计={metrics.BackupSummaryDurationMs}ms; rollout={metrics.RolloutScan.EnumeratedRolloutFiles}; " + + $"内容扫描={metrics.RolloutScan.ContentScanPasses}" + : $"Refresh performance: total={metrics.TotalDurationMs}ms; rollout-scan={metrics.RolloutScanDurationMs}ms; " + + $"backup-summary={metrics.BackupSummaryDurationMs}ms; rollouts={metrics.RolloutScan.EnumeratedRolloutFiles}; " + + $"content-scans={metrics.RolloutScan.ContentScanPasses}"; + } + public static string FormatRestoreResult(RestoreResult result) => FormatRestoreResult(result, English); diff --git a/desktop/CodexProviderSync.Core/TransactionJournalService.cs b/desktop/CodexProviderSync.Core/TransactionJournalService.cs index 6f2a94c..2c4e608 100644 --- a/desktop/CodexProviderSync.Core/TransactionJournalService.cs +++ b/desktop/CodexProviderSync.Core/TransactionJournalService.cs @@ -3,7 +3,7 @@ namespace CodexProviderSync.Core; -internal sealed class FileTransactionJournal +internal sealed class FileTransactionJournal : IAsyncDisposable { internal const string FileName = "transaction-journal.jsonl"; private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); @@ -11,20 +11,46 @@ internal sealed class FileTransactionJournal private readonly string _filePath; private readonly string _operationId; private readonly SemaphoreSlim _appendGate = new(1, 1); + private FileStream? _writerLease; private int _sequence; + private PendingTransactionInfo? _current; + private long _expectedLength; + private byte[] _expectedTailRecord = []; + private bool _disposed; - private FileTransactionJournal(string filePath, string operationId, int sequence = 0) + private FileTransactionJournal( + string filePath, + string operationId, + PendingTransactionInfo? current = null, + FileStream? writerLease = null) { _filePath = filePath; _operationId = operationId; - _sequence = sequence; + _current = current; + _writerLease = writerLease; + _sequence = current?.LastSequence ?? 0; + _expectedLength = current is null ? 0 : -1; } internal string FilePath => _filePath; + internal int AppendFullJournalValidationCount { get; private set; } + internal Func? AppendFaultInjector { get; set; } - internal Task ReadCurrentInfoAsync() => ReadInfoAsync(_filePath); + internal async Task ReadCurrentInfoAsync() + { + await _appendGate.WaitAsync(); + try + { + ObjectDisposedException.ThrowIf(_disposed, this); + return (await ReadJournalForCurrentInstanceAsync()).Info; + } + finally + { + _appendGate.Release(); + } + } internal static async Task CreateAsync( string backupDir, @@ -32,23 +58,62 @@ internal static async Task CreateAsync( string targetProvider, IEnumerable potentialTargets) { + return await CreateCoreAsync( + backupDir, + codexHome, + targetProvider, + potentialTargets, + acquireWriterLease: false); + } + + internal static async Task CreateOwnedAsync( + string backupDir, + string codexHome, + string targetProvider, + IEnumerable potentialTargets) + { + return await CreateCoreAsync( + backupDir, + codexHome, + targetProvider, + potentialTargets, + acquireWriterLease: OperatingSystem.IsWindows()); + } + + private static async Task CreateCoreAsync( + string backupDir, + string codexHome, + string targetProvider, + IEnumerable potentialTargets, + bool acquireWriterLease) + { + string filePath = Path.Combine(backupDir, FileName); string operationId = Guid.NewGuid().ToString("D"); - FileTransactionJournal journal = new( - Path.Combine(backupDir, FileName), - operationId); - await journal.AppendAsync("prepared", new Dictionary - { - ["protocolVersion"] = 1, - ["backupDir"] = Path.GetFullPath(backupDir), - ["codexHome"] = Path.GetFullPath(codexHome), - ["targetProvider"] = targetProvider, - ["potentialTargets"] = potentialTargets - .Select(Path.GetFullPath) - .Distinct(PathComparer) - .Order(PathComparer) - .ToArray() - }); - return journal; + FileStream? writerLease = acquireWriterLease + ? OpenWriterLease(filePath, FileMode.CreateNew) + : null; + FileTransactionJournal journal = new(filePath, operationId, writerLease: writerLease); + try + { + await journal.AppendAsync("prepared", new Dictionary + { + ["protocolVersion"] = 1, + ["backupDir"] = Path.GetFullPath(backupDir), + ["codexHome"] = Path.GetFullPath(codexHome), + ["targetProvider"] = targetProvider, + ["potentialTargets"] = potentialTargets + .Select(Path.GetFullPath) + .Distinct(PathComparer) + .Order(PathComparer) + .ToArray() + }); + return journal; + } + catch + { + await journal.DisposeAsync(); + throw; + } } internal Task ApplyingAsync(string kind, string targetPath) => AppendAsync( @@ -96,29 +161,9 @@ private async Task AppendAsync(string state, IReadOnlyDictionary 0) - { - before = await ReadInfoAsync(_filePath); - if (before.InvalidTail) - { - throw new InvalidOperationException( - state == "committed" - ? $"Transaction journal is invalid and cannot commit until recovery: {_filePath}" - : $"Transaction journal is invalid and requires recovery before append: {_filePath}"); - } - if (!string.Equals(before.OperationId, _operationId, StringComparison.Ordinal)) - { - throw new InvalidOperationException( - $"Transaction journal operationId changed before append: {_filePath}"); - } - _sequence = before.LastSequence; - } - else if (_sequence != 0) - { - throw new InvalidOperationException( - $"Transaction journal disappeared after it was created: {_filePath}"); - } + ObjectDisposedException.ThrowIf(_disposed, this); + bool terminal = state is "committed" or "rolledBack"; + PendingTransactionInfo? before = await EnsureJournalFrontierAsync(forceFullValidation: terminal); ValidateAppendTransition(before, state, details); int nextSequence = _sequence + 1; @@ -141,20 +186,38 @@ private async Task AppendAsync(string state, IReadOnlyDictionary EnsureJournalFrontierAsync(bool forceFullValidation) + { + if (!File.Exists(_filePath)) + { + if (_sequence != 0 || _current is not null) + { + throw new InvalidOperationException( + $"Transaction journal disappeared after it was created: {_filePath}"); + } + _expectedLength = 0; + _expectedTailRecord = []; + return null; + } + + long actualLength = new FileInfo(_filePath).Length; + if (actualLength == 0 && _sequence == 0 && _current is null) + { + _expectedLength = 0; + _expectedTailRecord = []; + return null; + } + bool frontierMatches = _writerLease is not null + && OperatingSystem.IsWindows() + && !forceFullValidation + && actualLength == _expectedLength + && await TailMatchesAsync(actualLength, _expectedTailRecord); + if (!frontierMatches) + { + JournalReadResult read = await ReadJournalForAppendAsync(); + AdoptJournalState(read); + } + + if (_current is null) + { + return null; + } + if (_current.InvalidTail) + { + throw new InvalidOperationException( + forceFullValidation + ? $"Transaction journal is invalid and cannot commit until recovery: {_filePath}" + : $"Transaction journal is invalid and requires recovery before append: {_filePath}"); + } + if (!string.Equals(_current.OperationId, _operationId, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Transaction journal operationId changed before append: {_filePath}"); + } + _sequence = _current.LastSequence; + return _current; + } + + private async Task ReadJournalForAppendAsync() + { + AppendFullJournalValidationCount += 1; + return await ReadJournalForCurrentInstanceAsync(); + } + + private Task ReadJournalForCurrentInstanceAsync() + { + return _writerLease is null + ? ReadJournalAsync(_filePath) + : ReadJournalAsync(_filePath, _writerLease); + } + + private void AdoptJournalState(JournalReadResult read) + { + _current = read.Info; + _sequence = read.Info.LastSequence; + _expectedLength = _writerLease?.Length ?? new FileInfo(_filePath).Length; + _expectedTailRecord = read.ValidLines.Count == 0 + ? [] + : Encoding.UTF8.GetBytes(read.ValidLines[^1] + "\n"); + } + + private async Task VerifyAppendedRecordAsync(long appendOffset, byte[] expectedRecord) + { + if (_writerLease is not null) + { + long leasedExpectedLength = appendOffset + expectedRecord.Length; + if (_writerLease.Length != leasedExpectedLength) + { + throw new InvalidOperationException( + $"Transaction journal changed while appending: {_filePath}"); + } + _writerLease.Seek(appendOffset, SeekOrigin.Begin); + byte[] leasedActual = new byte[expectedRecord.Length]; + await _writerLease.ReadExactlyAsync(leasedActual); + if (!leasedActual.AsSpan().SequenceEqual(expectedRecord)) + { + throw new InvalidOperationException( + $"Transaction journal append bytes could not be verified: {_filePath}"); + } + return; + } + + await using FileStream stream = new( + _filePath, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + 4096, + FileOptions.Asynchronous | FileOptions.SequentialScan); + long expectedLength = appendOffset + expectedRecord.Length; + if (stream.Length != expectedLength) + { + throw new InvalidOperationException( + $"Transaction journal changed while appending: {_filePath}"); + } + + stream.Seek(appendOffset, SeekOrigin.Begin); + byte[] actual = new byte[expectedRecord.Length]; + await stream.ReadExactlyAsync(actual); + if (!actual.AsSpan().SequenceEqual(expectedRecord)) + { + throw new InvalidOperationException( + $"Transaction journal append bytes could not be verified: {_filePath}"); + } + } + + private async Task TailMatchesAsync(long actualLength, byte[] expectedTail) + { + if (actualLength == 0) + { + return expectedTail.Length == 0; + } + if (expectedTail.Length == 0 || actualLength < expectedTail.Length) + { + return false; + } + + if (_writerLease is not null) + { + if (_writerLease.Length != actualLength) + { + return false; + } + _writerLease.Seek(actualLength - expectedTail.Length, SeekOrigin.Begin); + byte[] leasedTail = new byte[expectedTail.Length]; + await _writerLease.ReadExactlyAsync(leasedTail); + return leasedTail.AsSpan().SequenceEqual(expectedTail); + } + + await using FileStream stream = new( + _filePath, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete, + 4096, + FileOptions.Asynchronous | FileOptions.SequentialScan); + if (stream.Length != actualLength) + { + return false; + } + stream.Seek(actualLength - expectedTail.Length, SeekOrigin.Begin); + byte[] actualTail = new byte[expectedTail.Length]; + await stream.ReadExactlyAsync(actualTail); + return actualTail.AsSpan().SequenceEqual(expectedTail); + } + + public async ValueTask DisposeAsync() + { + await _appendGate.WaitAsync(); + try + { + if (_disposed) + { + return; + } + _disposed = true; + FileStream? writerLease = Interlocked.Exchange(ref _writerLease, null); + if (writerLease is not null) + { + await writerLease.DisposeAsync(); + } + } + finally + { + _appendGate.Release(); + } + } + + private static FileStream OpenWriterLease(string filePath, FileMode mode) + { + return new FileStream( + filePath, + mode, + FileAccess.ReadWrite, + FileShare.Read, + 4096, + FileOptions.Asynchronous | FileOptions.WriteThrough | FileOptions.SequentialScan); + } + + private PendingTransactionInfo AdvanceJournalState( + PendingTransactionInfo? current, + string nextState, + IReadOnlyDictionary? details, + int nextSequence) + { + IReadOnlyList potentialTargets = current?.PotentialTargets + ?? ReadPreparedPotentialTargets(details); + Dictionary affected = new(PathComparer); + if (current is not null) + { + foreach (TransactionTargetInfo target in current.AffectedTargets) + { + affected[target.Kind + "\0" + Path.GetFullPath(target.TargetPath)] = target; + } + } + + if (nextState is "applying" or "applied" or "skipped") + { + string kind = ReadRequiredDetail(details, "kind"); + string targetPath = Path.GetFullPath(ReadRequiredDetail(details, "targetPath")); + string key = kind + "\0" + targetPath; + if (nextState == "skipped") + { + affected.Remove(key); + } + else + { + affected[key] = new TransactionTargetInfo(kind, targetPath, nextState); + } + } + + bool terminal = nextState is "committed" or "rolledBack"; + string journalPath = current?.JournalPath ?? _filePath; + return new PendingTransactionInfo( + journalPath, + current?.BackupDir ?? Path.GetDirectoryName(_filePath)!, + current?.OperationId ?? _operationId, + nextSequence, + nextState, + terminal, + InvalidTail: false, + LastValidState: nextState, + potentialTargets, + affected.Values.ToArray()); + } + + private static IReadOnlyList ReadPreparedPotentialTargets( + IReadOnlyDictionary? details) + { + if (details is not null + && details.TryGetValue("potentialTargets", out object? value) + && value is IEnumerable targets) + { + return targets + .Select(Path.GetFullPath) + .Distinct(PathComparer) + .Order(PathComparer) + .ToArray(); + } + return []; + } + private static void ValidateAppendTransition( PendingTransactionInfo? current, string nextState, @@ -436,7 +786,7 @@ await replacement.RollingBackAsync( FileTransactionJournal journal = new( journalPath, info.OperationId!, - info.LastSequence); + info); if (info.State is not ("rollingBack" or "recoveryRequired")) { await journal.RollingBackAsync(new InvalidOperationException("Explicit managed-backup restore")); @@ -465,6 +815,23 @@ internal static async Task ReadInfoAsync(string journalP private static async Task ReadJournalAsync(string journalPath) { byte[] journalBytes = await File.ReadAllBytesAsync(journalPath); + return ParseJournal(journalPath, journalBytes); + } + + private static async Task ReadJournalAsync(string journalPath, FileStream stream) + { + if (stream.Length > int.MaxValue) + { + throw new InvalidOperationException($"Transaction journal is too large to validate: {journalPath}"); + } + byte[] journalBytes = new byte[(int)stream.Length]; + stream.Seek(0, SeekOrigin.Begin); + await stream.ReadExactlyAsync(journalBytes); + return ParseJournal(journalPath, journalBytes); + } + + private static JournalReadResult ParseJournal(string journalPath, byte[] journalBytes) + { bool missingTerminalLf = journalBytes.Length > 0 && journalBytes[^1] != (byte)'\n'; string journalText = Encoding.UTF8.GetString(journalBytes); string? operationId = null; diff --git a/src/backup.js b/src/backup.js index 8d0ca4c..c7f21ce 100644 --- a/src/backup.js +++ b/src/backup.js @@ -322,26 +322,18 @@ export async function createBackup({ "utf8" ); - await writeFileAtomic( - path.join(backupDir, "metadata.json"), - JSON.stringify( - { - version: 2, - namespace: BACKUP_NAMESPACE, - codexHome, - sqliteHome: actualSqliteHome, - targetProvider, - createdAt: sessionManifest.createdAt, - dbFiles: copiedDbFiles, - sqliteDbFiles: copiedSqliteDbFiles, - globalStateFiles, - changedSessionFiles: sessionChanges.length - }, - null, - 2 - ), - "utf8" - ); + await writeMetadataWithInventory(backupDir, { + version: 2, + namespace: BACKUP_NAMESPACE, + codexHome, + sqliteHome: actualSqliteHome, + targetProvider, + createdAt: sessionManifest.createdAt, + dbFiles: copiedDbFiles, + sqliteDbFiles: copiedSqliteDbFiles, + globalStateFiles, + changedSessionFiles: sessionChanges.length + }); return backupDir; } @@ -382,12 +374,9 @@ export async function updateSessionBackupManifest(backupDir, sessionChanges, opt "utf8", { faultInjector: options.faultInjector } ); - await writeFileAtomic( - metadataPath, - JSON.stringify(metadata, null, 2), - "utf8", - { faultInjector: options.faultInjector } - ); + await writeMetadataWithInventory(backupDir, metadata, { + faultInjector: options.faultInjector + }); } export async function getBackupSummary(codexHome) { @@ -395,7 +384,7 @@ export async function getBackupSummary(codexHome) { const backupDirs = await listManagedBackupDirectories(backupRoot); let totalBytes = 0; for (const entry of backupDirs) { - totalBytes += await getDirectorySize(entry.fullPath); + totalBytes += await getBackupDirectorySize(entry.fullPath); } return { @@ -420,7 +409,7 @@ export async function pruneBackups(codexHome, keepCount = DEFAULT_BACKUP_RETENTI .filter((entry) => !protectedBackups.has(pathComparisonKey(entry.fullPath))); let freedBytes = 0; for (const entry of toDelete) { - freedBytes += await getDirectorySize(entry.fullPath); + freedBytes += await getBackupDirectorySize(entry.fullPath); await fs.rm(entry.fullPath, { recursive: true, force: true }); } @@ -715,29 +704,77 @@ async function isManagedBackupDirectory(backupDir) { } } +async function writeMetadataWithInventory(backupDir, metadata, options = {}) { + const metadataPath = path.join(backupDir, "metadata.json"); + const payload = await getDirectoryInventory(backupDir, metadataPath); + const fileCount = payload.fileCount + 1; + let sizeBytes = 0; + let serialized = ""; + for (let attempt = 0; attempt < 8; attempt += 1) { + serialized = JSON.stringify({ ...metadata, sizeBytes, fileCount }, null, 2); + const nextSizeBytes = payload.sizeBytes + Buffer.byteLength(serialized, "utf8"); + if (nextSizeBytes === sizeBytes) { + await writeFileAtomic(metadataPath, serialized, "utf8", options); + return; + } + sizeBytes = nextSizeBytes; + } + throw new Error(`Backup metadata inventory did not converge: ${metadataPath}`); +} + +async function getBackupDirectorySize(backupDir) { + try { + const metadata = JSON.parse( + await fs.readFile(path.join(backupDir, "metadata.json"), "utf8") + ); + if (metadata?.namespace === BACKUP_NAMESPACE + && Number.isSafeInteger(metadata.sizeBytes) + && metadata.sizeBytes >= 0 + && Number.isSafeInteger(metadata.fileCount) + && metadata.fileCount >= 1) { + return metadata.sizeBytes; + } + } catch { + // Older or damaged inventory fields fall back to the recursive scan below. + } + return getDirectorySize(backupDir); +} + async function getDirectorySize(directoryPath) { + return (await getDirectoryInventory(directoryPath)).sizeBytes; +} + +async function getDirectoryInventory(directoryPath, excludedFilePath = null) { let entries; try { entries = await fs.readdir(directoryPath, { withFileTypes: true }); } catch (error) { if (error?.code === "ENOENT") { - return 0; + return { sizeBytes: 0, fileCount: 0 }; } throw error; } - let total = 0; + let sizeBytes = 0; + let fileCount = 0; + const excluded = excludedFilePath === null ? null : path.resolve(excludedFilePath); for (const entry of entries) { const fullPath = path.join(directoryPath, entry.name); if (entry.isDirectory()) { - total += await getDirectorySize(fullPath); + const child = await getDirectoryInventory(fullPath, excluded); + sizeBytes += child.sizeBytes; + fileCount += child.fileCount; continue; } if (entry.isFile()) { + if (excluded !== null && path.resolve(fullPath) === excluded) { + continue; + } const stat = await fs.stat(fullPath); - total += stat.size; + sizeBytes += stat.size; + fileCount += 1; } } - return total; + return { sizeBytes, fileCount }; } diff --git a/test/sync-service.test.js b/test/sync-service.test.js index 2f7d2b5..5259b09 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -1637,6 +1637,10 @@ test("runSync rewrites rollout files and sqlite, then restore reverts both", asy assert.equal(backupMetadata.version, 2); assert.equal(backupMetadata.sqliteHome, path.join(codexHome, SQLITE_DIR_BASENAME)); assert.deepEqual(backupMetadata.sqliteDbFiles, [DB_FILE_BASENAME]); + assert.ok(Number.isSafeInteger(backupMetadata.sizeBytes)); + assert.ok(backupMetadata.sizeBytes > 0); + assert.ok(Number.isSafeInteger(backupMetadata.fileCount)); + assert.ok(backupMetadata.fileCount > 0); assert.deepEqual( backupMetadata.dbFiles.map((fileName) => fileName.replaceAll("\\", "/")), ["sqlite/state_5.sqlite"] @@ -3609,6 +3613,33 @@ test("pruneBackups removes the oldest backup directories", async () => { await fs.access(path.join(backupRoot(codexHome), "20260321T000000000Z")); }); +test("backup summary and prune use cached inventory with legacy fallback", async () => { + const { codexHome } = await makeTempCodexHome(); + const cachedDir = path.join(backupRoot(codexHome), "20260319T000000000Z"); + const legacyDir = path.join(backupRoot(codexHome), "20260320T000000000Z"); + await fs.mkdir(cachedDir, { recursive: true }); + await fs.mkdir(legacyDir, { recursive: true }); + await fs.writeFile(path.join(cachedDir, "metadata.json"), JSON.stringify({ + namespace: "provider-sync", + sizeBytes: 123, + fileCount: 1 + }), "utf8"); + await fs.writeFile(path.join(cachedDir, "added-later.bin"), "x".repeat(4096), "utf8"); + await fs.writeFile(path.join(legacyDir, "metadata.json"), JSON.stringify({ + namespace: "provider-sync" + }), "utf8"); + await fs.writeFile(path.join(legacyDir, "payload.bin"), "legacy", "utf8"); + const legacyBytes = (await fs.stat(path.join(legacyDir, "metadata.json"))).size + + (await fs.stat(path.join(legacyDir, "payload.bin"))).size; + + const summary = await getBackupSummary(codexHome); + assert.deepEqual(summary, { count: 2, totalBytes: 123 + legacyBytes }); + + const pruned = await pruneBackups(codexHome, 1); + assert.equal(pruned.deletedCount, 1); + assert.equal(pruned.freedBytes, 123); +}); + test("pruneBackups ignores directories without managed backup metadata", async () => { const { codexHome } = await makeTempCodexHome(); await writeBackup(codexHome, "20260320T000000000Z", [ From f487720a4311115be40f4dc260d6b9c731e1f5a9 Mon Sep 17 00:00:00 2001 From: Dailin521 <71995555+Dailin521@users.noreply.github.com> Date: Fri, 7 Aug 2026 19:50:26 +0800 Subject: [PATCH 2/5] fix: refresh backup inventory after journal completion --- .../CoreIntegrationTests.cs | 11 ++++- .../CodexProviderSync.Core/BackupService.cs | 16 +++++++ .../CodexSyncService.cs | 2 + src/backup.js | 10 +++++ src/service.js | 3 ++ test/sync-service.test.js | 45 ++++++++++++++++++- 6 files changed, 84 insertions(+), 3 deletions(-) diff --git a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs index 2ed819f..e9d7f7d 100644 --- a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs @@ -962,10 +962,19 @@ await fixture.WriteStateDbAsync( Assert.Equal(2, syncResult.SqliteRowsUpdated); Assert.Equal(2, syncResult.PerformanceMetrics.RolloutScan.EnumeratedRolloutFiles); Assert.Equal(2, syncResult.PerformanceMetrics.RolloutScan.ContentScanPasses); - Assert.Equal(2, syncResult.PerformanceMetrics.JournalFullValidationCount); + Assert.Equal( + OperatingSystem.IsWindows() ? 2 : 8, + syncResult.PerformanceMetrics.JournalFullValidationCount); BackupMetadataFile backupMetadata = JsonSerializer.Deserialize( await File.ReadAllTextAsync(Path.Combine(syncResult.BackupDir, "metadata.json")), new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase })!; + Assert.Equal( + Directory.EnumerateFiles(syncResult.BackupDir, "*", SearchOption.AllDirectories) + .Sum(static path => new FileInfo(path).Length), + backupMetadata.SizeBytes); + Assert.Equal( + Directory.EnumerateFiles(syncResult.BackupDir, "*", SearchOption.AllDirectories).Count(), + backupMetadata.FileCount); Assert.Equal( [ Path.Combine(AppConstants.SqliteDirBasename, AppConstants.DbFileBasename) diff --git a/desktop/CodexProviderSync.Core/BackupService.cs b/desktop/CodexProviderSync.Core/BackupService.cs index 0cfdc35..8322717 100644 --- a/desktop/CodexProviderSync.Core/BackupService.cs +++ b/desktop/CodexProviderSync.Core/BackupService.cs @@ -471,6 +471,22 @@ await File.ReadAllTextAsync(metadataPath), }; } + internal async Task RefreshMetadataInventoryAsync(string backupDir) + { + string normalizedBackupDir = Path.GetFullPath(backupDir); + string metadataPath = Path.Combine(normalizedBackupDir, "metadata.json"); + BackupMetadataFile metadata = JsonSerializer.Deserialize( + await File.ReadAllTextAsync(metadataPath), + JsonOptions()) ?? throw new InvalidOperationException($"Backup metadata is invalid: {backupDir}"); + if (!string.Equals(metadata.Namespace, AppConstants.BackupNamespace, StringComparison.Ordinal) + || metadata.Version is not (1 or 2)) + { + throw new InvalidOperationException($"Unsupported backup metadata in {metadataPath}."); + } + + await WriteMetadataWithInventoryAsync(normalizedBackupDir, metadata); + } + public Task GetBackupSummaryAsync(string codexHome) { string backupRoot = AppConstants.DefaultBackupRoot(codexHome); diff --git a/desktop/CodexProviderSync.Core/CodexSyncService.cs b/desktop/CodexProviderSync.Core/CodexSyncService.cs index b8d0aa1..cda7dfc 100644 --- a/desktop/CodexProviderSync.Core/CodexSyncService.cs +++ b/desktop/CodexProviderSync.Core/CodexSyncService.cs @@ -635,6 +635,7 @@ await FaultInjector( await journal.CommittedAsync(); transactionCommitted = true; await journal.DisposeAsync(); + await _backupService.RefreshMetadataInventoryAsync(backupDir); long mutationDurationMs = ElapsedMilliseconds(mutationStarted); if (FaultInjector is not null) { @@ -1203,6 +1204,7 @@ await FileTransactionJournal.MarkBackupRolledBackAsync( preparation.BackupDirectory, codexHome, result.TargetProvider); + await _backupService.RefreshMetadataInventoryAsync(preparation.BackupDirectory); return result; } diff --git a/src/backup.js b/src/backup.js index c7f21ce..2fa3606 100644 --- a/src/backup.js +++ b/src/backup.js @@ -379,6 +379,16 @@ export async function updateSessionBackupManifest(backupDir, sessionChanges, opt }); } +export async function refreshBackupInventory(backupDir, options = {}) { + const normalizedBackupDir = path.resolve(backupDir); + const metadataPath = path.join(normalizedBackupDir, "metadata.json"); + const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); + if (metadata?.namespace !== BACKUP_NAMESPACE || !new Set([1, 2]).has(metadata.version)) { + throw new Error(`Unsupported backup metadata in ${metadataPath}.`); + } + await writeMetadataWithInventory(normalizedBackupDir, metadata, options); +} + export async function getBackupSummary(codexHome) { const backupRoot = defaultBackupRoot(codexHome); const backupDirs = await listManagedBackupDirectories(backupRoot); diff --git a/src/service.js b/src/service.js index eb1c7e4..f9e143b 100644 --- a/src/service.js +++ b/src/service.js @@ -22,6 +22,7 @@ import { getBackupRecoveryCoverage, getBackupSummary, pruneBackups, + refreshBackupInventory, restoreBackup, restoreGlobalStateFilesFromBackup } from "./backup.js"; @@ -643,6 +644,7 @@ async function runSyncCore({ await faultInjector?.({ point: "before_transaction_commit", completedCount: completedTargets.length }); await commitJournalWithReconciliation(journal, faultInjector); transactionCommitted = true; + await refreshBackupInventory(backupDir); await faultInjector?.({ point: "after_transaction_commit", completedCount: completedTargets.length }); let autoPruneResult = null; let autoPruneWarning = null; @@ -1011,6 +1013,7 @@ export async function runRestore({ allowSqliteHomeRelocation }); await markBackupTransactionRolledBack(normalizedBackupDir); + await refreshBackupInventory(normalizedBackupDir); return result; } finally { await releaseLock(); diff --git a/test/sync-service.test.js b/test/sync-service.test.js index 5259b09..2811df0 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -724,7 +724,7 @@ test("repeated sync is idempotent for rollout and SQLite state", async () => { assert.deepEqual(await findPendingTransactions(codexHome), []); }); -test("runSync leaves the backup manifest and metadata unchanged after creation", async () => { +test("runSync leaves the backup manifest and metadata payload unchanged after creation", async () => { const { codexHome } = await makeTempCodexHome(); await writeConfig(codexHome, 'model_provider = "openai"'); const sessionPath = path.join(codexHome, "sessions", "2026", "03", "19", "rollout-immutable-backup.jsonl"); @@ -753,7 +753,27 @@ test("runSync leaves the backup manifest and metadata unchanged after creation", assert.ok(manifestBefore); assert.ok(metadataBefore); assert.deepEqual(await fs.readFile(path.join(backupDir, "session-meta-backup.json")), manifestBefore); - assert.deepEqual(await fs.readFile(path.join(backupDir, "metadata.json")), metadataBefore); + const metadataBeforeValue = JSON.parse(metadataBefore.toString("utf8")); + const metadataAfterValue = JSON.parse( + await fs.readFile(path.join(backupDir, "metadata.json"), "utf8") + ); + const { + sizeBytes: sizeBytesBefore, + fileCount: fileCountBefore, + ...metadataPayloadBefore + } = metadataBeforeValue; + const { + sizeBytes: sizeBytesAfter, + fileCount: fileCountAfter, + ...metadataPayloadAfter + } = metadataAfterValue; + assert.deepEqual(metadataPayloadAfter, metadataPayloadBefore); + assert.ok(sizeBytesAfter > sizeBytesBefore); + assert.equal(fileCountAfter, fileCountBefore + 1); + assert.deepEqual( + { sizeBytes: sizeBytesAfter, fileCount: fileCountAfter }, + await getDirectoryInventory(backupDir) + ); const manifest = JSON.parse(manifestBefore.toString("utf8")); assert.equal(manifest.appliedPaths, null); }); @@ -1416,6 +1436,24 @@ async function writeBackup(codexHome, directoryName, files) { return totalBytes; } +async function getDirectoryInventory(directoryPath) { + const entries = await fs.readdir(directoryPath, { withFileTypes: true }); + let sizeBytes = 0; + let fileCount = 0; + for (const entry of entries) { + const fullPath = path.join(directoryPath, entry.name); + if (entry.isDirectory()) { + const child = await getDirectoryInventory(fullPath); + sizeBytes += child.sizeBytes; + fileCount += child.fileCount; + } else if (entry.isFile()) { + sizeBytes += (await fs.stat(fullPath)).size; + fileCount += 1; + } + } + return { sizeBytes, fileCount }; +} + async function writeConfig(codexHome, modelProviderLine = "") { const config = `${modelProviderLine}${modelProviderLine ? "\n" : ""}sandbox_mode = "danger-full-access"\n\n[model_providers.apigather]\nbase_url = "https://example.com"\n`; await fs.writeFile(path.join(codexHome, "config.toml"), config, "utf8"); @@ -1641,6 +1679,9 @@ test("runSync rewrites rollout files and sqlite, then restore reverts both", asy assert.ok(backupMetadata.sizeBytes > 0); assert.ok(Number.isSafeInteger(backupMetadata.fileCount)); assert.ok(backupMetadata.fileCount > 0); + const backupInventory = await getDirectoryInventory(syncResult.backupDir); + assert.equal(backupMetadata.sizeBytes, backupInventory.sizeBytes); + assert.equal(backupMetadata.fileCount, backupInventory.fileCount); assert.deepEqual( backupMetadata.dbFiles.map((fileName) => fileName.replaceAll("\\", "/")), ["sqlite/state_5.sqlite"] From 54fc74263c64a1a02cad7ee8e28d03a3889c0881 Mon Sep 17 00:00:00 2001 From: Dailin521 <71995555+Dailin521@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:45:28 +0800 Subject: [PATCH 3/5] fix: skip live rollouts and keep committed syncs successful Two regressions from this branch's sync hardening. A rollout that grows while its content digest is being folded threw CoreWritePlanStaleException out of CollectSessionChangesAsync. The skipLockedReads catch filters only match busy and unreadable I/O errors, so an ordinary unchecked sync aborted entirely instead of skipping the file an active Codex session is appending to. Report it as a locked rollout when skipLockedReads is set, matching the documented partial-success contract, and keep throwing on the strict path where a stale hint must not be cached. Refreshing the backup inventory ran unguarded right after the journal reached committed. Because the outer handler rethrows once transactionCommitted is set, a failed metadata.json write reported a fully durable sync as failed and skipped automatic backup pruning. Degrade it to a warning on both the sync and restore paths, in Core and in the Node service. Adds a scan-phase test seam so the mid-scan mutation is exercised deterministically, and covers both fixes on both implementations. --- AGENTS.md | 2 +- .../CoreIntegrationTests.cs | 114 ++++++++++++++++++ .../CodexSyncService.cs | 48 +++++++- desktop/CodexProviderSync.Core/Models.cs | 8 ++ .../SessionRolloutService.cs | 24 ++++ .../CodexProviderSync.Core/TextFormatter.cs | 7 ++ src/service.js | 27 ++++- test/sync-service.test.js | 49 ++++++++ 8 files changed, 273 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 870da3a..8ff672d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,7 +137,7 @@ If the output says Windows cannot safely access SQLite through a WSL UNC path: If sync reports `Skipped locked rollout files`: - treat the sync as mostly successful -- explain that the active session still holds one or more rollout files open +- explain that an active session either still holds one or more rollout files open, or appended to one while it was being scanned - tell the user to rerun `codex-provider sync` after that session ends if they want a full rewrite If `switch ` fails because the provider is missing: diff --git a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs index e9d7f7d..098ec82 100644 --- a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Text.Json.Nodes; using System.Diagnostics; using Microsoft.Data.Sqlite; @@ -1594,6 +1595,67 @@ await File.AppendAllTextAsync( Assert.Contains("\"message\":\"later\"", rollout); } + [Fact] + public async Task CollectSessionChanges_ReportsLockedInsteadOfAborting_WhenRolloutChangesDuringScan() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + await fixture.WriteConfigAsync("model_provider = \"openai\""); + string movingPath = fixture.RolloutPath("sessions", "rollout-a.jsonl"); + string stablePath = fixture.RolloutPath("sessions", "rollout-b.jsonl"); + await fixture.WriteRolloutAsync(movingPath, "thread-a", "apigather"); + await fixture.WriteRolloutAsync(stablePath, "thread-b", "apigather"); + + // Emulate an active Codex session appending to its own rollout inside the + // window between the digest fold and the post-scan snapshot. + SessionRolloutService service = new() + { + ScanFaultInjector = async scannedPath => + { + if (string.Equals(scannedPath, movingPath, StringComparison.Ordinal)) + { + await File.AppendAllTextAsync( + scannedPath, + "{\"timestamp\":\"2026-03-19T00:00:01.000Z\",\"type\":\"event_msg\",\"payload\":{\"type\":\"assistant_message\",\"message\":\"live\"}}\n"); + } + } + }; + + SessionChangeCollection collected = await service.CollectSessionChangesAsync( + fixture.CodexHome, + "openai", + skipLockedReads: true); + + Assert.Contains(movingPath, collected.LockedPaths); + Assert.DoesNotContain(movingPath, collected.Changes.Select(static change => change.Path)); + Assert.Contains(stablePath, collected.Changes.Select(static change => change.Path)); + + // The rest of the corpus is still rewritten; only the live rollout waits. + SessionApplyResult applyResult = await service.ApplySessionChangesAsync(collected.Changes); + Assert.Equal(1, applyResult.AppliedCount); + Assert.Empty(applyResult.SkippedPaths); + Assert.Contains("\"model_provider\":\"apigather\"", await File.ReadAllTextAsync(movingPath)); + Assert.Contains("\"model_provider\":\"openai\"", await File.ReadAllTextAsync(stablePath)); + } + + [Fact] + public async Task CollectSessionChanges_Throws_WhenRolloutChangesDuringScanWithoutSkipLockedReads() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + await fixture.WriteConfigAsync("model_provider = \"openai\""); + string movingPath = fixture.RolloutPath("sessions", "rollout-a.jsonl"); + await fixture.WriteRolloutAsync(movingPath, "thread-a", "apigather"); + + SessionRolloutService service = new() + { + ScanFaultInjector = async scannedPath => await File.AppendAllTextAsync( + scannedPath, + "{\"timestamp\":\"2026-03-19T00:00:01.000Z\",\"type\":\"event_msg\",\"payload\":{\"type\":\"assistant_message\",\"message\":\"live\"}}\n") + }; + + await Assert.ThrowsAsync(() => + service.CollectSessionChangesAsync(fixture.CodexHome, "openai", skipLockedReads: false)); + } + [Fact] public async Task ApplySessionChanges_RewritesFile_WhenRolloutIsUnchanged() { @@ -1813,6 +1875,58 @@ await fixture.WriteBackupAsync( Assert.True(string.IsNullOrWhiteSpace(result.AutoPruneWarning)); } + [Fact] + public async Task RunSync_SucceedsWithWarningAndStillPrunes_WhenInventoryRefreshFailsAfterCommit() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + await fixture.WriteConfigAsync("model_provider = \"openai\""); + string sessionPath = fixture.RolloutPath("sessions", "rollout-a.jsonl"); + await fixture.WriteRolloutAsync(sessionPath, "thread-a", "apigather"); + await fixture.WriteStateDbAsync([("thread-a", "apigather", false)]); + + for (int index = 0; index < AppConstants.DefaultBackupRetentionCount; index += 1) + { + await fixture.WriteBackupAsync( + $"20240101T0000{index:00}000Z", + ("note.txt", $"backup-{index}")); + } + + // Break the inventory refresh that runs immediately after the journal + // commit. The transaction is durable by then, so the sync must report + // success with a warning rather than failing. + CodexSyncService service = new(); + service.FaultInjector = (point, _, _) => + { + if (point == "before_transaction_commit") + { + // The backup directory this transaction owns is the only one + // still carrying a journal file. Keep the namespace so it stays a + // managed backup for the prune pass, but make the version + // unreadable so only the inventory refresh fails. + string activeBackupDir = Directory + .GetDirectories(fixture.BackupRoot()) + .Single(dir => File.Exists(Path.Combine(dir, FileTransactionJournal.FileName))); + string metadataPath = Path.Combine(activeBackupDir, "metadata.json"); + JsonNode metadata = JsonNode.Parse(File.ReadAllText(metadataPath))!; + metadata["version"] = 99; + File.WriteAllText(metadataPath, metadata.ToJsonString()); + } + + return Task.CompletedTask; + }; + + SyncResult result = await service.RunSyncAsync(fixture.CodexHome); + + Assert.NotNull(result.AutoPruneResult); + Assert.Equal(1, result.AutoPruneResult!.DeletedCount); + Assert.Contains("Backup inventory refresh failed", result.AutoPruneWarning); + + // The mutation itself still landed. + Assert.Contains("\"model_provider\":\"openai\"", await File.ReadAllTextAsync(sessionPath)); + Assert.Equal("openai", await ReadProviderAsync(fixture.StateDbPath(), "thread-a")); + Assert.Empty(await FileTransactionJournal.FindPendingAsync(fixture.CodexHome)); + } + [Fact] public async Task RunSync_UsesCustomAutomaticBackupRetentionCount() { diff --git a/desktop/CodexProviderSync.Core/CodexSyncService.cs b/desktop/CodexProviderSync.Core/CodexSyncService.cs index cda7dfc..6531065 100644 --- a/desktop/CodexProviderSync.Core/CodexSyncService.cs +++ b/desktop/CodexProviderSync.Core/CodexSyncService.cs @@ -635,7 +635,20 @@ await FaultInjector( await journal.CommittedAsync(); transactionCommitted = true; await journal.DisposeAsync(); - await _backupService.RefreshMetadataInventoryAsync(backupDir); + // The transaction is already committed and every target is on disk. + // Refreshing the backup inventory only corrects the recorded size + // and file count in metadata.json, so a failure here must degrade + // to a warning: throwing would report a successful sync as failed + // and skip the automatic backup pruning below. + string? backupInventoryWarning = null; + try + { + await _backupService.RefreshMetadataInventoryAsync(backupDir); + } + catch (Exception error) + { + backupInventoryWarning = $"Backup inventory refresh failed: {error.Message}"; + } long mutationDurationMs = ElapsedMilliseconds(mutationStarted); if (FaultInjector is not null) { @@ -658,6 +671,7 @@ await FaultInjector( autoPruneWarning = $"Automatic backup cleanup failed: {error.Message}"; } long pruneDurationMs = ElapsedMilliseconds(pruneStarted); + autoPruneWarning = JoinBackupWarnings(backupInventoryWarning, autoPruneWarning); SyncResult result = new() { @@ -1204,8 +1218,27 @@ await FileTransactionJournal.MarkBackupRolledBackAsync( preparation.BackupDirectory, codexHome, result.TargetProvider); - await _backupService.RefreshMetadataInventoryAsync(preparation.BackupDirectory); - return result; + // The restore and its journal marker are already durable. Refreshing the + // inventory only corrects metadata.json bookkeeping, so surface a + // failure as a warning instead of reporting a completed restore as + // failed. + try + { + await _backupService.RefreshMetadataInventoryAsync(preparation.BackupDirectory); + return result; + } + catch (Exception error) + { + return new RestoreResult + { + CodexHome = result.CodexHome, + BackupDir = result.BackupDir, + TargetProvider = result.TargetProvider, + CreatedAt = result.CreatedAt, + ChangedSessionFiles = result.ChangedSessionFiles, + BackupInventoryWarning = $"Backup inventory refresh failed: {error.Message}" + }; + } } private async Task PrepareRestoreAsync( @@ -1498,6 +1531,15 @@ public Task GetBackupStorageInfoAsync(string backupDir) private static long ElapsedMilliseconds(long started) => (long)Math.Round(Stopwatch.GetElapsedTime(started).TotalMilliseconds); + private static string? JoinBackupWarnings(string? first, string? second) + { + string[] parts = new[] { first, second } + .Where(static part => !string.IsNullOrWhiteSpace(part)) + .Select(static part => part!.Trim()) + .ToArray(); + return parts.Length == 0 ? null : string.Join(" | ", parts); + } + private async Task PrepareStorageAsync( string codexHome, string? explicitSqliteHome, diff --git a/desktop/CodexProviderSync.Core/Models.cs b/desktop/CodexProviderSync.Core/Models.cs index 66888c0..e7158b7 100644 --- a/desktop/CodexProviderSync.Core/Models.cs +++ b/desktop/CodexProviderSync.Core/Models.cs @@ -245,6 +245,14 @@ public sealed class RestoreResult public required string TargetProvider { get; init; } public DateTimeOffset? CreatedAt { get; init; } public int ChangedSessionFiles { get; init; } + + /// + /// Set when the restore itself succeeded but refreshing the backup + /// directory inventory afterwards did not. The restored state is already + /// authoritative; only the recorded size and file count in metadata.json + /// may be stale. + /// + public string? BackupInventoryWarning { get; init; } } public sealed class BackupStorageInfo diff --git a/desktop/CodexProviderSync.Core/SessionRolloutService.cs b/desktop/CodexProviderSync.Core/SessionRolloutService.cs index 8d857d0..2396166 100644 --- a/desktop/CodexProviderSync.Core/SessionRolloutService.cs +++ b/desktop/CodexProviderSync.Core/SessionRolloutService.cs @@ -13,6 +13,13 @@ public sealed class SessionRolloutService private const string StatusOnlyProvider = "__status_only__"; internal Func? ApplyFaultInjector { get; set; } + /// + /// Test seam invoked after a rollout's content digest is folded but before + /// its post-scan snapshot is taken, so a test can deterministically mutate + /// the file inside that window. + /// + internal Func? ScanFaultInjector { get; set; } + public async Task CollectSessionChangesAsync( string codexHome, string targetProvider, @@ -94,6 +101,10 @@ record = await ReadFirstLineRecordAsync(rolloutPath); collectFingerprint: providerMayChange); contentScanPasses += 1; modelScanFiles += collectModels ? 1 : 0; + if (ScanFaultInjector is not null) + { + await ScanFaultInjector(rolloutPath); + } } catch (Exception error) when (skipLockedReads && IsRolloutFileBusyError(error)) { @@ -139,6 +150,19 @@ record = await ReadFirstLineRecordAsync(rolloutPath); FileSnapshot snapshot = GetFileSnapshot(rolloutPath); if (contentScan.ContentFingerprint is not null && snapshot != scanStart) { + // The rollout grew or was touched while we were folding + // its content digest, so the fingerprint describes a + // state that no longer exists and must not be cached as + // a plan hint. An active Codex session appending to its + // own rollout is the ordinary cause, so report it the + // same way a busy file is reported and keep rewriting + // the rest instead of aborting the whole sync. + if (skipLockedReads) + { + lockedPaths.Add(rolloutPath); + continue; + } + throw new CoreWritePlanStaleException(); } if (providerChanged) diff --git a/desktop/CodexProviderSync.Core/TextFormatter.cs b/desktop/CodexProviderSync.Core/TextFormatter.cs index 83f5d41..b58da7c 100644 --- a/desktop/CodexProviderSync.Core/TextFormatter.cs +++ b/desktop/CodexProviderSync.Core/TextFormatter.cs @@ -83,6 +83,13 @@ public static string FormatRestoreResult(RestoreResult result, string language) : $"Backup created at: {result.CreatedAt:O}"); } + if (!string.IsNullOrWhiteSpace(result.BackupInventoryWarning)) + { + lines.Add(IsChinese(language) + ? $"备份清单警告: {result.BackupInventoryWarning}" + : $"Backup inventory warning: {result.BackupInventoryWarning}"); + } + return string.Join(Environment.NewLine, lines); } diff --git a/src/service.js b/src/service.js index f9e143b..4586069 100644 --- a/src/service.js +++ b/src/service.js @@ -644,7 +644,16 @@ async function runSyncCore({ await faultInjector?.({ point: "before_transaction_commit", completedCount: completedTargets.length }); await commitJournalWithReconciliation(journal, faultInjector); transactionCommitted = true; - await refreshBackupInventory(backupDir); + // The transaction is committed and every target is on disk. Refreshing the + // inventory only corrects the recorded size and file count in + // metadata.json, so a failure must degrade to a warning: throwing would + // report a successful sync as failed and skip the pruning below. + let backupInventoryWarning = null; + try { + await refreshBackupInventory(backupDir); + } catch (inventoryError) { + backupInventoryWarning = `Backup inventory refresh failed: ${inventoryError instanceof Error ? inventoryError.message : String(inventoryError)}`; + } await faultInjector?.({ point: "after_transaction_commit", completedCount: completedTargets.length }); let autoPruneResult = null; let autoPruneWarning = null; @@ -664,6 +673,10 @@ async function runSyncCore({ deletedCount: autoPruneResult?.deletedCount ?? 0, warning: autoPruneWarning }); + autoPruneWarning = [backupInventoryWarning, autoPruneWarning] + .filter((part) => typeof part === "string" && part.trim().length > 0) + .map((part) => part.trim()) + .join(" | ") || null; const result = { codexHome, sqliteHome: storage.sqliteHome, @@ -1013,7 +1026,17 @@ export async function runRestore({ allowSqliteHomeRelocation }); await markBackupTransactionRolledBack(normalizedBackupDir); - await refreshBackupInventory(normalizedBackupDir); + // The restore and its journal marker are already durable. Refreshing the + // inventory only corrects metadata.json bookkeeping, so surface a failure as + // a warning instead of reporting a completed restore as failed. + try { + await refreshBackupInventory(normalizedBackupDir); + } catch (inventoryError) { + return { + ...result, + backupInventoryWarning: `Backup inventory refresh failed: ${inventoryError instanceof Error ? inventoryError.message : String(inventoryError)}` + }; + } return result; } finally { await releaseLock(); diff --git a/test/sync-service.test.js b/test/sync-service.test.js index 2811df0..f1ab0af 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -3741,6 +3741,55 @@ test("runSync auto-prunes backups to the default retention count", async () => { assert.equal(result.autoPruneWarning, null); }); +test("runSync succeeds with a warning and still prunes when the inventory refresh fails after commit", async () => { + const { codexHome } = await makeTempCodexHome(); + await writeConfig(codexHome, 'model_provider = "openai"'); + const sessionPath = path.join(codexHome, "sessions", "2026", "03", "19", "rollout-a.jsonl"); + await writeRollout(sessionPath, "thread-a", "apigather"); + await writeStateDb(codexHome, [ + { id: "thread-a", model_provider: "apigather", archived: false } + ]); + + for (let index = 0; index < DEFAULT_BACKUP_RETENTION_COUNT; index += 1) { + await writeBackup(codexHome, `20240101T0000${String(index).padStart(2, "0")}000Z`, [ + ["note.txt", `backup-${index}`] + ]); + } + + const result = await runSync({ + codexHome, + // Break the inventory refresh that runs right after the journal commit. The + // transaction is durable by then, so the sync must report success with a + // warning instead of failing and skipping the prune below. + async faultInjector({ point }) { + if (point !== "before_transaction_commit") { + return; + } + const dirs = await fs.readdir(backupRoot(codexHome)); + for (const dir of dirs) { + const candidate = path.join(backupRoot(codexHome), dir); + try { + await fs.access(path.join(candidate, "transaction-journal.jsonl")); + } catch { + continue; + } + // Keep the namespace so the directory is still a managed backup for the + // prune pass, but make the version unreadable so only the inventory + // refresh fails. + const metadataPath = path.join(candidate, "metadata.json"); + const metadata = JSON.parse(await fs.readFile(metadataPath, "utf8")); + await fs.writeFile(metadataPath, JSON.stringify({ ...metadata, version: 99 }, null, 2)); + } + } + }); + + assert.equal(result.autoPruneResult.deletedCount, 1); + assert.match(result.autoPruneWarning, /Backup inventory refresh failed/); + assert.match(await fs.readFile(sessionPath, "utf8"), /"model_provider":"openai"/); + assert.equal(await readProvider(codexHome, "thread-a"), "openai"); + assert.deepEqual(await findPendingTransactions(codexHome), []); +}); + test("runSync uses a custom automatic backup retention count", async () => { const { codexHome } = await makeTempCodexHome(); await writeConfig(codexHome, 'model_provider = "openai"'); From 62c4bf287f526ca225f76410a9a4f5fa013f6687 Mon Sep 17 00:00:00 2001 From: Dailin521 <71995555+Dailin521@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:54:33 +0800 Subject: [PATCH 4/5] fix: refresh rollback inventory and stop reporting false open failures Second review round on the backup inventory work, plus the folder-open false error. Rollback paths left metadata.json stale. RolledBackAsync and RecoveryRequiredAsync append journal records into the retained backup after its inventory was written, and GetBackupDirectorySize trusts the recorded sizeBytes unconditionally, so status and pruning read a size and file count that no longer match disk. Both rollback exits now refresh the inventory, swallowing any failure so the original sync error and its rollback details reach the caller unchanged. The Node CLI dropped restore's backupInventoryWarning, silently hiding a warning the service already produced. OpenPath treated a null Process as a failure. With UseShellExecute the shell may satisfy the request through a process it already owns - Explorer reusing an open window is the common case - and then returns no handle even though the path opened, so the folder opened while the GUI reported "Unable to open". A genuine failure still surfaces as Win32Exception, and OpenBackupFolder now reports it the same way OpenLogFolder does instead of escaping to the message loop. --- .../MainFormPresentationTests.cs | 64 +++++++++++++++++++ desktop/CodexProviderSync.App/AppPaths.cs | 7 +- desktop/CodexProviderSync.App/MainForm.cs | 12 +++- .../CoreIntegrationTests.cs | 51 +++++++++++++++ .../CodexSyncService.cs | 22 +++++++ src/cli.js | 3 + src/service.js | 16 +++++ test/sync-service.test.js | 32 ++++++++++ 8 files changed, 204 insertions(+), 3 deletions(-) diff --git a/desktop/CodexProviderSync.App.Tests/MainFormPresentationTests.cs b/desktop/CodexProviderSync.App.Tests/MainFormPresentationTests.cs index c86c95c..2139a85 100644 --- a/desktop/CodexProviderSync.App.Tests/MainFormPresentationTests.cs +++ b/desktop/CodexProviderSync.App.Tests/MainFormPresentationTests.cs @@ -226,6 +226,70 @@ private static void PerformLayoutRecursively(Control control) } } + [Fact] + public void OpenBackupFolder_ReportsNoError_WhenShellReusesAnExistingProcess() + { + string root = Path.Combine(Path.GetTempPath(), $"codex-provider-ui-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + // Explorer commonly satisfies a folder request through a window it + // already owns and then returns no Process handle. The boundary must + // treat that as a successful open, not a failure. + RecordingPlatformBoundary boundary = new(); + using MainForm form = new(new ExecutionLogService(root), platformBoundary: boundary); + SetField(form, "_currentStatus", StatusWithBackupRoot(root)); + + Invoke(form, "OpenBackupFolder"); + + Assert.Equal(root, Assert.Single(boundary.OpenedPaths)); + Assert.DoesNotContain("打开备份目录失败", Field(form, "_logBox").Text); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + // The failure branch of OpenBackupFolder is deliberately not covered here: + // its catch reports through MessageBox.Show, which has no injection seam in + // MainForm, so a test driving it would block the runner on a modal dialog. + + private static StatusSnapshot StatusWithBackupRoot(string backupRoot) => new() + { + CodexHome = @"C:\Users\user\.codex", + SqliteAccess = new SqliteAccessInfo(true, "windows", null), + CurrentProvider = new CurrentProviderInfo("openai", false), + ConfiguredProviders = ["openai"], + RolloutCounts = new ProviderCounts(), + LockedRolloutFiles = [], + UnreadableRolloutFiles = [], + EncryptedContentCounts = new ProviderCounts(), + SqliteCounts = null, + BackupRoot = backupRoot, + BackupSummary = new BackupSummary { Count = 0, TotalBytes = 0 } + }; + + private sealed class RecordingPlatformBoundary : IAppPlatformBoundary + { + public List OpenedPaths { get; } = []; + + public bool UpdatesEnabled => false; + + public string? PickFolder(IWin32Window owner, FolderPickerRequest request) => null; + + public Task CheckForUpdateAsync( + UpdateService updateService, + Version currentVersion, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public void OpenPath(string path) => OpenedPaths.Add(path); + + public void StartUpdate(string downloadedExePath, string targetExePath, string expectedSha256) => + throw new NotSupportedException(); + } + private static T Field(MainForm form, string name) where T : class { return typeof(MainForm) diff --git a/desktop/CodexProviderSync.App/AppPaths.cs b/desktop/CodexProviderSync.App/AppPaths.cs index 8110bb7..120ac73 100644 --- a/desktop/CodexProviderSync.App/AppPaths.cs +++ b/desktop/CodexProviderSync.App/AppPaths.cs @@ -227,11 +227,16 @@ public Task CheckForUpdateAsync( public void OpenPath(string path) { + // With UseShellExecute the shell may satisfy the request through a + // process it already owns - Explorer reusing an open window is the + // common case - and then returns no Process handle even though the + // path opened. A null result therefore carries no failure information; + // a genuine failure such as a missing path surfaces as Win32Exception. _ = Process.Start(new ProcessStartInfo { FileName = path, UseShellExecute = true - }) ?? throw new InvalidOperationException($"Unable to open {path}."); + }); } public void StartUpdate(string downloadedExePath, string targetExePath, string expectedSha256) => diff --git a/desktop/CodexProviderSync.App/MainForm.cs b/desktop/CodexProviderSync.App/MainForm.cs index 40d0f42..5ce74ee 100644 --- a/desktop/CodexProviderSync.App/MainForm.cs +++ b/desktop/CodexProviderSync.App/MainForm.cs @@ -1079,8 +1079,16 @@ private void OpenBackupFolder() { string path = _currentStatus?.BackupRoot ?? AppConstants.DefaultBackupRoot(CurrentCodexHome()); EnsureAutomationPath(path, GuiAutomationCatalog.Ids.OpenBackupDirectory); - Directory.CreateDirectory(path); - _platformBoundary.OpenPath(path); + try + { + Directory.CreateDirectory(path); + _platformBoundary.OpenPath(path); + } + catch (Exception error) + { + AppendLog($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss}] 打开备份目录失败: {error}"); + MessageBox.Show(this, $"无法打开备份目录。{Environment.NewLine}{Environment.NewLine}{error.Message}", Text, MessageBoxButtons.OK, MessageBoxIcon.Error); + } } private void OpenLogFolder() diff --git a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs index 098ec82..8e21897 100644 --- a/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs +++ b/desktop/CodexProviderSync.Core.Tests/CoreIntegrationTests.cs @@ -46,6 +46,57 @@ await fixture.WriteStateDbAsync([ Assert.Equal("apigather", await ReadProviderAsync(fixture.StateDbPath(), "thread-a")); Assert.Equal("apigather", await ReadProviderAsync(fixture.StateDbPath(), "thread-b")); Assert.Empty(await FileTransactionJournal.FindPendingAsync(fixture.CodexHome)); + + // The rollback appended more journal records after the inventory was + // written, so the retained backup must record its real size. Status and + // pruning read these cached values without revalidating them. + AssertBackupInventoryMatchesDisk(error.BackupDirectory); + } + + [Fact] + public async Task RunSync_RefreshesRetainedBackupInventory_WhenRollbackCompletes() + { + TestCodexHomeFixture fixture = await TestCodexHomeFixture.CreateAsync(); + await fixture.WriteConfigAsync("model_provider = \"openai\""); + string sessionPath = fixture.RolloutPath("sessions", "rollout-a.jsonl"); + await fixture.WriteRolloutAsync(sessionPath, "thread-a", "apigather"); + await fixture.WriteStateDbAsync([("thread-a", "apigather", false)]); + + // Fail the apply so the rollback path runs and the journal appends its + // terminal records after metadata.json was already written. + CodexSyncService service = new(); + service.FaultInjector = (point, _, _) => + { + if (point == "before_rollout_apply") + { + throw new IOException("injected apply failure"); + } + return Task.CompletedTask; + }; + + SyncTransactionException error = await Assert.ThrowsAsync( + () => service.RunSyncAsync(fixture.CodexHome, provider: "openai")); + + // Pin the path this covers: the rollback succeeded, so the journal wrote + // its rolledBack record into the retained backup after the inventory was + // captured. The recorded size and file count must still match disk. + Assert.Equal("complete", error.RollbackStatus); + Assert.False(error.RecoveryRequired); + AssertBackupInventoryMatchesDisk(error.BackupDirectory); + } + + private static void AssertBackupInventoryMatchesDisk(string backupDirectory) + { + string metadataPath = Path.Combine(backupDirectory, "metadata.json"); + using JsonDocument document = JsonDocument.Parse(File.ReadAllText(metadataPath)); + long recordedSize = document.RootElement.GetProperty("sizeBytes").GetInt64(); + int recordedCount = document.RootElement.GetProperty("fileCount").GetInt32(); + + string[] actualFiles = Directory.GetFiles(backupDirectory, "*", SearchOption.AllDirectories); + long actualSize = actualFiles.Sum(file => new FileInfo(file).Length); + + Assert.Equal(actualFiles.Length, recordedCount); + Assert.Equal(actualSize, recordedSize); } [Fact] diff --git a/desktop/CodexProviderSync.Core/CodexSyncService.cs b/desktop/CodexProviderSync.Core/CodexSyncService.cs index 6531065..6a7bc2b 100644 --- a/desktop/CodexProviderSync.Core/CodexSyncService.cs +++ b/desktop/CodexProviderSync.Core/CodexSyncService.cs @@ -776,6 +776,7 @@ await FaultInjector( // Preserve the original and rollback failures when the // journal itself is no longer writable. } + await TryRefreshBackupInventoryAsync(backupDir); IReadOnlyList reportedCompletedTargets = BuildReportedCompletedTargets( completedTargets, observedMutatedTargets); @@ -793,6 +794,7 @@ await FaultInjector( recoveryRequired: true); } + await TryRefreshBackupInventoryAsync(backupDir); IReadOnlyList completedAfterRollback = BuildReportedCompletedTargets( completedTargets, observedMutatedTargets); @@ -1531,6 +1533,26 @@ public Task GetBackupStorageInfoAsync(string backupDir) private static long ElapsedMilliseconds(long started) => (long)Math.Round(Stopwatch.GetElapsedTime(started).TotalMilliseconds); + /// + /// Rewrites the retained backup's recorded size and file count after the + /// journal reached a terminal state, so status and pruning do not trust an + /// inventory captured before those journal records existed. Used on the + /// rollback paths, where the caller is already reporting a failure: a + /// bookkeeping problem here must never replace the original error. + /// + private async Task TryRefreshBackupInventoryAsync(string backupDir) + { + try + { + await _backupService.RefreshMetadataInventoryAsync(backupDir); + } + catch + { + // The original sync failure and its rollback details are the + // authoritative diagnosis and must reach the caller unchanged. + } + } + private static string? JoinBackupWarnings(string? first, string? second) { string[] parts = new[] { first, second } diff --git a/src/cli.js b/src/cli.js index 836253a..8052edb 100644 --- a/src/cli.js +++ b/src/cli.js @@ -328,6 +328,9 @@ async function main() { console.log(`Restored backup from ${path.resolve(backupDir)}`); console.log(`Codex home: ${result.codexHome}`); console.log(`Provider at backup time: ${result.targetProvider}`); + if (result.backupInventoryWarning) { + console.log(`Backup inventory warning: ${result.backupInventoryWarning}`); + } return; } diff --git a/src/service.js b/src/service.js index 4586069..90453f2 100644 --- a/src/service.js +++ b/src/service.js @@ -201,6 +201,20 @@ async function commitJournalWithReconciliation(journal, faultInjector) { throw new Error(`Transaction journal did not persist a valid committed terminal state: ${journal.filePath}`); } +// Rewrites the retained backup's recorded size and file count after the journal +// reached a terminal state, so status and pruning do not trust an inventory +// captured before those journal records existed. Used on the rollback paths, +// where the caller is already reporting a failure: a bookkeeping problem here +// must never replace the original error. +async function tryRefreshBackupInventory(backupDir) { + try { + await refreshBackupInventory(backupDir); + } catch { + // The original sync failure and its rollback details are the authoritative + // diagnosis and must reach the caller unchanged. + } +} + async function rollbackJournalWithReconciliation(journal, faultInjector) { let acknowledgementError = null; try { @@ -810,6 +824,7 @@ async function runSyncCore({ // Preserve the original and rollback errors even if the journal is // no longer writable. } + await tryRefreshBackupInventory(backupDir); const persistedCompletedTargets = journalSnapshot ? uniqueResolvedPaths([...getAppliedJournalTargets(journalSnapshot), ...completedTargets]) : uniqueResolvedPaths(completedTargets); @@ -828,6 +843,7 @@ async function runSyncCore({ { rollbackStatus: "incomplete", recoveryRequired: true } ); } + await tryRefreshBackupInventory(backupDir); const persistedCompletedTargets = journalSnapshot ? uniqueResolvedPaths([...getAppliedJournalTargets(journalSnapshot), ...completedTargets]) : uniqueResolvedPaths(completedTargets); diff --git a/test/sync-service.test.js b/test/sync-service.test.js index f1ab0af..cac6fa6 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -492,6 +492,38 @@ test("unfinished journal blocks writes until the bound backup is restored", asyn assert.equal(result.targetProvider, "openai"); }); +test("runRestore reports a warning instead of failing when the inventory refresh fails", async () => { + const { codexHome } = await makeTempCodexHome(); + await writeConfig(codexHome, 'model_provider = "openai"'); + const sessionPath = path.join(codexHome, "sessions", "2026", "03", "19", "rollout-a.jsonl"); + await writeRollout(sessionPath, "thread-a", "apigather"); + await writeStateDb(codexHome, [ + { id: "thread-a", model_provider: "apigather", archived: false } + ]); + + const synced = await runSync({ codexHome }); + const backupDir = synced.backupDir; + + // Leave the metadata perfectly readable so the restore itself proceeds, and + // fail only the write the post-restore inventory refresh performs: the atomic + // writer reopens its staged replacement with the original file's mode, which + // a read-only metadata.json makes impossible. + const metadataPath = path.join(backupDir, "metadata.json"); + await fs.chmod(metadataPath, 0o444); + + let result; + try { + result = await runRestore({ backupDir, codexHome }); + } finally { + await fs.chmod(metadataPath, 0o644); + } + + // The restore itself is authoritative and must still be reported as done. + assert.equal(result.targetProvider, "openai"); + assert.match(result.backupInventoryWarning, /Backup inventory refresh failed/); + assert.match(await fs.readFile(sessionPath, "utf8"), /"model_provider":"apigather"/); +}); + test("crash recovery restores actually mutated rollout and database from a pending journal", async () => { const { codexHome } = await makeTempCodexHome(); await writeConfig(codexHome, 'model_provider = "openai"'); From 496494be3e8dd18ba001c1ebef7218ff20567410 Mon Sep 17 00:00:00 2001 From: Dailin521 <71995555+Dailin521@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:02:12 +0800 Subject: [PATCH 5/5] test: inject the restore inventory failure instead of using file mode The chmod(0o444) fault only works when the suite runs as a user that Unix permission bits actually constrain. As root the atomic metadata rewrite succeeds, backupInventoryWarning stays undefined and the test fails, which is what happens in the repository container. runRestore now accepts a faultInjector and threads it into the post-restore refresh, matching how runSync already exposes one, so the test drives the failure through a deterministic seam. --- src/service.js | 5 +++-- test/sync-service.test.js | 25 ++++++++++++------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/service.js b/src/service.js index 90453f2..82bc191 100644 --- a/src/service.js +++ b/src/service.js @@ -968,7 +968,8 @@ export async function runRestore({ restoreDatabase = true, restoreSessions = true, allowSqliteHomeRelocation = false, - platform + platform, + faultInjector }) { if (!backupDir) { throw new Error("Missing backup path. Usage: codex-provider restore "); @@ -1046,7 +1047,7 @@ export async function runRestore({ // inventory only corrects metadata.json bookkeeping, so surface a failure as // a warning instead of reporting a completed restore as failed. try { - await refreshBackupInventory(normalizedBackupDir); + await refreshBackupInventory(normalizedBackupDir, { faultInjector }); } catch (inventoryError) { return { ...result, diff --git a/test/sync-service.test.js b/test/sync-service.test.js index cac6fa6..d69163a 100644 --- a/test/sync-service.test.js +++ b/test/sync-service.test.js @@ -504,19 +504,18 @@ test("runRestore reports a warning instead of failing when the inventory refresh const synced = await runSync({ codexHome }); const backupDir = synced.backupDir; - // Leave the metadata perfectly readable so the restore itself proceeds, and - // fail only the write the post-restore inventory refresh performs: the atomic - // writer reopens its staged replacement with the original file's mode, which - // a read-only metadata.json makes impossible. - const metadataPath = path.join(backupDir, "metadata.json"); - await fs.chmod(metadataPath, 0o444); - - let result; - try { - result = await runRestore({ backupDir, codexHome }); - } finally { - await fs.chmod(metadataPath, 0o644); - } + // Fail only the write the post-restore inventory refresh performs. The + // injected seam is deterministic regardless of the user the suite runs as, + // which file permission bits are not: root bypasses them entirely. + const result = await runRestore({ + backupDir, + codexHome, + faultInjector: ({ point }) => { + if (point === "before_atomic_replace") { + throw new Error("injected inventory write failure"); + } + } + }); // The restore itself is authoritative and must still be reported as done. assert.equal(result.targetProvider, "openai");