diff --git a/pages/mdsource/temporal-helper.source.md b/pages/mdsource/temporal-helper.source.md index ec36d5a8..9c58ab33 100644 --- a/pages/mdsource/temporal-helper.source.md +++ b/pages/mdsource/temporal-helper.source.md @@ -2,7 +2,9 @@ `SetCurrentPeriodStart` re-stamps a row's `PeriodStart` (and aligns the most recent history row's `PeriodEnd`) on a [SQL Server temporal table](https://learn.microsoft.com/sql/relational-databases/tables/temporal-tables). It exists to give back-to-back `SaveChanges` calls in tests distinct, deterministic temporal timestamps without relying on `Task.Delay` between them. -The method is exposed on both `SqlInstance` and `SqlDatabase`. Schema lookups (table, history table, period columns, PK column) are performed once at `SqlInstance` construction and cached, so per-call overhead is only the SQL execution. +`SetHistoryColumn` is a second helper on the same schema lookup. It writes one column on the history rows of a single entity, to put a history table into a state a freshly migrated database never reaches. See [Simulating a damaged history row](#simulating-a-damaged-history-row). + +Both are exposed on `SqlInstance` and `SqlDatabase`. Schema lookups (table, history table, period columns, PK column) are performed once at `SqlInstance` construction and cached, so per-call overhead is only the SQL execution. ## Why @@ -45,6 +47,23 @@ For each call the helper runs, in separate batches: Steps 5 and 6 run in a `finally` so a failed UPDATE doesn't leave the table without versioning. +## Simulating a damaged history row + +A history table can hold values the current model says are impossible. The usual cause is a migration that drops and re-adds a column on a temporal pair — done to keep column ordinals matching between the two tables, which SQL Server requires. The current table repopulates (or recomputes, if the column is computed); the rows already in the history table are left NULL, and SQL Server does not backfill them. + +Nothing in a test suite reproduces that on its own, because every test database is built by migrating from empty. So the read path that trips over those NULLs — typically materialising an entity whose property is non-nullable, which throws `SqlNullValueException` — is exercised for the first time in production. + +`SetHistoryColumn` reproduces it: + +snippet: SetHistoryColumnUsage + +The period columns and the primary key are rejected: rewriting a period on a history row corrupts the timeline `SetCurrentPeriodStart` maintains, and does so silently, while rewriting the key detaches the row from the entity it is history for. Any other mapped column can be set, computed columns included — those are plain columns on the history table. + +Versioning is turned off for the write and back on in a `finally`. Unlike `SetCurrentPeriodStart` the `PERIOD` is not dropped, since the period columns are ordinary columns on the history table. + +For a null value the column has to permit NULL **in the database**. It is not widened here, and cannot be: SQL Server refuses to re-enable versioning when the current and history tables disagree on nullability, so a row like that could not exist in production either. The case worth reproducing is the column that is nullable in the database while the model declares the property required — a stored computed column whose `CASE` has no `ELSE` is the common way to end up there, and is what the snippet above uses. + + ## Performance Each call performs three round trips to SQL Server (opening DDL pair, the two UPDATEs combined, closing DDL pair). The dominant costs are: diff --git a/pages/temporal-helper.md b/pages/temporal-helper.md index f3751f44..d6495ce8 100644 --- a/pages/temporal-helper.md +++ b/pages/temporal-helper.md @@ -9,7 +9,9 @@ To change this file edit the source file and then run MarkdownSnippets. `SetCurrentPeriodStart` re-stamps a row's `PeriodStart` (and aligns the most recent history row's `PeriodEnd`) on a [SQL Server temporal table](https://learn.microsoft.com/sql/relational-databases/tables/temporal-tables). It exists to give back-to-back `SaveChanges` calls in tests distinct, deterministic temporal timestamps without relying on `Task.Delay` between them. -The method is exposed on both `SqlInstance` and `SqlDatabase`. Schema lookups (table, history table, period columns, PK column) are performed once at `SqlInstance` construction and cached, so per-call overhead is only the SQL execution. +`SetHistoryColumn` is a second helper on the same schema lookup. It writes one column on the history rows of a single entity, to put a history table into a state a freshly migrated database never reaches. See [Simulating a damaged history row](#simulating-a-damaged-history-row). + +Both are exposed on `SqlInstance` and `SqlDatabase`. Schema lookups (table, history table, period columns, PK column) are performed once at `SqlInstance` construction and cached, so per-call overhead is only the SQL execution. ## Why @@ -31,6 +33,12 @@ public class TravelRequest public Guid Id { get; set; } public string Status { get; set; } = "Draft"; + // Declared required here. The CASE below always matches a branch, so + // this is never actually null - but it has no ELSE, which leaves the + // column itself nullable. That gap between what the model promises and + // what the schema permits is what SetHistoryColumn exploits. + public int StatusRank { get; set; } + [Timestamp] public byte[] RowVersion { get; set; } = null!; } @@ -40,12 +48,21 @@ public class MyDbContext(DbContextOptions options) : { public DbSet TravelRequests { get; set; } = null!; - protected override void OnModelCreating(ModelBuilder model) => - model.Entity() - .ToTable("TravelRequests", _ => _.IsTemporal()); + protected override void OnModelCreating(ModelBuilder model) + { + var entity = model.Entity(); + entity.ToTable("TravelRequests", _ => _.IsTemporal()); + entity.Property(_ => _.StatusRank) + .HasComputedColumnSql( + """ + CASE WHEN [Status] = 'Approved' THEN 1 + WHEN [Status] <> 'Approved' THEN 0 END + """, + stored: true); + } } ``` -snippet source | anchor +snippet source | anchor @@ -76,7 +93,7 @@ await database.SetCurrentPeriodStart(request, anchor.AddMilliseconds(200)); // Subsequent TemporalAsOf queries can now resolve each transition by its // distinct, deterministic PeriodStart instead of relying on Task.Delay. ``` -snippet source | anchor +snippet source | anchor Two overloads are available on `SqlDatabase`: @@ -101,6 +118,62 @@ For each call the helper runs, in separate batches: Steps 5 and 6 run in a `finally` so a failed UPDATE doesn't leave the table without versioning. +## Simulating a damaged history row + +A history table can hold values the current model says are impossible. The usual cause is a migration that drops and re-adds a column on a temporal pair — done to keep column ordinals matching between the two tables, which SQL Server requires. The current table repopulates (or recomputes, if the column is computed); the rows already in the history table are left NULL, and SQL Server does not backfill them. + +Nothing in a test suite reproduces that on its own, because every test database is built by migrating from empty. So the read path that trips over those NULLs — typically materialising an entity whose property is non-nullable, which throws `SqlNullValueException` — is exercised for the first time in production. + +`SetHistoryColumn` reproduces it: + + + +```cs +await using var database = await instance.Build(); + +var request = new TravelRequest { Id = Guid.NewGuid(), Status = "Draft" }; +database.Context.Add(request); +await database.Context.SaveChangesAsync(); + +// Separate the two saves, or they land in the same tick, SQL Server +// discards the zero-length history row, and there is no history left +// to blank. The two helpers are meant to be used together. +var anchor = DateTime.UtcNow.AddSeconds(-10); +await database.SetCurrentPeriodStart(request, anchor); + +request.Status = "Approved"; +await database.Context.SaveChangesAsync(); +await database.SetCurrentPeriodStart(request, anchor.AddMilliseconds(100)); + +// Blank the column on the history rows only. A column dropped and +// re-added on a temporal pair leaves exactly this: the current row is +// repopulated, the rows already in history are not, and SQL Server +// never backfills them. +await database.SetHistoryColumn( + request.Id, + nameof(TravelRequest.StatusRank), + null); + +// Materialising that history row now fails on a SqlNullValueException, +// because the model reads StatusRank into a non-nullable int. That is +// the production failure, reproduced in a test. +var exception = CatchAsync( + () => database.Context.Set() + .TemporalAll() + .Where(_ => _.Id == request.Id) + .ToListAsync()); +NotNull(exception); +``` +snippet source | anchor + + +The period columns and the primary key are rejected: rewriting a period on a history row corrupts the timeline `SetCurrentPeriodStart` maintains, and does so silently, while rewriting the key detaches the row from the entity it is history for. Any other mapped column can be set, computed columns included — those are plain columns on the history table. + +Versioning is turned off for the write and back on in a `finally`. Unlike `SetCurrentPeriodStart` the `PERIOD` is not dropped, since the period columns are ordinary columns on the history table. + +For a null value the column has to permit NULL **in the database**. It is not widened here, and cannot be: SQL Server refuses to re-enable versioning when the current and history tables disagree on nullability, so a row like that could not exist in production either. The case worth reproducing is the column that is nullable in the database while the model declares the property required — a stored computed column whose `CASE` has no `ELSE` is the common way to end up there, and is what the snippet above uses. + + ## Performance Each call performs three round trips to SQL Server (opening DDL pair, the two UPDATEs combined, closing DDL pair). The dominant costs are: diff --git a/src/EfLocalDb.Tests/Snippets/TemporalSnippetTests.cs b/src/EfLocalDb.Tests/Snippets/TemporalSnippetTests.cs index 49c7f218..5921e792 100644 --- a/src/EfLocalDb.Tests/Snippets/TemporalSnippetTests.cs +++ b/src/EfLocalDb.Tests/Snippets/TemporalSnippetTests.cs @@ -9,6 +9,12 @@ public class TravelRequest public Guid Id { get; set; } public string Status { get; set; } = "Draft"; + // Declared required here. The CASE below always matches a branch, so + // this is never actually null - but it has no ELSE, which leaves the + // column itself nullable. That gap between what the model promises and + // what the schema permits is what SetHistoryColumn exploits. + public int StatusRank { get; set; } + [Timestamp] public byte[] RowVersion { get; set; } = null!; } @@ -18,9 +24,18 @@ public class MyDbContext(DbContextOptions options) : { public DbSet TravelRequests { get; set; } = null!; - protected override void OnModelCreating(ModelBuilder model) => - model.Entity() - .ToTable("TravelRequests", _ => _.IsTemporal()); + protected override void OnModelCreating(ModelBuilder model) + { + var entity = model.Entity(); + entity.ToTable("TravelRequests", _ => _.IsTemporal()); + entity.Property(_ => _.StatusRank) + .HasComputedColumnSql( + """ + CASE WHEN [Status] = 'Approved' THEN 1 + WHEN [Status] <> 'Approved' THEN 0 END + """, + stored: true); + } } #endregion @@ -63,4 +78,54 @@ public async Task SetCurrentPeriodStartUsage() #endregion } -} + + [Test] + public async Task SetHistoryColumnUsage() + { + #region SetHistoryColumnUsage + + await using var database = await instance.Build(); + + var request = new TravelRequest { Id = Guid.NewGuid(), Status = "Draft" }; + database.Context.Add(request); + await database.Context.SaveChangesAsync(); + + // Separate the two saves, or they land in the same tick, SQL Server + // discards the zero-length history row, and there is no history left + // to blank. The two helpers are meant to be used together. + var anchor = DateTime.UtcNow.AddSeconds(-10); + await database.SetCurrentPeriodStart(request, anchor); + + request.Status = "Approved"; + await database.Context.SaveChangesAsync(); + await database.SetCurrentPeriodStart(request, anchor.AddMilliseconds(100)); + + // Blank the column on the history rows only. A column dropped and + // re-added on a temporal pair leaves exactly this: the current row is + // repopulated, the rows already in history are not, and SQL Server + // never backfills them. + await database.SetHistoryColumn( + request.Id, + nameof(TravelRequest.StatusRank), + null); + + // Materialising that history row now fails on a SqlNullValueException, + // because the model reads StatusRank into a non-nullable int. That is + // the production failure, reproduced in a test. + var exception = CatchAsync( + () => database.Context.Set() + .TemporalAll() + .Where(_ => _.Id == request.Id) + .ToListAsync()); + NotNull(exception); + + #endregion + + // The current row is untouched, so ordinary queries keep working. + var current = await database.Context.Set() + .Where(_ => _.Id == request.Id) + .Select(_ => _.StatusRank) + .SingleAsync(); + AreEqual(1, current); + } +} \ No newline at end of file diff --git a/src/EfLocalDb.Tests/TemporalTests.cs b/src/EfLocalDb.Tests/TemporalTests.cs index ecd141df..36514e95 100644 --- a/src/EfLocalDb.Tests/TemporalTests.cs +++ b/src/EfLocalDb.Tests/TemporalTests.cs @@ -77,6 +77,74 @@ public async Task EntityOverload_ReloadsRowVersion() await database.Context.SaveChangesAsync(); } + [Test] + public async Task SetHistoryColumn_LeavesCurrentRowUntouched() + { + await using var database = await instance.Build(); + var entity = new TemporalEntity { Id = Guid.NewGuid(), Property = "v1" }; + database.Context.Add(entity); + await database.Context.SaveChangesAsync(); + + var anchor = DateTime.UtcNow.AddSeconds(-30); + await database.SetCurrentPeriodStart(entity, anchor); + + entity.Property = "v2"; + await database.Context.SaveChangesAsync(); + await database.SetCurrentPeriodStart(entity, anchor.AddSeconds(1)); + + await database.SetHistoryColumn(entity.Id, nameof(TemporalEntity.Property), null); + + // The history row now holds a value the model says is impossible, which is the point: + // this is what a column dropped and re-added on a temporal pair leaves behind. + var maxPeriod = new DateTime(9999, 12, 31, 23, 59, 59, 999, DateTimeKind.Utc); + var history = await database.Context.Set() + .TemporalAll() + .Where(_ => _.Id == entity.Id && EF.Property(_, "PeriodEnd") < maxPeriod) + .Select(_ => _.Property) + .SingleAsync(); + Null(history); + + var current = await database.Context.Set() + .Where(_ => _.Id == entity.Id) + .Select(_ => _.Property) + .SingleAsync(); + AreEqual("v2", current); + } + + [Test] + public async Task SetHistoryColumn_LeavesVersioningOn() + { + await using var database = await instance.Build(); + var entity = new TemporalEntity { Id = Guid.NewGuid(), Property = "v1" }; + database.Context.Add(entity); + await database.Context.SaveChangesAsync(); + + await database.SetHistoryColumn(entity.Id, nameof(TemporalEntity.Property), null); + + // Versioning has to be off to write to a history table, so a helper that failed to turn + // it back on would silently stop every later save in the test from being versioned. + // The saves need distinct periods or SQL Server drops the zero-length history row, and + // the count below then fails for a reason that has nothing to do with versioning. + await database.SetCurrentPeriodStart(entity, DateTime.UtcNow.AddSeconds(-10)); + + entity.Property = "v2"; + await database.Context.SaveChangesAsync(); + + var versions = await database.Context.Set() + .TemporalAll() + .CountAsync(_ => _.Id == entity.Id); + AreEqual(2, versions); + } + + [Test] + public async Task SetHistoryColumn_ThrowsForPeriodColumn() + { + await using var database = await instance.Build(); + var ex = ThrowsAsync(() => + database.SetHistoryColumn(Guid.NewGuid(), "PeriodEnd", DateTime.UtcNow)); + That(ex!.Message, Does.Contain("not a settable history column")); + } + [Test] public async Task Throws_WhenEntityNotTemporal() { diff --git a/src/EfLocalDb/SqlDatabase_Temporal.cs b/src/EfLocalDb/SqlDatabase_Temporal.cs index c26f66a3..a9f12a99 100644 --- a/src/EfLocalDb/SqlDatabase_Temporal.cs +++ b/src/EfLocalDb/SqlDatabase_Temporal.cs @@ -16,6 +16,19 @@ public Task SetCurrentPeriodStart(object id, DateTime periodStart) where TEntity : class => instance.SetCurrentPeriodStart(Context, id, periodStart); + /// + /// Sets one column on every history row for , leaving the current row + /// untouched. Use in tests to reproduce a history table that a migration has left in a state + /// a freshly built database never reaches - most usefully a NULL in a column that was dropped + /// and re-added on the temporal pair, since SQL Server does not backfill such a column into + /// the rows already in history. + /// The period columns cannot be set this way: rewriting those corrupts the timeline + /// maintains. + /// + public Task SetHistoryColumn(object id, string propertyName, object? value) + where TEntity : class => + instance.SetHistoryColumn(Context, id, propertyName, value); + /// /// Convenience overload that extracts the PK from and reloads /// it from the database afterward (so the bumped RowVersion doesn't break optimistic diff --git a/src/EfLocalDb/SqlInstance_Temporal.cs b/src/EfLocalDb/SqlInstance_Temporal.cs index 6f4518e1..5536d3a6 100644 --- a/src/EfLocalDb/SqlInstance_Temporal.cs +++ b/src/EfLocalDb/SqlInstance_Temporal.cs @@ -44,6 +44,19 @@ public Task SetCurrentPeriodStart(TDbContext context, object id, DateTi where TEntity : class => ResolveSchema().Apply(context, id, periodStart); + /// + /// Sets one column on every history row for , leaving the current row + /// untouched. Use in tests to reproduce a history table that a migration has left in a state + /// a freshly built database never reaches - most usefully a NULL in a column that was dropped + /// and re-added on the temporal pair, since SQL Server does not backfill such a column into + /// the rows already in history. + /// The period columns cannot be set this way: rewriting those corrupts the timeline + /// maintains. + /// + public Task SetHistoryColumn(TDbContext context, object id, string propertyName, object? value) + where TEntity : class => + ResolveSchema().SetHistoryColumn(context, id, propertyName, value); + internal TemporalSchema ResolveSchema() where TEntity : class { diff --git a/src/EfLocalDb/TemporalSchema.cs b/src/EfLocalDb/TemporalSchema.cs index 20fb6b56..b0321a1e 100644 --- a/src/EfLocalDb/TemporalSchema.cs +++ b/src/EfLocalDb/TemporalSchema.cs @@ -4,7 +4,12 @@ sealed record TemporalSchema( string OpenSql, string UpdateSql, string CloseSql, - string KeyPropertyName) + string KeyPropertyName, + string VersioningOffSql, + string VersioningOnSql, + string HistoryTable, + string KeyColumn, + Dictionary HistoryColumns) { public static TemporalSchema? TryBuild(IReadOnlyEntityType entityType) { @@ -72,7 +77,44 @@ periodStart is null || ALTER TABLE {qTable} SET (SYSTEM_VERSIONING = ON (HISTORY_TABLE = {qHistory})); """; - return new(openSql, updateSql, closeSql, keyProperty.Name); + // Writing to a history table needs versioning off, but not the PERIOD dropped: the + // period columns are plain columns over there. So this is a lighter pair than + // openSql/closeSql, and leaves the main table's GENERATED ALWAYS definition alone. + var versioningOffSql = $"ALTER TABLE {qTable} SET (SYSTEM_VERSIONING = OFF);"; + var versioningOnSql = $"ALTER TABLE {qTable} SET (SYSTEM_VERSIONING = ON (HISTORY_TABLE = {qHistory}));"; + + // The period columns and the key are deliberately absent: rewriting a period on a + // history row breaks the timeline SetCurrentPeriodStart depends on, and silently, while + // rewriting the key detaches the row from the entity it is history for. + var historyColumns = new Dictionary(StringComparer.Ordinal); + foreach (var property in root.GetProperties()) + { + if (property.Name == periodStart || + property.Name == periodEnd || + property.Name == keyProperty.Name) + { + continue; + } + + var column = property.GetColumnName(storeObject); + if (column is null) + { + continue; + } + + historyColumns[property.Name] = $"[{column}]"; + } + + return new( + openSql, + updateSql, + closeSql, + keyProperty.Name, + versioningOffSql, + versioningOnSql, + qHistory, + qKey, + historyColumns); } public async Task Apply(DbContext db, object id, DateTime periodStart) @@ -88,8 +130,40 @@ public async Task Apply(DbContext db, object id, DateTime periodStart) } } + /// + /// Sets one column on every history row for . Reproduces states a + /// deployed database can be in but a freshly migrated one never is - most usefully a NULL + /// left behind when a column is dropped and re-added on a temporal pair, which SQL Server + /// does not backfill into the rows already in the history table. + /// The column has to permit NULL in the database for a null . + /// It cannot be widened here: SQL Server rejects re-enabling versioning when the current and + /// history tables disagree on nullability, so such a row could not exist in production + /// either. The case worth reproducing is the column that is nullable in the database while + /// the model declares the property required. + /// + public async Task SetHistoryColumn(DbContext db, object id, string propertyName, object? value) + { + if (!HistoryColumns.TryGetValue(propertyName, out var column)) + { + throw new InvalidOperationException( + $"'{propertyName}' is not a settable history column. The period columns are " + + $"excluded, since changing those corrupts the temporal timeline. Available: " + + string.Join(", ", HistoryColumns.Keys.Order())); + } + + await Exec(db, VersioningOffSql); + try + { + await Exec(db, $"UPDATE {HistoryTable} SET {column} = {{0}} WHERE {KeyColumn} = {{1}};", value, id); + } + finally + { + await Exec(db, VersioningOnSql); + } + } + // Identifiers (table/column names) come from the EF model so cannot carry user input; // values are passed as positional parameters via FormattableString. - static Task Exec(DbContext db, string sql, params object[] args) => + static Task Exec(DbContext db, string sql, params object?[] args) => db.Database.ExecuteSqlAsync(FormattableStringFactory.Create(sql, args)); }