[IN PROGRESS] Migration engine for the primary instance

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
         # Categories are declared by the <TestCategory> property in each test project. A category can
         # span several projects that share infrastructure, so that the container is provisioned once
         # and the union of their build closures is compiled once.
-        test-category: [ DefaultCore, DefaultMonitoring, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ]
+        test-category: [ DefaultCore, DefaultMonitoring, Migration, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ]
         include:
           - os: windows-latest
             os-name: Windows
@@ -87,7 +87,7 @@ jobs:
         id: select
         run: ./tools/select-test-projects.ps1 -Category ${{ matrix.test-category }}
       - name: Download RavenDB Server
-        if: matrix.test-category == 'DefaultCore'
+        if: contains(fromJSON('["DefaultCore", "Migration"]'), matrix.test-category)
         run: ./tools/download-ravendb-server.ps1
       - name: Build
         id: build

README.md

@@ -33,6 +33,10 @@ It's also possible to [locally test containers built from PRs in GitHub Containe
 If the instance is executed for the first time, it must set up the required infrastructure. To do so, once the instance is configured to use the selected transport and persister, run it in setup mode. This can be done by using the `Setup {instance name}` launch profile that is defined in 
 the `launchSettings.json` file of each instance. When started in setup mode, the instance will start as usual, execute the setup process, and exit. At this point the instance can be run normally by using the non-setup launch profile.
 
+## Migrating from RavenDB to SQL Server or PostgreSQL
+
+- [Migrate from RavenDB to SQL Server or PostgreSQL](docs/migration/ravendb-to-sql-migration-instructions.md): what an operator can run today, and what is not built yet.
+
 ## Secrets
 
 Testing using the [CI workflow](/.github/workflows/ci.yml) depends on the following secrets. The Particular values for these secrets are stored in the secure note named **ServiceControl Repo Secrets**.

src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj

@@ -12,6 +12,9 @@
   <ItemGroup>
     <ProjectReference Include="..\ServiceControl.AcceptanceTesting\ServiceControl.AcceptanceTesting.csproj" />
     <ProjectReference Include="..\ServiceControl.Persistence.EFCore.PostgreSql\ServiceControl.Persistence.EFCore.PostgreSql.csproj" />
+    <!-- Built so its manifest is where DevelopmentPersistenceLocations looks; not copied, or its persistence.manifest would collide with this project's own. -->
+    <ProjectReference Include="..\ServiceControl.Persistence.RavenDB\ServiceControl.Persistence.RavenDB.csproj" ReferenceOutputAssembly="false" Private="false" />
+    <ProjectReference Include="..\ServiceControl.RavenDB\ServiceControl.RavenDB.csproj" />
     <ProjectReference Include="..\ServiceControl.Transports.Learning\ServiceControl.Transports.Learning.csproj" />
     <ProjectReference Include="..\ServiceControl\ServiceControl.csproj" />
     <ProjectReference Include="..\TestHelper\TestHelper.csproj" />
@@ -20,6 +23,7 @@
   <ItemGroup>
     <PackageReference Include="GitHubActionsTestLogger" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
+    <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="NServiceBus.Heartbeat" />
     <PackageReference Include="NServiceBus.SagaAudit" />
@@ -31,6 +35,9 @@
 
   <ItemGroup>
     <Compile Include="..\ServiceControl.AcceptanceTests\**\*.cs" LinkBase="Shared" />
+    <Compile Include="..\ServiceControl.Migration.AcceptanceTests\**\*.cs" LinkBase="Migration" />
+    <Compile Include="..\ServiceControl.Persistence.Tests\IngestedFailure.cs" />
+    <Compile Include="..\ServiceControl.Persistence.Tests\EFCore\InMemoryBodyStoragePersistence.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\PostgreSqlSharedContainer.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\TestSchema.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\StopSharedPostgreSql.cs" />

src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj

@@ -12,6 +12,9 @@
   <ItemGroup>
     <ProjectReference Include="..\ServiceControl.AcceptanceTesting\ServiceControl.AcceptanceTesting.csproj" />
     <ProjectReference Include="..\ServiceControl.Persistence.EFCore.SqlServer\ServiceControl.Persistence.EFCore.SqlServer.csproj" />
+    <!-- Built so its manifest is where DevelopmentPersistenceLocations looks; not copied, or its persistence.manifest would collide with this project's own. -->
+    <ProjectReference Include="..\ServiceControl.Persistence.RavenDB\ServiceControl.Persistence.RavenDB.csproj" ReferenceOutputAssembly="false" Private="false" />
+    <ProjectReference Include="..\ServiceControl.RavenDB\ServiceControl.RavenDB.csproj" />
     <ProjectReference Include="..\ServiceControl.Transports.Learning\ServiceControl.Transports.Learning.csproj" />
     <ProjectReference Include="..\ServiceControl\ServiceControl.csproj" />
     <ProjectReference Include="..\TestHelper\TestHelper.csproj" />
@@ -20,6 +23,7 @@
   <ItemGroup>
     <PackageReference Include="GitHubActionsTestLogger" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
+    <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="NServiceBus.Heartbeat" />
     <PackageReference Include="NServiceBus.SagaAudit" />
@@ -31,6 +35,9 @@
 
   <ItemGroup>
     <Compile Include="..\ServiceControl.AcceptanceTests\**\*.cs" LinkBase="Shared" />
+    <Compile Include="..\ServiceControl.Migration.AcceptanceTests\**\*.cs" LinkBase="Migration" />
+    <Compile Include="..\ServiceControl.Persistence.Tests\IngestedFailure.cs" />
+    <Compile Include="..\ServiceControl.Persistence.Tests\EFCore\InMemoryBodyStoragePersistence.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.SqlServer\SqlServerSharedContainer.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.SqlServer\TestSchema.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.SqlServer\StopSharedSqlServer.cs" />

src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs

@@ -29,10 +29,9 @@ namespace ServiceControl.AcceptanceTests.Recoverability
     using ServiceControl.Infrastructure;
     using ServiceControl.MessageFailures;
     using ServiceControl.Operations;
+    using ServiceControl.Persistence.DataMigration;
     using ServiceControl.Persistence.EFCore.DbContexts;
     using ServiceControl.Persistence.EFCore.Entities;
-    using ServiceControl.Persistence.EFCore.Infrastructure;
-    using ServiceControl.Recoverability;
     using ServiceControl.Transports;
 
     class When_hosting_error_ingestion_only : AcceptanceTest
@@ -97,9 +96,36 @@ public async Task Should_default_the_concurrency_to_10_when_none_is_configured()
             "InternalCustomChecksHostedService",    // reports this node's ingestion health to the database
             "MetricsReporterHostedService",
             "HealthCheckPublisherHostedService",  // inert, no IHealthCheckPublisher is registered
-            "ExternalIntegrationRequestsDataStore"  // its drain is inert here, nothing calls Subscribe
+            "ExternalIntegrationRequestsDataStore",  // its drain is inert here, nothing calls Subscribe
+            "FinishedCopyBeforeAnIngestionNodeOpens"  // keeps this node out of a database a copy has not finished filling
         ];
 
+        [Test]
+        public async Task Should_refuse_to_start_while_a_copy_into_the_database_is_unfinished()
+        {
+            var settings = await CreateSettings();
+
+            await new SetupCommand().Execute(new HostArguments([]), settings);
+
+            var host = ErrorIngestionOnlyCommand.BuildHost(settings);
+
+            try
+            {
+                await host.Services.GetRequiredService<IMigrationCheckpointStore>().Upsert(
+                    new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Halted, null, 0, 0, null, null, null, null, null, "The source stopped responding."));
+
+                var exception = Assert.ThrowsAsync<Exception>(() => host.StartAsync());
+
+                Assert.That(exception.Message, Does.Contain("has not finished")
+                    .And.Contain(MigrationCategoryIds.KnownEndpoints)
+                    .And.Contain("--migration-abandon KnownEndpoints"));
+            }
+            finally
+            {
+                await host.DisposeAsync();
+            }
+        }
+
         [Test]
         public void Should_refuse_to_start_against_unsupported_storage()
         {
@@ -204,6 +230,7 @@ public async Task Should_serve_health_over_https_with_the_configured_certificate
                 await new SetupCommand().Execute(new HostArguments([]), settings);
 
                 host = ErrorIngestionOnlyCommand.BuildHost(settings);
+                host.Urls.Clear();
                 host.Urls.Add("https://127.0.0.1:0");
 
                 await host.StartAsync();

src/ServiceControl.Migration.AcceptanceTests/.editorconfig

@@ -0,0 +1,6 @@
+[*.cs]
+
+# Justification: Test project
+dotnet_diagnostic.CA2007.severity = none
+dotnet_diagnostic.PS0013.severity = none
+dotnet_diagnostic.PS0018.severity = none