diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f9632f42a..c0419fceb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: # Categories are declared by the property in each test project. A category can # span several projects that share infrastructure, so that the container is provisioned once # and the union of their build closures is compiled once. - test-category: [ DefaultCore, DefaultMonitoring, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ] + test-category: [ DefaultCore, DefaultMonitoring, Migration, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ] include: - os: windows-latest os-name: Windows @@ -87,7 +87,7 @@ jobs: id: select run: ./tools/select-test-projects.ps1 -Category ${{ matrix.test-category }} - name: Download RavenDB Server - if: matrix.test-category == 'DefaultCore' + if: contains(fromJSON('["DefaultCore", "Migration"]'), matrix.test-category) run: ./tools/download-ravendb-server.ps1 - name: Build id: build diff --git a/README.md b/README.md index f2e4c32b88..08413b4950 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ It's also possible to [locally test containers built from PRs in GitHub Containe If the instance is executed for the first time, it must set up the required infrastructure. To do so, once the instance is configured to use the selected transport and persister, run it in setup mode. This can be done by using the `Setup {instance name}` launch profile that is defined in the `launchSettings.json` file of each instance. When started in setup mode, the instance will start as usual, execute the setup process, and exit. At this point the instance can be run normally by using the non-setup launch profile. +## Migrating from RavenDB to SQL Server or PostgreSQL + +- [Migrate from RavenDB to SQL Server or PostgreSQL](docs/migration/ravendb-to-sql-migration-instructions.md): what an operator can run today, and what is not built yet. + ## Secrets Testing using the [CI workflow](/.github/workflows/ci.yml) depends on the following secrets. The Particular values for these secrets are stored in the secure note named **ServiceControl Repo Secrets**. diff --git a/docs/migration/migration-system-design-diagram.png b/docs/migration/migration-system-design-diagram.png new file mode 100644 index 0000000000..85636355e7 Binary files /dev/null and b/docs/migration/migration-system-design-diagram.png differ diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md new file mode 100644 index 0000000000..bf7a425abc --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -0,0 +1,234 @@ +# Migration Engine: Instructions + +> [!NOTE] +> This build does not copy data yet. Only `--migration-source-report` works, and ServiceControl does not start if `ServiceControl/Migration/Enabled` is `true`. + +This page tells you how to move the data of a ServiceControl error instance from RavenDB to SQL Server or PostgreSQL. ServiceControl copies the data itself, so you do not need a separate tool. [The migration overview](ravendb-to-sql-migration-overview.md) explains how the migration works and why. + +The migration does not move the audit instance or the monitoring instance. The audit instance has no migration path and the monitoring instance keeps its data in memory, so it has nothing to move. + +## How the migration runs + +ServiceControl copies the data it needs to open while it is closed. **This part is your outage**. Then it opens and copies the event log and the archived and resolved failed messages in the background. **The migration never writes to RavenDB**. + +> [!IMPORTANT] +> Until ServiceControl opens, you can go back to RavenDB and lose nothing. After it opens, you cannot go back. + +The migration copies the data in categories, and each category is Copying, Done, Failed or Abandoned. [The overview](ravendb-to-sql-migration-overview.md#when-something-goes-wrong-you-decide-what-happens-next) explains the categories and the states. + +## Before you start + +Make sure that your setup is supported: + +- The RavenDB source is one of these: + - embedded on the host (Windows installation only) + - self-hosted on your network, or RavenDB Cloud +- The primary database and the throughput database of RavenDB are on the same server or cluster. +- The SQL target is SQL Server or PostgreSQL. +- A SQL Server target has Full-Text Search installed. Message search needs it, so `--setup` fails without it. The stock SQL Server container image does not have it. PostgreSQL needs nothing extra. +- The ServiceControl host can connect to RavenDB, to the SQL database, and to the message body storage at the same time. +- If ServiceControl runs in a container, RavenDB is an external server. The container image does not contain the RavenDB server, so it cannot open an embedded database. + +Get the data ready: + +- Resolve or archive as many unresolved failed messages as you can. They never age out, and ServiceControl stays closed until they are all copied. **A large backlog makes the outage long**. +- Run `--import-failed-errors` while the instance is still on RavenDB. A failed error import is a message that ServiceControl took off the error queue but failed to store. The migration copies these either way, but imported ones arrive as ordinary failed messages. + +Keep these settings as they are: + +- Keep `ServiceControl/RetryHistoryDepth` above 0. If it is 0 or less, the migration refuses to start. At that value, the first completed retry after the move deletes all the copied retry history. +- Do not change `ServiceControl/ErrorRetentionPeriod` or `ServiceControl/EventRetentionPeriod` until the migration is finished. The migration uses them to decide which rows are too old to copy. If one changes after the copy starts, ServiceControl refuses to start. + +## Keep the RavenDB settings + +The migration reads RavenDB through the RavenDB settings that the instance already has. You do not add new settings for the source. Keep these settings in the configuration of the instance, also after you change `ServiceControl/PersistenceType` to SQL Server or PostgreSQL. + +| Setting | Environment variable | What it is | +| --- | --- | --- | +| `ServiceControl/RavenDB/ConnectionString` | `SERVICECONTROL_RAVENDB_CONNECTIONSTRING` | The address of an external RavenDB server. Leave it unset for an embedded database. | +| `ServiceControl/DbPath` | `SERVICECONTROL_DBPATH` | The data directory of the embedded database. | +| `ServiceControl/RavenDB/DatabaseName` | `SERVICECONTROL_RAVENDB_DATABASENAME` | The primary database. The default is `primary`. | +| `LicensingComponent/RavenDB/ThroughputDatabaseName` | `LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME` | The throughput database. The default is `throughput`. | +| `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`, with `ServiceControl/RavenDB/ClientCertificatePassword` | `SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEPATH` and so on | The client certificate for a secured external server. | +| `ServiceControl/ErrorRetentionPeriod` | `SERVICECONTROL_ERRORRETENTIONPERIOD` | Required. ServiceControl does not start without it. | + +In an environment variable, you can leave out the `SERVICECONTROL_` prefix. You cannot leave out the `LICENSINGCOMPONENT_` prefix. + +## Step 1: Upgrade ServiceControl and make the source report + +> [!IMPORTANT] +> +> - Upgrade the instance to the latest version in the usual way, and keep it on RavenDB. The version that copies the data must be the version that last ran against RavenDB. +> - Do not start ServiceControl again until step 6. If ServiceControl starts on the new database without the migration turned on, it writes rows to the database, and the migration then refuses that database. If this happens, start again from a new, empty database. + +Then make the source report. The report shows what ServiceControl can read from RavenDB, so you find a wrong setting early. Start the instance executable with the `--migration-source-report` argument: + +```powershell +# Windows installation. Start this from the installation folder of the instance. +.\ServiceControl.exe --migration-source-report +``` + +```shell +# Container, against an external RavenDB server. +docker run --rm --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --migration-source-report +``` + +Where you run the report depends on the source: + +- External server: run the report while ServiceControl runs. The report only reads. +- Embedded database: stop the ServiceControl service first. Run the report, then start the service again. The report starts its own RavenDB process on the data directory, and it cannot do that while the instance holds the directory. + +The report prints: + +- The source persistence that it read. +- The RavenDB server version. +- Whether the source is embedded or external, and where it is. +- The name of each of the two databases, and the setting that each name came from. +- A row count for every collection in both databases. + +The report exits with 0 when it prints a report, and with 1 when it cannot. If the report fails, read [Errors from the RavenDB source](#errors-from-the-ravendb-source). + +## Step 2: Create an empty SQL database + +1. Create a new database on SQL Server or PostgreSQL. +2. In the configuration of the instance, set `ServiceControl/PersistenceType` to `SQLServer` or `PostgreSQL`. +3. Set `ServiceControl/Database/ConnectionString` to the new database. +4. Set `ServiceControl/MessageBody/StorageType` to `FileSystem`, `AzureBlob` or `S3`, and set the settings for that storage type. +5. Run `ServiceControl.exe --setup`. This creates the SQL schema, which includes the table where the migration saves its progress. + +## Step 3: Turn the migration on + +1. Set `ServiceControl/Migration/Enabled` to `true`. +2. If you want less optional data, set a shorter window for the optional categories. A window is how far back an optional category copies. Set it to `0` to turn the category off. + +| Setting | Default | What it does | +| --- | --- | --- | +| `ServiceControl/Migration/EventLogWindow` | The value of `ServiceControl/EventRetentionPeriod` | How far back the event log copy goes. | +| `ServiceControl/Migration/ArchivedAndResolvedFailedMessagesWindow` | The value of `ServiceControl/ErrorRetentionPeriod` | How far back the copy of archived and resolved messages goes. | + +By default, the migration copies all the optional data that RavenDB still holds (current retension period setting). A window longer than the retention period copies nothing extra. After a category starts copying, you cannot change its window. To stop it early, abandon it. + +## Step 4: Run the dry run + +The dry run reads RavenDB and tells you what the migration will do, including a range for how long ServiceControl will be closed. It does not write anything. Run it with the `--migration-dry-run` argument, in the same way as the source report. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) lists what it reports. + +Book the outage from the range in the report. The range is the shortest time the outage can take, not a promise. If the report says that a required category will end Failed and that a retry cannot fix it, plan to abandon that category after it fails. + +## Step 5: Back up RavenDB + +> [!IMPORTANT] +> Back up both RavenDB databases, primary and throughput, before you start the copy. Or turn off RavenDB expiration on them. + +RavenDB keeps deleting old failed messages and event log items during the migration and after it. Without a backup, the RavenDB database that you keep to go back to loses data every day. + +## Step 6: Start ServiceControl + +Start the ServiceControl service, or the container. ServiceControl then does these things in order: + +1. It runs every startup check. If a check fails, ServiceControl does not start and names the check. It copies nothing. +2. It copies every required category while it is closed. If a required category fails, ServiceControl stays closed. It says which category failed, why, and which command to run. +3. It opens when every required category is Done or Abandoned. You do not need to restart it. +4. It copies the optional categories in the background. + +> [!IMPORTANT] +> Do not start any `--error-ingestion-only` worker until ServiceControl opens. A worker refuses to start while a required category is unfinished, but a worker that starts at the same time as ServiceControl can start before the copy saves its first progress, and nothing stops it. + +If something goes wrong, read [If something goes wrong](#if-something-goes-wrong). + +## Step 7: Watch the background copy + +ServiceControl is open and works as usual while the optional categories copy. You can see the progress in three places: + +- The ServicePulse activity feed. It shows when a category fails and when RavenDB cannot be reached. +- The ServiceControl log. +- The output of `--migration-status`. This command does not open RavenDB, so you can run it at any time. + +If the copy slows down your production work, raise `ServiceControl/Migration/ThrottlePauseMilliseconds` and restart. This is the pause between two background batches. The default is 100 milliseconds. + +> [!IMPORTANT] +> Turning the migration off does not stop the copy, because ServiceControl refuses to start until every category is Done or Abandoned. See [how to abandon a category](#a-category-is-failed). + +Keep RavenDB running until every category is Done or Abandoned. If RavenDB goes down after ServiceControl opens, ServiceControl keeps running. The optional categories wait for RavenDB to come back, or you can abandon them. + +## Step 8: Finish the migration + +1. Stop ServiceControl. +2. Run `--migration-verify`. It counts the rows in RavenDB again, and compares them with the rows that the copy read. It also shows the skipped and merged rows. It exits with 0 only when every category is Done or Abandoned. +3. Set `ServiceControl/Migration/Enabled` to `false`. +4. Start ServiceControl. + +Verify reads all the data in RavenDB again, but not the message bodies. On a large instance this takes time, and ServiceControl is stopped during that time. + +If verify shows rows that the copy did not read, keep RavenDB. Those rows are only in RavenDB. RavenDB can show fewer rows than the copy read when nothing is wrong, because RavenDB deletes old rows. [The overview](ravendb-to-sql-migration-overview.md#the-dry-run) explains why. + +If you turn the migration off before every category is Done or Abandoned, ServiceControl refuses to start. The refusal names each unfinished category and the ways to continue. + +## If something goes wrong + +### A startup check fails + +ServiceControl does not start, and its log names the check that failed. Nothing is copied. Correct the cause, then start ServiceControl again. The dry run runs the same checks, so you can use it to test your correction. + +### A category is Failed + +A category becomes Failed when it cannot copy rows, or when an error stops it. The rows that it already copied stay in SQL. A restart does not change a Failed category. You must run one of two commands. + +#### To retry a category + +1. Stop ServiceControl. +2. Read why the category failed. The reason is in the startup refusal and in the output of `--migration-status`. +3. Correct the cause. It is usually outside the migration, for example the body storage cannot be reached, a certificate expired, a database is down, or a disk is full. +4. Run `--migration-retry ` to reset the migration progress for that category. +5. Start ServiceControl. It copies the category again from the start. + +#### To abandon a category + +1. Stop ServiceControl. +2. Run `--migration-abandon `. When you abandon a category, the categories that depend on it are abandoned too. +3. Start ServiceControl. + +> [!IMPORTANT] +> **Abandon is final**. The rows that the category already copied stay in SQL, and the rest are not copied. Abandon a category when a retry cannot fix it, or when the data is not worth the time. You cannot retry the event log, so you can only abandon it. + +You can abandon a required category only after it starts copying. You can abandon an optional category at any time, for example to stop a large copy early. + +### You want to go back to RavenDB + +You can go back to RavenDB without losing data only while ServiceControl is still closed: + +1. Set `ServiceControl/Migration/Enabled` to `false`. +2. Set `ServiceControl/PersistenceType` back to RavenDB. +3. Start ServiceControl. + +You lose only the copy. To try the migration again later, start from step 2 with a new, empty SQL database. + +> [!IMPORTANT] +> Do not use the old copy again. The migration skips the categories that it finished, and misses everything that RavenDB received after that. + +After ServiceControl opens on SQL, you cannot go back. You can only finish the migration, or abandon what is left. + +### Errors from the RavenDB source + +The source report, the dry run and the migration show these errors when they cannot read RavenDB: + +- `has no database named ...`: the database name in the setting that the message names is wrong. Correct that setting. +- `refused its client certificate access ...`: give that certificate Read access to the database, or use a certificate that already has this access. +- `is secured but no client certificate is configured`: the connection string starts with `https://`. Set `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`. +- `is valid from ... to ..., which does not include now`: the client certificate expired, or it is not valid yet. Use a current certificate. +- `ServiceControl expects RavenDB Server version ... or higher`: the external RavenDB server is older than this version of ServiceControl supports. Upgrade the RavenDB server. +- `could not start a server for the embedded database ...`: the rest of the line gives the reason. Usually a ServiceControl instance still holds the data directory. Stop that instance and try again. The other reasons are a port that is already in use, and a missing RavenDB server. +- `did not finish loading within 5 minutes`: another process holds the embedded data directory, or the directory is damaged. You cannot change the 5 minutes. + +## Commands and settings + +On an embedded source, stop ServiceControl before you run a command that opens RavenDB. **Only one RavenDB process can use the data directory**. The overview lists [every command and when it can run](ravendb-to-sql-migration-overview.md#the-dry-run), [the retry and abandon commands](ravendb-to-sql-migration-overview.md#retry-and-abandon), and [every migration setting](ravendb-to-sql-migration-overview.md#monitoring-and-settings). + +### Run a command in a container + +In a container, run each command as a one-off container of the same image, against the same databases: + +```shell +docker run --rm --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --migration-status +``` + +The RavenDB source must be an external server. diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md new file mode 100644 index 0000000000..613dca1680 --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -0,0 +1,496 @@ +# Migration Engine: Design Overview + +> [!NOTE] +> This build does not carry the whole migration yet. This page describes it as it will be when it ships. + +This page describes the migration as designed, not what is built today. [The instructions](ravendb-to-sql-migration-instructions.md) describe how an operator runs the migration, and the [system design](ravendb-to-sql-migration-system-design.md) provides the system architecture. See the [glossary](#glossary) for terms used throughout. + +## Overview + +### What this is + +- The migration moves an **error instance's** data from RavenDB to SQL Server or PostgreSQL, so a customer can switch persisters and keep their data. +- It runs inside ServiceControl itself. There is no separate tool. +- **The audit instance is out of scope.** There is no migration path for the audit instance. +- **The monitoring instance is out of scope.** It keeps its data in memory, so there is nothing to move. + +### TL;DR + +- **Retention does most of the work.** Error retention is 5 to 45 days and event retention defaults to 14 days, so most of RavenDB ages out on its own within weeks. +- **What the instance needs to open is copied while it is closed.** This is the *required* data. Most of it is small. +- **Everything else is copied in the background while it runs.** This is the *optional* data: the event log and archived and resolved messages. +- **Anything not copied ages out of RavenDB**, and the customer deletes the old database when ready. +- **The main thing that makes the outage long is a big backlog of unresolved failures.** They never age out, they are required, and they can legitimately be months old. The strategy assumes you keep them low by resolving and archiving. The *dry run* tells you which case you are in. + +```mermaid +flowchart LR + A["Checks"] --> B["Copy required data
ServiceControl closed"] + B --> V["Required all Done
or Abandoned"] --> C["ServiceControl opens
optional data copies
in the background"] + C --> D["Everything Done or
Abandoned: turn
migration off"] +``` + +### When something goes wrong, you decide what happens next + +The end goal is simple: **either all the data comes across, or you explicitly give up on a named part of it.** + +- **Each *category* of data is in one of four states:** Copying, Done, Failed or Abandoned. +- **A category goes Failed when something went wrong**, for example a row it could not copy. It stays Failed, and the status says why. +- **You act with one of two commands**, with ServiceControl stopped: + - `--migration-retry `, after fixing the cause: the next start copies the category again from the beginning. + - `--migration-abandon `: give up on what it has not copied, and keep what it has. +- **A restart never decides anything for you.** A Failed category stays Failed until you run a command, so a restart policy cannot loop or lose data. +- **ServiceControl opens once every required category is Done or Abandoned**, and the migration is finished once every category is. + +```mermaid +stateDiagram-v2 + [*] --> Copying + Copying --> Done + Copying --> Failed: something went wrong + Failed --> Copying: --migration-retry + Failed --> Abandoned: --migration-abandon + Copying --> Abandoned: --migration-abandon + Done --> [*] + Abandoned --> [*] +``` + +### Goals + +- **Short outage.** Only the required data copies while ServiceControl is closed. +- **Support every RavenDB hosting shape.** Embedded, self-hosted or RavenDB Cloud, on one code path. +- **Never write to RavenDB.** But RavenDB's own expiration keeps deleting old failed messages and event log items, during the migration and afterwards. So the old database is a fallback that degrades from the moment you start, which is why [the instructions](ravendb-to-sql-migration-instructions.md#step-5-back-up-ravendb) have you back it up first. +- **No duplicates and no gaps.** The copied rows and the *checkpoint* commit together, so a crash needs no clean-up. +- **Every identifier anything points at is kept.** Only the event log, historic retry operations and pending integration events get new ids, because nothing points at theirs. +- **Refuses rather than half-migrates.** Every check runs before the first row moves. +- **No silent loss.** Every skipped row is counted with a reason, and a migration cannot finish with a category unfinished unless someone deliberately *abandons* it. +- **Gentle on a live instance.** The copy is streamed, so memory does not grow with the database, and there is a pause between background batches. +- **Known before it starts, visible while it runs.** The dry run predicts the outage, and progress shows up as it happens. + +### Non-goals + +- **Zero downtime.** There is a real, if short, outage. +- **Rollback once ServiceControl opens.** Nothing copies SQL rows back to RavenDB. +- **Pausing or steering a running copy.** There is no pause, resume or abort. Decisions are two commands run with ServiceControl stopped, and every other change is a configuration edit and a restart. +- **A general-purpose migration tool.** The source is always RavenDB and the target is always one of ServiceControl's SQL persisters, both at versions this build can read. The migration engine was design to be expandable in the future. + +### The moving parts + +```mermaid +flowchart LR + subgraph host["One ServiceControl process"] + src["RavenDB
source"] --> eng["Migration
engine"] --> tgt["SQL
target"] + end + old[("RavenDB
read only")] --> src + tgt --> sql[("SQL Server or PostgreSQL
plus the checkpoint table")] + tgt --> bod[("Body store
filesystem, Blob or S3")] +``` + +- **Both persisters load into one ServiceControl process**, each in its own `AssemblyLoadContext`, and both are live during the copy. +- **The engine knows neither database.** It knows only `IMigrationSource` and `IMigrationTarget`, and deals in categories, *cursors* and counts. This allows it to be tested, and expanded in the future. + - The source maps each category to what it reads, and describes itself as labelled facts. + - The target maps each category to where it writes and how to count it. + - Each side adds its own startup checks. + - The engine passes the cursor from one side to the other without looking inside it. + - RavenDB to SQL is the only supported pair, but the engine does not depend on it. +- **The source reuses the instance's existing RavenDB settings**, so an existing customer configures nothing new for it. Leave them in place when switching `ServiceControl/PersistenceType`. + +### A migration from start to finish + +[The instructions](ravendb-to-sql-migration-instructions.md) define the migration steps. + +- **The [free abort](#going-back-to-ravendb) window runs from the first start with migration on until ServiceControl opens.** Until then, you can go back to RavenDB and lose only the copy. +- **While required data is still copying, none of ServiceControl's own clean-up runs**: the retention sweep, the purge API, the heartbeat settings sync and throughput collection. The required copy runs before any of them starts, so they start only once it finishes. Optional categories never hold them back. + +## Design + +### Supported locations + +The copy goes from RavenDB, through the ServiceControl host, to SQL. There is no database-to-database transfer. RavenDB holds documents and SQL holds tables, so the rows have to be reshaped, and ServiceControl's own persisters already know how to read one and write the other. So what matters is whether the host can reach both ends, and how far the data travels. + +| | Supported locations | +| --- | --- | +| RavenDB source | Embedded on the host (Windows installations only), self-hosted on the network (container, VM or bare metal), or RavenDB Cloud | +| SQL target | SQL Server or PostgreSQL on the host, on the network, in a container, or managed (Azure SQL, Amazon RDS, Google Cloud SQL) | +| Body store | Wherever the target is configured to put bodies: filesystem, Azure Blob or S3. Small text bodies stay in the database. This choice is made at the same time as the database move | + +| Route | What to expect | +| --- | --- | +| On-prem to on-prem | The common case, and the fastest: embedded or self-hosted RavenDB to SQL on the same host or network | +| On-prem to cloud | Works. Each batch is a round trip, so write latency adds up per batch | +| Cloud to on-prem | Works. The customer pays egress on everything copied, mostly archived messages and their bodies | +| Cloud to cloud | Works, but only sensible when ServiceControl runs next to one of them. A host on-prem between two clouds pulls every byte down and pushes it straight back up | + +**Requirements:** + +- The host reaches the RavenDB source, the SQL target and the body store at the same time. +- The SQL database is new and empty: only the schema `--setup` created. +- Both RavenDB databases, primary and throughput, are on one server or cluster. The migration opens both through one `IDocumentStore`, and so does the licensing component. +- A SQL Server target has Full-Text Search installed, because message search needs it. + - The `AddFullTextSearch` EF Core migration checks `SERVERPROPERTY('IsFullTextInstalled')` and fails if it is absent. + - An EF Core migration runs once, so this is checked the first time `--setup` brings a database up to that migration, not on every `--setup`. + - A stock SQL Server container image does not include Full-Text Search. + - PostgreSQL needs nothing extra, because its search index is a GIN index over `to_tsvector`. +- Transient failures on a managed target are already survivable. Retry on failure is on by default, and no setting turns it off. + +**Not supported:** + +- Any server-to-server copy: backup and restore, RavenDB ETL or replication, or an external pipeline. +- A host that can reach only one database at a time, so no staged move through an offline copy. +- An embedded RavenDB source when ServiceControl runs in a container. Reading an embedded database means starting a RavenDB server process, and the container image does not include one. +- Primary and throughput RavenDB databases in different places. +- Any source other than RavenDB, or any target other than a ServiceControl SQL persister. + +### What data moves + +There are eighteen *categories*: sixteen required and two optional. + +| Category | Kind | Why | +| --- | --- | --- | +| Unresolved and retry-issued failed messages, with bodies | Required | The point of the instance. Issuing a retry removes the expiry, so retry-issued messages never age out. Leave one behind and the retry confirmation finds no row to resolve, so the message stays missing even though the retry succeeded | +| Message redirects | Required | | +| Known endpoints, with the monitored flag | Required | The flag is part of the endpoint row | +| Endpoint settings | Required | Copied after known endpoints | +| Notification settings | Required | | +| Licence trial end date | Required | | +| Licensing endpoint records | Required | The per-endpoint rows in the throughput database | +| Throughput history | Required | Copied after licensing endpoint records, because each day's row hangs off one | +| Retry operations, unacknowledged and historic | Required | One category, because RavenDB holds both lists in one document | +| Licensing report masks | Required | | +| Uploaded licensed endpoint details file | Required | Nothing recomputes it. Losing it means downloading it again from the licence portal and uploading it again | +| Subscriptions | Required | | +| Integration events still waiting to be sent | Required | Each is an event a subscriber has not received yet. There are only any if the old instance fell behind or could not reach the broker. They are sent once ServiceControl opens, later than they would have been. RavenDB sends them in no particular order anyway, so no ordering is lost | +| Custom checks, with their last status | Required | Not every check reports again. An endpoint that is down never does, and a check with no repeat interval reports only when its endpoint starts. Leaving one out could hide a known failure until that endpoint restarts | +| Failed error imports, with bodies | Required | Messages taken off the error queue but not ingested. They exist nowhere else and never expire. After the move the "Error Message Ingestion" custom check keeps flagging them and `--import-failed-errors` imports them into SQL. Importing them on RavenDB first is better still | +| Group comments | Required | Copied after unresolved failed messages, so a note like "do not retry" is there when ServiceControl opens. Blank comments are not copied. A comment on a group whose messages are all archived or resolved is removed by ServiceControl's usual clean-up soon after it opens, because that group has no failed messages in SQL yet | +| Event log | Optional | The history behind the ServicePulse activity feed. On a busy instance it holds a row for every failure, retry and submission | +| Archived and resolved failed messages, with bodies | Optional | The biggest category by far, and most of the copying time | + +**Never copied, because nothing needs them:** + +- RavenDB index definitions. +- In-flight work collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `FailedMessageRetries`, `ArchiveOperations` and `UnarchiveOperations`. +- `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works. +- The `ConnectedApplications` document, written only by versions 6.0 and 6.1. Since 6.2 the MassTransit connector status, which ServicePulse uses to turn features on and off, comes from the connector's own heartbeat. ServiceControl holds that in memory and refills it when the connector next reports, so nothing reads the document. +- Broker and audit service version details, which refill on the next throughput collection. +- Failed message edit locks, which stop one failed message being edited twice. An edited message is resolved, so the lock only matters if it fails again: on RavenDB it can then never be edited again, and after the move it can be edited once more. +- Heartbeat state, which neither persister stores and which is rebuilt in memory after any restart. The known endpoints and monitored flags are copied, so heartbeat monitoring after the move behaves exactly as after any restart. An endpoint that is down shows as failing with no last heartbeat time, and raises no alert. + +### Optional data windows + +- **Each optional category copies only rows inside its *window*:** events raised within it, or messages archived or resolved within it. +- **The window defaults to the instance's own retention period**: `ServiceControl/EventRetentionPeriod` for the event log, `ServiceControl/ErrorRetentionPeriod` for archived and resolved messages. So by default everything RavenDB still holds is copied. +- **`0` turns a category off before it starts.** A longer window than the retention period copies nothing extra, because older rows would only be deleted by SQL's clean-up. +- **The window counts back from when the category first started copying**, which the checkpoint records, so a restart copies the same window rather than one that has slid forward. +- **The window is fixed once the category starts.** A changed value, including a changed retention period behind a default window, is refused at startup, naming the category. To stop a started category short, abandon it. + +### Data that changes shape + +- **Attempt history collapses to the newest attempt.** SQL has no attempts table, so a message that failed five times arrives showing one attempt. This applies to every failed message, whatever its status. +- **Subscriptions that differ only in message-type version merge into one row**, because the SQL key leaves the version out. +- **On SQL Server, endpoint settings whose names differ only in case merge into one row.** The name column compares without case by default. PostgreSQL keeps both. + - The name column's own collation decides, not the database default. A case-sensitive database whose name column was given a case-insensitive collation still merges. + - The copy keeps the first in RavenDB document-id order. That id is a hash of the name, so which one survives is effectively arbitrary. + - The copy logs a warning naming both endpoints and the one it kept. + - The dry run lists each pair by asking SQL Server how the column compares. For unusual characters its list can differ from what the copy does. +- **Event log items, historic retry operations and pending integration events get new numbers.** Nothing refers to the old ones, so this is safe. + +The dry run counts both kinds of merge before anything moves. + +### Skipped rows + +Every skipped row is counted under a reason, and its id is written to the log. + +> [!IMPORTANT] +> The log is the only record of each skipped row. Look them up in RavenDB while you still have it. + +- *Fault* skips mean something went wrong. +- *Harmless* skips are rows SQL would have removed anyway. + +| Reason | Kind | Can a retry fix it? | +| --- | --- | --- | +| A message body that cannot be read after three tries, or that RavenDB does not hold at all. The whole message is skipped, because a message with no body is worse than none | Fault | Yes, once the cause is fixed | +| An error from the SQL target | Fault | Yes, once the cause is fixed | +| A `UniqueMessageId` that is not a GUID. It is the key, the ServicePulse URL and the retry key, so it is never regenerated | Fault | No | +| A failed message with no processing attempts, so nothing to fill its required failure time, failing endpoint and exception from | Fault | No | +| A subscription whose message type or address is over 200 characters, the SQL key limit | Fault | No | +| A row missing a value SQL requires, such as a known endpoint with no name | Fault | No | +| A blank group comment, which SQL never stores | Harmless | Not needed | +| An archived or resolved message, or event log item, already past its retention period | Harmless | Not needed | + +- **Any fault skip makes the category [Failed](#failed-categories)**, even one. Harmless skips leave it Done. +- **Endpoint settings for an endpoint ServiceControl does not know are copied, not skipped.** ServiceControl removes them shortly after it opens, by the same rule it always applies, so the migration does not repeat that rule. +- **Rows outside a window are not skips.** They are simply not part of the copy. +- **Rows RavenDB expires during the copy are not skips either.** A deleted row is never read, so it is an absence, not a loss. + - Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: archived or resolved failed messages, and event log items (`ExpirationManager.cs`). + - A failed message loses its expiry whenever it becomes unresolved or retry-issued again: when it fails again, is unarchived, or is retried. + - The exception: a message that failed again after being archived or resolved on version 6.18 or earlier kept its old expiry, and upgrading does not remove it. So a few unresolved messages can still expire during the copy. + - Both shrinking categories copy in the background, so the copy has hours to race the sweep. A window shorter than the retention period keeps the copy clear of it, by the difference between the two. + +### Startup checks + +In this order, cheapest first: + +1. Source and target are the supported pair, RavenDB to SQL. Answered from settings alone. A RavenDB instance with migration on is refused here, because RavenDB is only ever the source. +2. The optional category windows are valid time spans of zero or more. +3. `ServiceControl/RetryHistoryDepth` is above zero. At zero or less, the first completed retry after the migration deletes the whole copied retry history, and no row count would ever show it. +4. The SQL schema is current. This also refuses a database upgraded without `--setup`. +5. Until a required category has started copying: the SQL database holds no ServiceControl data. `--setup` writes none, but even one plain start of ServiceControl writes rows. Copying into a database that already served on SQL would mix the two. +6. Message body storage is writable. The probe body it writes is deleted again. +7. The SQL target opens, and no optional category's window has changed since that category started. +8. The RavenDB source opens: + - an embedded server starts; + - on an external server, the client certificate is in date and an `https://` address has one, checked before the first request; + - an external server is at least the RavenDB client's version; + - both databases load, within 5 minutes each on an embedded source. + + If every required category is already Done or Abandoned, a source that will not open does not stop the start. ServiceControl opens, and the optional categories wait for the source. + +Checks stop at the first one that refuses, and the refusal names it. Once open, the source can add checks of its own. The RavenDB source adds none. + +```mermaid +flowchart LR + A["Checks pass?"] -->|No| X["Host stays closed
and says why"] + A -->|Yes| B["Copy every required
category"] + B --> C["All Done
or Abandoned?"] + C -->|No| X + C -->|Yes| D["ServiceControl opens"] + D --> E["Copy optional
categories"] +``` + +- **The host also stays closed** if a second instance is writing the same checkpoints. +- **When ServiceControl opens**, the target is marked as opened, which ends the [free abort](#going-back-to-ravendb). +- **Once it has opened, a RavenDB outage does not stop it.** Only the background copy needs the source then, so ServiceControl opens anyway, and the optional categories [wait for the source](#failed-categories) to come back. + +### Reading from RavenDB + +- **The source opens RavenDB read-only**: connect, check the version, stop. It never runs RavenDB's database setup, and it refuses any write request. +- **Both source databases are opened through one `IDocumentStore`**, which is why they must be on the same server or cluster. +- **Upgrade on RavenDB first**, so the build doing the copy is the one that last ran against the source. Nothing checks this. A RavenDB start rewrites no stored document, so the copy reads exactly what this build's RavenDB persister would read. The only version check compares the RavenDB server to the RavenDB client, and only for an external source. +- **Distance to the source sets the pace.** The copier already holds each document from the stream, so each body costs one round trip rather than two. But it is one per message, and they are not batched. Egress out of RavenDB Cloud is billed to the customer. + +### Writing to SQL + +- **A failed message is written whole**, with its status unchanged. +- **`UniqueMessageId` keeps its value** but changes type, from a string in RavenDB to a `uniqueidentifier` column in SQL. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. +- **`StatusChangedAt` is rebuilt.** + - For archived and resolved messages it comes from `@expires`, the only place RavenDB records it. + - For unresolved and retry-issued messages it is the newest attempt's time, because the column is `NOT NULL` and cannot be left empty. That is harmless, because the retention sweep only considers archived and resolved rows. +- **Deciding whether a row is past retention needs two retention periods.** The source's turns `@expires` back into the status-change time. The target's current one decides whether that time is past the cutoff. +- **Bodies go through `IBodyStoragePersistence`**, which owns compression and the choice of filesystem, Azure Blob or S3. The copier applies the inline threshold itself, because that threshold lives on the ingestion path. It defaults to 102,400 bytes and is set by `ServiceControl/MaxBodySizeToStore`. +- **Throughput rows are written directly, not through the collector, and each day's count is set, not added.** + - Throughput is required, so it copies while ServiceControl is closed, before any collector has written to SQL. + - Setting makes the category safe to resume after a crash, where adding would double-count. The collector's own path does add (`LicensingDataStore.cs:198`). + - Copying the rows stops the audit and broker collectors gathering the same days again, because `LastCollectedDate` is worked out from the newest throughput row (`LicensingDataStore.cs:45`). + - The checkpoint stops a second pass overwriting days the collectors have written since. + +### Batch size and throttling + +- **Batch size comes from the database.** SQL Server divides its parameter budget by the column count, PostgreSQL uses a flat 50 rows. +- **Background batches have a pause between them**, 100 ms by default, set by `ServiceControl/Migration/ThrottlePauseMilliseconds`. The first batch of each category is not paused. +- **Raise the pause if the copy competes with production.** Turning migration off is not a remedy: the host refuses to start until every category is Done or Abandoned. +- **The required copy is never paused**, because ServiceControl is closed and nothing competes with it. + +### Progress and checkpoints + +The checkpoint is one row per category, kept in the SQL database and created by `--setup`. It holds the category's *state*, its cursor, its copied, skipped and already-present counts, how many of the already-present rows were merges, the count per skip reason, how many rows SQL held for the category when it settled, timings, the last error, the window it copies with, and a version number to catch a second writer. + +```mermaid +sequenceDiagram + participant E as Engine + participant S as RavenDB + participant T as SQL + loop each batch + E->>S: Read rows after the cursor, and their bodies + E->>T: Write rows + new counts + new cursor + Note over T: One transaction + E->>E: Stop as Failed if too much of this run was skipped + end + E->>T: Settle as Done or Failed +``` + +- **Progress never gets ahead of the data.** Rows and cursor commit together, so a restart never skips rows that were not written. +- **A crash costs only the batch in flight.** The next start carries on from the committed cursor. +- **A batch read twice is counted once.** Rows already in SQL come back as *already present*, never as copies or skips, so every category is safe to run twice. Unreadable bodies stay off the checkpoint until the write commits. +- **Every skip has a reason, or the save is refused.** The per-reason counts must add up exactly to the skipped count, and the target's counts must match what it committed. +- **The stored counts always describe the rows in SQL.** The target adds its own outcome and saves the result beside the rows, so nothing is saved later or separately. +- **A second writer is caught, not merged.** Each save carries the version it read, so two hosts, or a command and a host, pointed at one database cannot interleave. +- **A copy stopped between two categories still shows the ones it never reached.** Before copying anything, a start saves a *not started* row for each category it is about to copy, so every gate sees them as unfinished. +- **Only the copier and the two commands write the checkpoint.** The status and verify commands only read it. + +### Failed categories + +A category is *Failed* when something went wrong. Everything already copied stays committed. Nothing is retried in the background and nothing waits for a timer: the category stays Failed, with the reason written on it, until you run a [command](#retry-and-abandon). + +**What makes a category Failed:** + +- **Any fault skip by the end of the category**, even one. +- **Too many fault skips in this run, which stops it early.** More than 5% of the rows processed so far *and* more than 100 rows (`ServiceControl/Migration/HaltThresholdPercent`, `ServiceControl/Migration/HaltThresholdMinimum`). Needing both means a tiny category does not stop on one bad row, and a huge one does not stop on its 101st. Under the threshold a category runs to the end, so you get the whole list of bad rows in one pass. +- **The category's own counts not balancing:** every row read must be copied, skipped or already present. +- **In a required category only, an exception or a stall of 30 minutes with nothing committed**, such as the body store unreachable, a certificate expired, the source or target down, or the disk full. The error and the cursor are recorded. + +**What does not make it Failed:** + +- **Harmless skips.** +- **A shutdown or a crash.** The category stays Copying and the next start carries on from the cursor. +- **An exception or a stall in an optional category.** The error is written on the category, shown in status, and the next start carries on from the cursor. A RavenDB outage after opening is handled the same way. So a network blip never forces anything to be abandoned. If the error keeps coming back, retry or abandon the category. + +**What a Failed category costs:** + +- **The other categories still run.** Every required category gets its go in one start, so one fix-and-retry covers them all. A category that must follow a Failed one waits. +- **A Failed required category** keeps ServiceControl closed until it is retried to Done or abandoned. That is deliberate: opening the host is the point of no return. +- **A Failed optional category** leaves the instance running with that slice missing, and stops the migration being finished. + +**What the checks cannot see, and accept:** + +- **Nothing counts RavenDB during the copy.** So a read that ends early, or skips a range, still balances, and the category ends Done. `--migration-verify` finds it afterwards, by counting RavenDB again and comparing that with the rows the copy read (see [the read-only commands](#the-dry-run)). +- **Verify counts with the same reader the copy used.** So it finds rows one run missed, but not a reader that always misses the same rows, or a source pointed at the wrong database. Tests on each reader cover those. +- **A write that silently stores fewer rows still ends Done.** Already present is what is left over after copied and skipped, so the counts balance. The counts check does catch rows dropped while a batch is prepared. Verify prints how many already-present rows were merges, so on a category copied once, already-present rows that merges do not explain point to it. +- **The threshold judges this run only.** A bad start can stop a category whose overall rate would have been fine, and the source must not read the rows most likely to be skipped first. + +### Retry and abandon + +Stop ServiceControl, run one of these, then start it again. In a container, run the command as a one-off `docker run --rm ` against the same database, the same way as `--migration-source-report`. + +| Command | Works on | What it does | +| --- | --- | --- | +| `--migration-retry ` | A Failed category, or an optional one still copying with an error | The next start copies it again from the beginning | +| `--migration-abandon ` | A Failed or started required category, or any optional one | Gives up on what it has not copied, and on any category whose rows hang off it. What they copied stays in SQL. Final | + +**To retry:** + +1. Read the reason in the refusal, the log or `--migration-status`. It names the skipped rows by reason, or the error. +2. Fix the cause. It is usually outside the migration: the body store unreachable, a certificate expired, the source or target down, or the disk full. +3. Run `--migration-retry `, then start ServiceControl. + +**What a retry does:** + +- **It re-reads the whole category from the start**, every time, which recovers from anything. Rows already in SQL are left alone, so only the rows that failed can change. Every message body is read again, so a retry takes about as long as the first copy. +- **Historic retry operations and pending integration events** get new keys from SQL on every insert, so a re-read could not recognise rows already copied. A retry of either first deletes what that category copied, in the same save as the reset. That is safe because both are required, so ServiceControl has not opened and the target started empty. +- **The event log cannot be retried.** It also gets new keys, but it copies after ServiceControl opens, when copied rows cannot be told apart from live ones. It can only be abandoned. + +**To abandon:** + +- **A required category can be abandoned once it is Failed or has started copying.** One that never ran cannot, so nothing required is given up blind. If RavenDB is gone before it starts, point back or start over: ServiceControl has not opened, so nothing being served is lost. +- **An optional category can be abandoned at any time.** That is how to stop a big optional copy short, or get out when RavenDB is gone. +- **Abandoning cascades to categories whose rows hang off it**, and the command names each: throughput history (it points at licensing endpoint records) and endpoint settings (it belongs to known endpoints). Group comments only waits for unresolved failed messages, so it is not abandoned with them. +- **Some skips no retry can fix**: a non-GUID id, a failed message with no processing attempts, a key over 200 characters, a missing required value. The refusal and the dry run say which, so abandon after the first failure rather than retrying. + +> [!IMPORTANT] +> Abandon is final. Nothing goes back for an abandoned category, so it is right when the data is not worth the outage, and wrong if picked by accident. + +**Nothing happens on a restart that you did not ask for.** A container or service restart policy restarts a refused host without anyone looking, and gets the same refusal each time, without reading anything. That is loud and loses nothing. + +### Ingestion workers + +- **An `--error-ingestion-only` worker never runs the copy.** On RavenDB it is refused, as before, because only SQL can scale out. +- **On SQL it starts only when every required category is Done or Abandoned**, whether or not its own migration setting is on. Otherwise it refuses and names each required category still outstanding. Optional categories still copying do not hold it back. +- **That check runs even on a worker someone forgot to flag**, because ingesting into a part-copied database would make going back a loss. +- **`--import-failed-errors` on SQL runs the same check**, because it writes failed messages too. +- **A worker or import that gets past the check marks the target as opened**, whether or not its own migration setting is on. Once it has written, going back to RavenDB would lose that data, so it ends the free abort just as the main host opening does. + +> [!IMPORTANT] +> **Start workers only after the main host has opened.** One started at the same moment as the first migration start can slip in before the copy has written its first checkpoint, and nothing stops it. + +### Going back to RavenDB + +**While ServiceControl is closed**, only the copier has written to SQL and the migration has written nothing to RavenDB, which is still the source of truth. This is the *free abort*. RavenDB's own expiration still runs, though, unless you disabled it. + +- **To go back:** set `ServiceControl/Migration/Enabled=false`, point `ServiceControl/PersistenceType` back at RavenDB, and start. You lose the copy, not your data. +- **To try again later:** start from a new, empty SQL database (`--setup`). + +> [!IMPORTANT] +> Never reuse the old copy. It would skip every category already finished and miss anything RavenDB received since, and nothing can detect the reuse, because it looks exactly like a normal resume. +> +> **Once ServiceControl opens**, new failed messages go to SQL and there is no way back. You can only finish the migration, or abandon what is left. + +**Turning migration off before every category is Done or Abandoned is refused**, not a quiet exit. That is what stops a migration ending by accident. + +- **Every SQL start checks the checkpoint table, with migration on or off.** It costs nothing when nothing is outstanding, and a RavenDB instance never reaches it. +- **With nothing outstanding, nothing changes.** A SQL instance starts exactly as before, without opening RavenDB. +- **An optional category that never started and whose window is `0` is not outstanding.** It was turned off before it began. +- **With anything outstanding, the host refuses.** It names each category with its state, counts and last error, and the ways out: + - turn migration back on to finish, after `--migration-retry` for any Failed category; + - `--migration-abandon` what you are giving up on; + - point back at RavenDB, which is free only if ServiceControl never opened on SQL (the refusal says which); + - start over against a new, empty SQL database. + +**Keep RavenDB until every category is Done or Abandoned.** Before ServiceControl has opened, nothing can start without it. After, ServiceControl still opens, and the optional categories wait for RavenDB to come back, or can be abandoned. + +### The dry run + +The dry run never writes to RavenDB, and can run again later against whatever is outstanding. It reports: + +- Whether the source is embedded or external, and which server. +- Both RavenDB database names, and the setting each came from. +- Rows and body volume per category, counting only what is inside each window. +- **A range for how long ServiceControl will be closed. The range is a floor, not a promise:** it times counting the rows and sizing the bodies, not reading every row and body or writing to SQL, so the required copy can take longer. +- **A duration for each optional category**, timed from a sample of its rows and bodies plus the pause between batches. ServiceControl is open while these copy. +- The result of the same startup checks a real start runs, so a missing setting surfaces before anyone books an outage. +- How many failed error imports there are, with advice to run `--import-failed-errors` on RavenDB first. They are copied either way, but ones imported first arrive as ordinary failed messages. +- Rows that would be skipped or merged, by reason, and which of those no retry can fix. If a required category holds any, it will end Failed, so plan to abandon it after its first failure. + +It says nothing about load on the source. + +**When each read-only command can run:** + +| Command | What it does | Opens RavenDB? | +| --- | --- | --- | +| `--migration-source-report` | Source facts and a document count per collection | Yes | +| `--migration-dry-run` | The report above | Yes | +| `--migration-verify` | Counts RavenDB again and compares each category with the rows the copy read, with skips and merges broken out. Exits 0 only when every category is Done or Abandoned, so a script can ask whether the migration is finished. Rows the copy never read are shown, not failed | Yes | +| `--migration-status` | Each category's state, progress, last error and the commands it can take | No, so it runs any time | + +- **On an embedded source, stop the ServiceControl service before any command that opens RavenDB.** A second RavenDB process cannot use a data directory the first one holds, so the dry run is part of the outage. +- **Verify reads every selected category in full**, though not the message bodies. On a big instance that takes a while, and on an embedded source that time is part of the outage too. +- **On an external source** they all run against a live instance. +- **In a container**, run each as a one-off `docker run` of the same image against an external RavenDB server. + +**What verify compares:** + +- **RavenDB against the rows the copy read, never against SQL as it is now.** Once ServiceControl opens, normal use changes most categories' SQL rows within minutes: ingestion, heartbeats, retries, archiving, the licensing collectors, ServiceControl's own custom checks and the retention sweep. A SQL count then measures that use, not the copy. +- **RavenDB holding more rows than the copy read means rows were never read.** Nothing writes to RavenDB once the copy starts, so it can only shrink. Verify flags these rows. They are not in SQL, so keep RavenDB. The same flag shows if something other than this migration wrote to RavenDB since, such as an old instance still running on it. +- **RavenDB holding fewer is shown, not judged.** RavenDB's expiry keeps deleting archived and resolved messages, event log items and a few unresolved messages from 6.18 or earlier, after the copy has read them. +- **The same expiry can hide rows that were never read**, because a deleted row and a missed row cancel out. A window shorter than the retention period keeps a windowed category clear of this, by the difference between the two. +- **A windowed category is counted inside the window it started with**, on RavenDB only. +- **Verify also prints how many rows SQL held for each category when it settled.** Before ServiceControl opens nothing else writes, so for a required category that is exactly what the copy left in SQL. For the two optional categories it includes what ServiceControl wrote while they copied. + +### Monitoring and settings + +| Setting | Default | What it does | +| --- | --- | --- | +| `ServiceControl/Migration/Enabled` | `false` | Turns the migration on | +| `ServiceControl/Migration/EventLogWindow` | `ServiceControl/EventRetentionPeriod` | How far back the event log copy goes. `0` turns it off | +| `ServiceControl/Migration/ArchivedAndResolvedFailedMessagesWindow` | `ServiceControl/ErrorRetentionPeriod` | How far back the archived and resolved copy goes. `0` turns it off | +| `ServiceControl/Migration/ThrottlePauseMilliseconds` | `100` | Pause between background batches | +| `ServiceControl/Migration/HaltThresholdPercent` | `5` | Percentage part of the early-stop threshold | +| `ServiceControl/Migration/HaltThresholdMinimum` | `100` | Floor part of the early-stop threshold | + +- **The source needs no new settings.** It reads the instance's existing RavenDB settings. +- **Settings are read fresh at every start**, except that an optional category's window is fixed once that category starts. +- **Progress shows in the ServicePulse activity feed, the log and `--migration-status`.** The activity feed records when the background copy starts and finishes, when a category fails, and when the source cannot be reached. A start with migration still on after every category is Done or Abandoned logs a warning to turn it off. There is no migration custom check, because custom checks are being removed from ServiceControl. +- **Decisions are two commands, run with ServiceControl stopped:** `--migration-retry` and `--migration-abandon`. There is no HTTP API, pause, resume or abort, and no way to add a category to a running instance. +- **Stopping a copy takes a restart.** + +## Glossary + +- **Category:** one kind of data copied as a unit, such as known endpoints or the event log. +- **Required category:** one ServiceControl needs the moment it opens, so it copies while ServiceControl is closed. +- **Optional category:** one copied in the background after ServiceControl opens. It can be shortened or turned off before it starts. +- **Window:** how far back an optional category copies. +- **Closed period:** the time ServiceControl is not serving traffic because the required copy is running. +- **Checkpoint:** the row in SQL recording one category's progress, saved in the same transaction as the rows it describes. +- **Cursor:** the position in RavenDB the copy has reached in a category. +- **State:** one of Copying, Done, Failed or Abandoned. +- **Copying:** not finished yet, including waiting to start, or stopped on an error that the next start will try again. +- **Done:** every row is in SQL, or was left out for a harmless reason, and the counts balance. +- **Failed:** something went wrong. Stays Failed until you run `--migration-retry` or `--migration-abandon`. +- **Abandoned:** you gave up on what a category had not copied, or on one it depends on. Final. +- **Already present:** a row the copier found in SQL already, so it left it alone. +- **Fault skip:** a row lost because something went wrong. +- **Harmless skip:** a row left behind because SQL would have removed it anyway. +- **Free abort:** going back to RavenDB at no cost, possible only while ServiceControl is still closed. +- **Dry run:** a read-only rehearsal that reports what would move, what would be skipped and how long the outage would be. + +## Further reading + +- [The instructions](ravendb-to-sql-migration-instructions.md): how an operator runs the migration. +- [How the migration is put together](ravendb-to-sql-migration-system-design.md): which class does what, and what is built so far. diff --git a/docs/migration/ravendb-to-sql-migration-system-design.md b/docs/migration/ravendb-to-sql-migration-system-design.md new file mode 100644 index 0000000000..2e8d087297 --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-system-design.md @@ -0,0 +1,289 @@ +# Migration Engine: System Architecture Design + +> [!NOTE] +> This build does not carry the whole migration yet. This page describes it as it will be when it ships. + +[The overview](ravendb-to-sql-migration-overview.md) covers behaviour and guarantees. [The instructions](ravendb-to-sql-migration-instructions.md) cover the operator procedure. This page maps both onto types, call order and assembly boundaries. + +> [!IMPORTANT] +> Startup check order and category order are load-bearing. Read the required copy and category sections before reordering checks or adding a category. + +## Overview + +```mermaid +flowchart LR + subgraph host["ServiceControl host"] + req["RequiredCopyBeforeTheHostOpens
then MigrationStartup"] + opt["OptionalCategoryCopier"] + guard["EndOfMigrationGuard"] + cmd["Migration commands"] + end + subgraph neutral["ServiceControl.Persistence"] + eng["MigrationEngine"] + end + subgraph raven["ServiceControl.Persistence.RavenDB"] + src["RavenMigrationSource
IMigrationCategoryReader per category"] + end + subgraph ef["ServiceControl.Persistence.EFCore"] + tgt["EFCoreMigrationTarget
IMigrationCategoryWriter per category"] + cp["EFMigrationCheckpointStore"] + end + req --> eng + opt --> eng + guard --> cp + eng --> src + eng --> tgt + eng --> cp + cmd --> cp + cmd --> src + cmd --> tgt + src --> rdb[("RavenDB
read only")] + tgt --> sql[("SQL Server or PostgreSQL")] + cp --> sql + tgt --> bod[("Body store")] +``` + +`MigrationEngine` is store-agnostic. It drives `IMigrationSource` and `IMigrationTarget` per category and treats the cursor as an opaque string that it hands from source to target. Progress is persisted per category as a checkpoint row in the target database, committed in the same transaction as the rows it describes. Every command except `--migration-source-report` reads the checkpoint table. + +## Assemblies + +Dependencies point inward: each persister knows only its own store, the neutral assembly knows neither, and the host is the only composition root that names both. + +| Assembly | Types | Boundary rationale | +| --- | --- | --- | +| `ServiceControl.Persistence` | `MigrationEngine`, `MigrationBodyLoader`. Contracts: `IMigrationSource`, `IMigrationTarget`, `IMigrationCheckpointStore`, `IMigrationStartupCheck`, `IMigrationTargetReadiness`, `IMigrationSourceFactory`. Data: `MigrationCategory`, `MigrationCategoryKind`, `MigrationCategoryIds`, `MigrationCategoryRegistry`, `MigrationBatch`, `MigrationRow`, `MigrationBody`, `MigrationCheckpoint`, `MigrationWriteResult`, `MigrationSkipReason`, `MigrationSourceDescription`, `MigrationSourceFact`. Rules: `MigrationCategoryStateExtensions`, `MigrationSkipReasonExtensions`, `MigrationCheckpointRules`, `HaltThreshold`. Options: `MigrationEngineOptions`, `MigrationSettings`. `MigrationCheckpointConflictException`. Read-only views: `MigrationStatusView`, `MigrationVerification`, `MigrationDryRunArithmetic` | Referenced by both persisters and the host, references neither persister, so the engine is unit-tested against in-memory fakes in `ServiceControl.UnitTests/Migration` | +| `ServiceControl.Persistence.RavenDB` | `RavenMigrationSource`, `RavenReadOnlySourceLifecycle`, `RavenDocumentStream`, `IMigrationCategoryReader`, `IWindowedMigrationCategoryReader`, one reader per category | Owns document id prefixes, sessions and the embedded server | +| `ServiceControl.Persistence.EFCore` | `EFCoreMigrationTarget`, `EFCoreMigrationTargetReadiness`, `IMigrationCategoryWriter`, `IStoreKeyedCategoryWriter`, one writer per category, `PreparedBatch`, `IMigrationSqlDialect`, `MigrationInsert`, `StoreKeyedInsert`, `EFMigrationCheckpointStore`, `MigrationCheckpointExtensions`, `EFCoreMigrationRowAssessor`, and the three target readiness checks | Owns tables, column widths and provider parameter limits | +| `ServiceControl` (host) | `RequiredCopyBeforeTheHostOpens`, `RecordHostOpenedOnTarget`, `MigrationStartup` with nested `ClosedWindowProgress`, `MigrationStartupCheckRunner`, the four host checks, `OptionalCategoryCopier`, `RequiredStartSourceOutage`, `MigrationComponent`, `EndOfMigrationGuard`, `FinishedCopyBeforeAnIngestionNodeOpens`, `MigrationProgressReporter`, the six migration commands with `MigrationOperatorCommands` and `MigrationSourceDescriptionPrinter`, `PersistenceFactory.CreateMigrationSource` and `OpenMigrationSource` | Pair support is the one question neither persister can answer, so it lives here | + +Supporting assemblies: + +- `SqlServerMigrationSqlDialect` and `PostgreSqlMigrationSqlDialect` implement `IMigrationSqlDialect` and are the only provider-specific migration code. SQL Server reads key-column collations in `Open` and inserts with `MERGE ... WITH (HOLDLOCK)`, de-duplicating the batch with `ROW_NUMBER() OVER (PARTITION BY COLLATE )`. PostgreSQL uses `INSERT ... ON CONFLICT DO NOTHING RETURNING`. Both implement `SetLicensingEndpointThroughput`. Each provider assembly carries its own `AddMigrationCheckpoints` EF migration and model snapshot, so a checkpoint schema change needs a migration in both. +- `ServiceControl.DomainEvents` holds `MigrationStarted`, `MigrationCategoryHalted`, `MigrationFinished` and `MigrationSourceUnreachable`. + +The RavenDB persister registers no `IMigrationTarget`, `IMigrationTargetReadiness` or `IMigrationCheckpointStore`, so the guards below resolve `null` on RavenDB and return. `MigrationPairIsSupportedCheck` is what rejects RavenDB as a target. + +## What runs on every SQL instance + +These run whether or not `Migration/Enabled` is set. Each one closes a path to data loss or to a migration ending unnoticed. + +- **Registrations.** `BasePersistence.RegisterDataStores`, reached through each provider's `AddPersistence`, registers `IMigrationCheckpointStore` as `EFMigrationCheckpointStore`, `IMigrationTargetReadiness` as `EFCoreMigrationTargetReadiness` and `IMigrationTarget` as `EFCoreMigrationTarget`, all singletons. +- **`EndOfMigrationGuard`.** With `Migration/Enabled` off, `RunCommand` registers a hosted service in place of `RequiredCopyBeforeTheHostOpens` that runs `MigrationStartup.RunEndOfMigrationGuard` in `StartingAsync`, before any other hosted service and before Kestrel binds. It reads every checkpoint row and refuses if any category `IsOutstanding`. `IMigrationTargetReadiness.HasHostOpened` only selects the refusal text: whether going back to RavenDB is still free. With no checkpoint store registered (RavenDB) it returns immediately. +- **`FinishedCopyBeforeAnIngestionNodeOpens`.** Registered by `ErrorIngestionOnlyCommand.BuildHost` on every ingestion-only host and by `ImportFailedErrorsCommand.BuildHost` on every SQL import, flag on or off. In `StartingAsync` it reads every checkpoint row, drops the rows the registry classifies as optional, and refuses unless every remaining row `IsFinished`. An id unknown to this build counts as required. The worker never copies: a single host owns the copy, because a second copier would race it on the checkpoint version. +- **`RecordHostOpenedOnTarget` on workers.** Every host that passes the ingestion gate also registers `RecordHostOpenedOnTarget`, flag on or off, and stamps `Migration/HostOpenedOnTarget` when checkpoint rows exist. Ingestion ends the free abort, so the end-of-migration guard must never offer a free abort after a worker has written. + +The ingestion gate sees categories the copy never reached because `MigrationStartup.RunRequiredCategories` upserts a `NotStarted` row for every category without one before copying the first. `MigrationEngine.RunCategories` does the same for the background copy. Without those rows, a copy interrupted between two categories leaves only terminal rows and the gate passes. The gate cannot see a copy that has not written its first row. + +> [!IMPORTANT] +> Workers must start only after the main host has opened. A worker started together with the first migration start can pass the gate before the first checkpoint row exists, and nothing enforces the order. + +## The required copy + +```mermaid +sequenceDiagram + participant Run as RunCommand + participant Copy as RequiredCopyBeforeTheHostOpens + participant Start as MigrationStartup + participant Checks as MigrationStartupCheckRunner + participant Source as RavenMigrationSource + participant Target as EFCoreMigrationTarget + participant Engine as MigrationEngine + + Run->>Run: registers Copy as a hosted service, then hostBuilder.Build() + Run->>Copy: StartingAsync, before any other hosted service starts + Copy->>Start: RunRequiredCopy(app.Services, settings) + Start->>Checks: MigrationPairIsSupportedCheck + Start->>Start: PersistenceFactory.CreateMigrationSource, not connected + Start->>Checks: RunChecksAndOpenTarget + Checks->>Target: Open + Start->>Start: IMigrationCheckpointStore.ReadAll + Start->>Source: Open + Start->>Checks: source.ContributedChecks(), empty for RavenDB + Start->>Engine: RunRequiredCategories under ClosedWindowProgress + Engine-->>Start: one checkpoint per category + Start->>Start: ReportWhatTheCopyLeftBehind + Start->>Start: RefuseIfAnyCategoryDidNotComplete + Start-->>Copy: returns, or throws and the host never opens + Note over Run: RecordHostOpenedOnTarget stamps the target in StartedAsync +``` + +Each position in the sequence is deliberate: + +1. **Inside host start.** `RunCommand` registers `RequiredCopyBeforeTheHostOpens` only when `Migration/Enabled` is true, before `hostBuilder.Build()`. `app.RunAsync` drives its `StartingAsync` ahead of every other hosted service, so the retention sweeper, heartbeat settings sync, throughput collectors and the API cannot run while a required category is unfinished. Running inside `RunAsync` also keeps the Windows Service Control Manager's 30-second start timeout satisfied. +2. **`MigrationPairIsSupportedCheck` first.** It needs no source, target or options. It resolves `PersistenceType` through `PersistenceManifestLibrary` and checks it against `PersistenceFactory.SqlPersistenceNames`; the source type is fixed at `PersistenceFactory.MigrationSourcePersistenceType`. RavenDB with the flag on is rejected here. +3. **`PersistenceFactory.CreateMigrationSource`** resolves the source persistence and hard-casts its configuration to `IMigrationSourceFactory`, implemented only by `RavenPersistenceConfiguration`. A non-implementing persister fails with `InvalidCastException`. No I/O happens yet. +4. **`MigrationStartup.RunChecksAndOpenTarget`**, the target half, in order: + - `OptionalCategoryWindowsAreValidCheck` runs `MigrationEngineOptions.FromSettings`, refusing a window that is not a non-negative `TimeSpan` and naming the key. A zero window removes the category from `SelectedOptionalCategoryIds`. + - `RetryHistoryDepthIsSafeCheck` is host-owned because it needs only `Settings.RetryHistoryDepth`, which keeps that value off `PersistenceSettings` and out of `/api/configuration`. + - `EFCoreMigrationTargetReadiness.ContributedChecks`: `SchemaIsCurrentCheck`, `TargetHoldsNoServiceControlDataCheck`, `BodyStorageIsWritableCheck`. The body probe is deleted after writing, so it does not skew body counts. + - `IMigrationTarget.Open`, where the SQL Server dialect loads key-column collations. + - `OptionalCategoryWindowsAreUnchangedCheck`, last because it reads checkpoint rows. It refuses a started, unfinished optional category whose configured window differs from `StartedWindowSeconds`. + + `MigrationStartupCheckRunner` short-circuits on the first refusal. +5. **`TargetHoldsNoServiceControlDataCheck`** passes immediately if a checkpoint row exists for any required category. Otherwise it refuses if any mapped table other than `MigrationCheckpoints` has a row, `Settings` included, naming tables by `GetTableName()` (`endpoint_settings`, `settings` on PostgreSQL). +6. **Source open, last.** It is the only step that can spawn a process. `RunRequiredCopy` reads the checkpoint rows and builds the engine first. `RavenReadOnlySourceLifecycle.Open` then: + - starts the embedded server, if configured; + - connects: on an external server it rejects an expired or not-yet-valid client certificate and an `https://` URL without one, then attaches `RefuseWrite` to the request pipeline before `Initialize`, so the version request is covered; + - checks an external server's version against the client; + - waits for both databases, with a 5-minute budget per database on an embedded server. The budget is only checked when RavenDB raises its own timeout, so the wait can exceed it. + + > [!IMPORTANT] + > Readers must stream by id prefix, load by id, or query a static index. `RefuseWrite` blocks writes but passes query POSTs (`/queries`, `/multi_get`, `/streams/queries`), and a dynamic query creates an `Auto/` index. Only the embedded server sets `DisableAutoIndexCreation`. +7. **Source outage tolerance.** A failed source open is tolerated only when every required category already `IsFinished`. The host logs it, `MigrationStartup.RecordSourceOutage` writes the exception (inner exception when present) as `LastError` on each selected optional category still copying, creating `NotStarted` rows where absent, the copy is skipped and the host opens. `OptionalCategoryCopier` then stays idle for that start. In every other case the host refuses and writes nothing; if a required category never started, the refusal says the only exits are pointing back at RavenDB or starting over. +8. **`source.ContributedChecks()`** runs once the source is open, because such a check needs a session. `RavenMigrationSource` returns none, and a refusal here is never tolerated. +9. **`MigrationStartup.RunRequiredCategories`** upserts `NotStarted` rows, then runs each required category under its own linked `CancellationTokenSource`, awaited with `WaitAsync` on that token. +10. **`ClosedWindowProgress`** polls the checkpoint store every 30 seconds and logs progress. When the running category has committed nothing for 30 minutes, measured from the later of `LastProgressAt` and its run start, it cancels that category's token. `WaitAsync` returns even if the call underneath ignores cancellation, the category settles `Halted` with the stall in `LastError`, and the next category runs. The abandoned call keeps running until the host refuses and the process exits. A host shutdown leaves the row `InProgress`. +11. **`CopyOrExplainWhyItStopped`** maps `MigrationCheckpointConflictException` to a message about a second instance writing the same checkpoints. +12. **`ReportWhatTheCopyLeftBehind`** logs skips per category. It offers `--migration-retry` only for an `IsFailed` row, and reports harmless skips on a `Complete` row as harmless. +13. **`RefuseIfAnyCategoryDidNotComplete`** keeps the host closed. One message lists every Failed required category with stored state, counts, skips by reason (permanent reasons flagged as not retryable through `IsPermanent`), `LastError` and both commands. A Copying required category gets a line without a command. Rollback advice comes last. +14. **`RecordHostOpenedOnTarget`** is registered beside `RequiredCopyBeforeTheHostOpens` with the flag on. In `StartedAsync` it writes `Migration/HostOpenedOnTarget` the first time a host opens on a database that holds checkpoint rows, so the marker means "a host opened since the copy began", not "a host ran here once". The marker ends the free abort. + +## The background copy + +`OptionalCategoryCopier` is a `BackgroundService` registered by `MigrationComponent` only with the flag on and only on the main instance; `--error-ingestion-only` passes its own component list. + +- Every 20 seconds it runs the optional categories in registry order, `EventLog` then `ArchivedAndResolvedFailedMessages`, skipping rows that are `IsFinished` or `IsFailed`. +- It opens its own source through `PersistenceFactory.OpenMigrationSource`, only when a category is still copying. On failure it calls `RecordSourceOutage` and idles for the rest of the start. It also idles when the required start already recorded an outage, which it learns from the in-process `RequiredStartSourceOutage` singleton (`Recorded`, `Error`) that `RunRequiredCopy` sets where it calls `RecordSourceOutage`. The checkpoint rows cannot carry this signal, because an optional row's `LastError` can survive from an earlier start. +- An exception leaves the category `InProgress` with `LastError`, untouched until the next start, so a transient failure never forces an abandon. + +Windowing: `MigrationEngineOptions.WindowFor` supplies the window and the reader applies it through `IWindowedMigrationCategoryReader`. The engine writes `StartedWindowSeconds` once, on the first `InProgress` upsert. `CopiesFrom` is `StartedAt` minus the currently configured window, so window stability across restarts depends entirely on `OptionalCategoryWindowsAreUnchangedCheck`. Verify counts from `StartedAt` minus `StartedWindowSeconds` instead, which is the same window while the category copies, because nothing stops the setting changing once it has finished. + +> [!IMPORTANT] +> Do not remove or reorder `OptionalCategoryWindowsAreUnchangedCheck`. The stored `StartedWindowSeconds` column does not protect the window by itself. + +`Migration/ThrottlePauseMilliseconds` inserts a delay between optional-category batches, skipping the first batch of each category. Required categories are never throttled. + +## Copying a batch + +`MigrationEngine.RunCategoryAsync` is the per-category loop. Every store-specific step is delegated. + +| Step | Owner | Rationale | +| --- | --- | --- | +| Load the checkpoint | `EFMigrationCheckpointStore.Read` | The table lives in the target database | +| Short-circuit terminal or Failed rows | `MigrationCategoryStateExtensions.IsFinished`, `IsFailed` | Single definition shared by every caller; the source is not read | +| Ordering | `MigrationCategory.MustFollow`, from `MigrationCategoryRegistry.All` | Data, not code. A follower whose predecessor is not `IsFinished` is upserted `Blocked` with `LastError` naming it | +| Start or resume | Engine: `NotStarted`, `Blocked`, or any row with `LastError` moves to `InProgress`, cursor kept | `Halted` never resumes by itself; only `--migration-retry` resets it | +| Batch size | `EFCoreMigrationTarget.BatchSizeFor` via the category's `IMigrationCategoryWriter` | Writers use `RowsPerStatementFor` from the dialect's parameter budget; `StoreKeyedInsert`-only writers use a fixed 500 | +| Read after cursor | `IMigrationCategoryReader`, usually via `RavenDocumentStream.ByPrefix`; single-document readers load by id; licensing settings readers use `LicensingSettingsSource` | No-tracking session. A cursor whose document no longer exists is refused, because RavenDB would otherwise resume after the missing id and skip rows. The cursor advances on every document seen | +| Fetch bodies | `MigrationBodyLoader` over `IMigrationSource.ReadBody` | Rows with a null body only, 8 concurrent, `MigrationEngine.MaxBodyReadAttempts` (3) with `MigrationEngineOptions.BodyRetryBackoff` (200 ms). `IsDefect` exceptions such as `NotSupportedException` are not retried and reach the batch catch. A missing attachment is skipped as `BodyUnreadable` after one attempt, since ingestion writes an attachment for every failed message. Body skips stay off the checkpoint until the write commits | +| Map documents to rows | `IMigrationCategoryWriter.Prepare`, returning `PreparedBatch` | Only the writer knows `NOT NULL` columns, key caps and what the product deletes anyway. Rows are inserted later inside the target transaction. Body-carrying writers write external bodies here, before the transaction, only for messages the target does not hold, so a crash leaves orphan bodies, never dangling references | +| Insert and checkpoint | `EFCoreMigrationTarget.Write`: `AccountForEveryRow`, then in one transaction under the execution strategy `PreparedBatch.Insert`, `AlreadyPresentIn`, `MigrationCheckpoint.Extend`, `MigrationCheckpointExtensions.UpsertCheckpoint` | Most writers use the dialect's `InsertMissing` built on `MigrationInsert`. Store-keyed rows use `StoreKeyedInsert`, because `MigrationInsert` rejects store-generated keys. Throughput days use `SetLicensingEndpointThroughput`. Each attempt clears the change tracker. Skips and merges are logged after commit, and `Extend` adds the batch's `PreparedBatch.Merges` to `MergedCount`. See [progress and checkpoints](ravendb-to-sql-migration-overview.md#progress-and-checkpoints) | +| Post-commit reconciliation | Engine compares `Write`'s saved checkpoint with the counts it restated | Mismatch settles `Halted`, either kind | +| Early stop | `HaltThreshold.Exceeded` on per-run counters | Fault skips (reasons not `IsBenign`, plus body skips) must exceed both `Migration/HaltThresholdPercent` and `Migration/HaltThresholdMinimum`. Counters are not persisted, so the threshold is per run | +| Settle | `MigrationEngine.Settle`, after `IMigrationTarget.Count` at the end of the source | Saves the category's SQL row count on the row it settles, as `TargetCountAtSettle`. A halt saves without it, because the target may be what failed, and `--migration-abandon` reads it later. Logs before saving, because the checkpoint store shares the target database and a failed save would hide the cause | + +Body placement lives in `FailedMessagesWriter`, shared by `ArchivedAndResolvedFailedMessagesWriter`. `MessageBodyClassifier.Classify` decides placement and `FailedMessageRowMapper.SetBody` fills the columns: text up to `MaxBodySizeToStore` (100 KB default) inline; longer text inline prefix plus external copy; binary, invalid UTF-8 or NUL-containing text external only; empty bodies not stored. External writes go through `IBodyStoragePersistence.WriteBody`, which owns compression and the filesystem, Azure Blob or S3 backend, and complete before the transaction opens. + +Invariants enforced across class boundaries: + +- `MigrationCheckpoint.Extend` throws unless the per-`MigrationSkipReason` counts sum exactly to the skipped count. `EFCoreMigrationTarget.AccountForEveryRow` throws unless prepared rows plus skips equal the batch size, catching rows dropped during preparation. `AlreadyPresentIn` throws when a writer over-reports. Every row is copied, skipped or already present. +- `MigrationSkipReasonExtensions.IsBenign` is fixed: `PastRetention`, `BlankGroupComment`. Everything else is a fault, `Unknown` included. +- `PreparedBatch.Merges` reports rows whose key collides with another under the target's collation. `EndpointSettingsWriter` detects them with `IMigrationSqlDialect.KeyComparer` against the batch and the existing table, and `EFCoreMigrationTarget` logs one warning per merge naming both keys and the one kept. The dropped row counts as already present, and also in `MergedCount`, the part of already present that merges explain. Every writer whose key can fold two source rows into one reports them here, or `MergedCount` understates. +- `TargetCountAtSettle` is read with `IMigrationTarget.Count` just before the save that settles a category at the end of the source, or that abandons it, and is never updated afterwards. For a required category it is exactly what the copy left in SQL: the host is closed, `TargetHoldsNoServiceControlDataCheck` started the target empty, and a second copier would move the row's version so the settle save is refused. That is why the count needs no shared transaction with the save. For an optional category it also includes what ServiceControl wrote while the copy ran. + +## Category states + +`MigrationCategoryState` has seven stored values, surfaced as four operator states: + +| Operator state | Stored | Meaning | +| --- | --- | --- | +| Copying | `NotStarted`, `InProgress`, `Blocked` | Not terminal, including a follower waiting on its predecessor and an optional category stopped by an error the next start retries | +| Done | `Complete` | Every row copied or benignly skipped, counts reconciled | +| Failed | `Halted`, `CompleteWithErrors` | Stays Failed until `--migration-retry` or `--migration-abandon` | +| Abandoned | `Abandoned` | Terminal, set by the operator directly or by cascade | + +The required-copy refusal prints both forms for a Failed row (`EndpointSettings is Failed (Halted)`) and the stored state alone for a Copying row. Retry and abandon refusals print the operator state only. + +Early stops during a run: + +- Threshold exceeded or post-commit mismatch: `Halted`, either kind. +- Required category exception or 30-minute stall: `Halted`. +- Optional category exception: stays `InProgress` with `LastError`. + +Settlement at end of source, in order: + +1. Rows read this run differ from rows processed: `Halted`. +2. Any fault skip in the checkpoint: `CompleteWithErrors`. +3. Otherwise: `Complete`. + +`MigrationCategoryStateExtensions.IsFinished` (`Complete` or `Abandoned`) and `IsFailed` (`Halted` or `CompleteWithErrors`) are the single definitions, used by the engine, the required-copy refusal, the background copier, the end-of-migration guard, the ingestion gate, status, verify and both operator commands. The host opens when every required category `IsFinished`; the migration is finished when every category is. + +`MigrationCheckpointRules` holds the rules shared by the guard, the commands and status: + +- `IsOutstanding`: not finished, except an optional `NotStarted` row whose window is `0`. +- `CanBeRetried`: an `IsFailed` row, or an optional `NotStarted` or `InProgress` row with `LastError` (which covers `RecordSourceOutage` rows). Never `Blocked` or a missing row. `MigrationOperatorCommands.Retry` rejects `EventLog` before consulting it. +- `CanBeAbandoned`: a required row that `IsFailed` or is `InProgress` with copied plus skipped plus already-present above 0, treating an id unknown to the registry as required; an optional category in any state except `Complete` or `Abandoned`, including no row. + +Checkpoint schema: one row per category in `MigrationCheckpoints` (`migration_checkpoints` on PostgreSQL), primary key `CategoryId`. Columns: `State`, `Cursor`, `CopiedCount`, `SkippedCount`, `AlreadyPresentCount`, `MergedCount`, `SkipReasons` (JSON keyed by enum name; unknown names deserialize to `Unknown`, a fault), `StartedAt`, `LastProgressAt`, `SettledAt`, `LastError`, `StartedWindowSeconds`, `TargetCountAtSettle` (null until the category first settles) and `Version`, the optimistic concurrency token. No attempt counter and no decision record. `AddMigrationCheckpoints` creates the table in both providers and has not been released, so `TargetCountAtSettle` replaces the never-written `SourceTotal` column inside it, and `MergedCount` is added there too, rather than in a second migration. Run-level facts are `Settings` rows under `Migration/`, such as `Migration/HostOpenedOnTarget`. + +## Commands + +Every migration command except the source report builds a host with `AddPersistence` and never calls `StartAsync`, so no hosted service runs, including the retention sweeper and the integration event dispatcher. The source report builds no host and calls `PersistenceFactory.OpenMigrationSource` directly. + +| Command | Class | Opens RavenDB | Reads and writes | +| --- | --- | --- | --- | +| `--migration-source-report` | `MigrationSourceReportCommand` | Yes | `IMigrationSource.Describe` and `Inventory` through `MigrationSourceDescriptionPrinter`; the copy never calls either. Each `MigrationSourceFact` carries its originating setting key. `Inventory` covers every collection, including uncopied ones | +| `--migration-dry-run` | `MigrationDryRunCommand`, `MigrationDryRunArithmetic` | Yes | `MigrationPairIsSupportedCheck`, then `MigrationStartup.RunChecksAndOpen` (both halves), so it refuses wherever a real start would. Reads `Count`, `Read`, `ReadBody`, `ReadBodyVolume`, `Inventory`, `Describe`, `IMigrationTarget.Assess` (via `EFCoreMigrationRowAssessor`) and `BatchSizeFor`. Writes no row; the body probe is the only write. Exit code 0 even when the report shows problems; a refused check throws | +| `--migration-status` | `MigrationStatusCommand`, `MigrationStatusView` | No | `IMigrationCheckpointStore.ReadAll` and `MigrationEngineOptions.FromSettings`. Read-only, safe while ServiceControl runs | +| `--migration-verify` | `MigrationVerifyCommand`, `MigrationVerification` | Yes | `ReadAll`, `FromSettings`, then `IMigrationSource.Count` per selected category, judged against the rows the copy read (below). A windowed category is counted from `CopiesFrom` set to `StartedAt` minus `StartedWindowSeconds`, the window the copy used, on the source side only. Never calls `IMigrationTarget.Count`. Streams every selected category, though not the bodies, so on an embedded source ServiceControl is stopped first. Exit code 1 only when a category `IsOutstanding`; rows never read on a Done category are reported, not failed | +| `--migration-retry ` | `MigrationRetryCommand`, `MigrationOperatorCommands` | No | `Retry` checks `CanBeRetried` and builds the reset row; `IMigrationTarget.ResetForRetry` persists it | +| `--migration-abandon ` | `MigrationAbandonCommand`, `MigrationOperatorCommands` | No | Checks `CanBeAbandoned`, then writes `Abandoned` rows through the checkpoint store only | + +`MigrationOperatorCommands.Retry` builds the reset row without I/O: state `NotStarted`; `Cursor`, the four counts, `SkipReasons`, `SettledAt`, `TargetCountAtSettle` and `LastError` cleared; `StartedAt` and `StartedWindowSeconds` kept. Change retry semantics there. `ResetForRetry` only persists the row it receives, in one transaction under the execution strategy. Store-keyed categories cannot be re-read idempotently, so their writers implement `IStoreKeyedCategoryWriter.DeleteCopiedRows` and the reset deletes their rows in the same transaction: + +- `RetryOperationsWriter`: `HistoricRetryOperations`, then `UnacknowledgedRetryOperations`. +- `PendingIntegrationEventsWriter`: `ExternalIntegrationDispatchRequests`. + +Both are required categories, so the host has not opened and the target started empty. `EventLog` is also store-keyed but copies after the host opens, when its rows are indistinguishable from live ones, so it is abandon-only. + +`--migration-abandon` first re-saves the named row unchanged, when it exists, to bump its version and fence off a concurrent copy. It then abandons dependants along `MigrationCategory.MustFollowIsDataLink` (`EndpointSettings` after `KnownEndpoints`, `LicensingThroughput` after `LicensingEndpoints`), deepest first, skipping `Complete` and `Abandoned` dependants and inserting `Abandoned` rows for dependants without one, and saves the named row last. `GroupComments` follows the unresolved failed messages for ordering only and is not cascaded. An optional category with no row gets a new `Abandoned` row. Every row it saves `Abandoned` carries `TargetCountAtSettle`, read just before the save, as a settle does. No domain event is raised, because ServiceControl is stopped. + +Both operator commands rely on `MigrationCheckpointConflictException` to detect a race with a running copy. + +`MigrationVerification` judges one thing per category: RavenDB now against the rows the copy read, `CopiedCount` plus `SkippedCount` plus `AlreadyPresentCount`. That sum is exactly the rows the last complete pass read, body skips included: `RunCategoryAsync` settles `Halted` when the rows a run read differ from the rows the target accounted for, a crash resume adds to the same totals, and a retry clears them. Once the source opens RavenDB only shrinks, because `RefuseWrite` refuses every client write and only RavenDB's expiry deletes. + +- **RavenDB higher than the sum** is rows the copy never read. Verify flags them, by category. +- **RavenDB lower than the sum** is rows RavenDB removed after the copy read them, normal for the event log, the archive and a few unresolved messages with a stale expiry. Printed, not judged. +- **Expiry can hide the same number of unread rows** in the categories it touches, so a flagged number is a floor. +- **The count streams the same reader the copy used** (`RavenMigrationSource.Count`), so verify finds rows one run missed, such as a resume that skipped a range or a stream that ended early. It cannot find a reader that always misses the same rows, or a source pointed at the wrong database. +- **No live SQL count is compared with anything.** Once the host opens, normal use rewrites most categories' tables within minutes. `MergedCount` and `TargetCountAtSettle` are printed beside the sum instead. +- **An abandoned category's sum covers its last pass only.** Rows an earlier pass copied before a retry are in `TargetCountAtSettle` but not in the sum, so RavenDB minus the sum overstates what was never copied. + +**Rows never read on a Done category are accepted, not fixed.** Verify prints `ROWS NOT READ`, names the category under the table and tells the operator to keep RavenDB, because those rows exist only there. The exit code ignores them, neither operator command takes a Done row, and nothing recounts RavenDB before the host opens: the misses verify can see are particular to one run and rare, and keeping RavenDB is the remedy. + +## Readers and writers + +Readers live at `ServiceControl.Persistence.RavenDB/DataMigration/Readers/Reader.cs` and are keyed in `RavenMigrationSource`'s reader dictionary. Writers live at `ServiceControl.Persistence.EFCore/DataMigration/Writers/Writer.cs` and are keyed in the `FrozenDictionary` initialised in `EFCoreMigrationTarget`. `MigrationCategoryCoverageTests` asserts that every registry category has both. + +Every reader derives from `MigrationCategoryReader` and every writer from `MigrationCategoryWriter`, and the two for one category take the same `TDocument`, the type of each row's `Document`. The reader base streams whole documents from the primary database through `WholeDocuments`, and exposes `Lifecycle` to a reader that projects or reads the throughput database itself. The writer base casts every row once and hands `PrepareDocuments` the typed documents, so a row of another type fails the batch with an `InvalidCastException` naming the category, the row and both types. Four tests hold this: `MigrationCategoryCoverageTests.Every_reader_and_its_writer_agree_on_the_document_type`, `MigrationCategoryReadersTests.Every_reader_derives_from_the_typed_base`, `MigrationCategoryWritersTests.Every_writer_derives_from_the_typed_base` and `MigrationCategoryWritersTests.Every_writer_names_its_category_and_the_row_when_a_document_is_not_its_type`. + +| Order | Category | Kind | Reader, writer | Notes | +| --- | --- | --- | --- | --- | +| 1 | `KnownEndpoints` | Required | `KnownEndpointsReader`, `KnownEndpointsWriter` | | +| 2 | `EndpointSettings` | Required | `EndpointSettingsReader`, `EndpointSettingsWriter` | Data link to `KnownEndpoints`. Copies every setting, including one for an unknown endpoint, which `HeartbeatEndpointSettingsSyncHostedService` removes once the host opens. Reports case merges on a case-insensitive name column | +| 3 | `MessageRedirects` | Required | `MessageRedirectsReader`, `MessageRedirectsWriter` | | +| 4 | `Subscriptions` | Required | `SubscriptionsReader`, `SubscriptionsWriter` | Rows differing only in message-type version merge | +| 5 | `NotificationSettings` | Required | `NotificationSettingsReader`, `NotificationSettingsWriter` | One `Settings` row via `SettingRowMapper` | +| 6 | `TrialEndDate` | Required | `TrialEndDateReader`, `TrialEndDateWriter` | One `Settings` row via `SettingRowMapper` | +| 7 | `RetryOperations` | Required | `RetryOperationsReader`, `RetryOperationsWriter` | Two tables: historic rows store-keyed via `StoreKeyedInsert`, unacknowledged rows via `InsertMissing`, which sizes the batch. Deletes its rows on retry | +| 8 | `LicensingEndpoints` | Required | `LicensingEndpointsReader`, `LicensingEndpointsWriter` | Throughput database | +| 9 | `LicensingThroughput` | Required | `LicensingThroughputReader`, `LicensingThroughputWriter` | Throughput database. Data link to `LicensingEndpoints`. `SetLicensingEndpointThroughput` sets each day's count rather than incrementing, so re-reads are idempotent | +| 10 | `LicensingReportMasks` | Required | `LicensingReportMasksReader`, `LicensingReportMasksWriter` | Throughput database via `LicensingSettingsSource`. One `Settings` row via `SettingRowMapper` | +| 11 | `LicensedEndpointDetails` | Required | `LicensedEndpointDetailsReader`, `LicensedEndpointDetailsWriter` | Throughput database via `LicensingSettingsSource`. One `Settings` row via `SettingRowMapper` | +| 12 | `UnresolvedAndRetryIssuedFailedMessages` | Required | `FailedMessagesReader`, `FailedMessagesWriter` | Carries bodies. `FailedMessageStream` projection, `FailedMessageRowMapper` | +| 13 | `PendingIntegrationEvents` | Required | `PendingIntegrationEventsReader`, `PendingIntegrationEventsWriter` | Store-keyed via `StoreKeyedInsert`. Deletes its rows on retry | +| 14 | `CustomChecks` | Required | `CustomChecksReader`, `CustomChecksWriter` | | +| 15 | `FailedErrorImports` | Required | `FailedErrorImportsReader`, `FailedErrorImportsWriter` | Carries bodies inline in `Read`, never calls `ReadBody` | +| 16 | `GroupComments` | Required | `GroupCommentsReader`, `GroupCommentsWriter` | Ordered after `UnresolvedAndRetryIssuedFailedMessages`, not a data link. Skips blanks as `BlankGroupComment` | +| 1 | `EventLog` | Optional | `EventLogReader`, `EventLogWriter` | Windowed, store-keyed, abandon-only. Skips as `PastRetention` | +| 2 | `ArchivedAndResolvedFailedMessages` | Optional | `ArchivedAndResolvedFailedMessagesReader`, `ArchivedAndResolvedFailedMessagesWriter` | Windowed, carries bodies, shares `FailedMessagesWriter` preparation. Skips as `PastRetention` | + +## Progress reporting + +There is no migration custom check: custom checks are being removed from ServiceControl, so progress uses the activity feed, the log and `--migration-status` only. + +- `MigrationProgressReporter` raises `MigrationStarted`, `MigrationCategoryHalted` (for both Failed states), `MigrationFinished` (carrying `CategoriesAbandoned`) and `MigrationSourceUnreachable`, mapped into the ServicePulse activity feed by their `EventLogMappingDefinition` classes. It logs and swallows publish failures but propagates cancellation. All four are raised from `OptionalCategoryCopier`, so they cover the background copy only, and a Failed required category produces no activity-feed entry; its record is the startup refusal and the log. +- `MigrationSourceUnreachable` is raised once per start when `OptionalCategoryCopier` idles on a source outage, its own failed open or one the required start recorded, carrying the error and the optional categories left waiting. +- On its first pass, `OptionalCategoryCopier` logs a warning to set `Migration/Enabled` to `false` and restart when `MigrationCheckpointRules.FinishedButStillEnabled` holds: no registry category `IsOutstanding`. `--migration-status` prints the same line. `IsOutstanding` rather than `IsFinished`, so an optional category turned off with a `0` window does not hold the warning back. +- `ClosedWindowProgress` logs running required categories every 30 seconds while the host is closed. +- `--migration-status` reports state, progress, last error and available commands per category at any time, and whether migration can be turned off. diff --git a/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj b/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj index 718b22bc78..a55fcba55b 100644 --- a/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj +++ b/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj @@ -12,6 +12,9 @@ + + + @@ -20,6 +23,7 @@ + @@ -31,6 +35,9 @@ + + + diff --git a/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj b/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj index cbc66d5b4a..0d67353174 100644 --- a/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj +++ b/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj @@ -12,6 +12,9 @@ + + + @@ -20,6 +23,7 @@ + @@ -31,6 +35,9 @@ + + + diff --git a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs index 092652e7d9..3285728bd5 100644 --- a/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs +++ b/src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs @@ -29,10 +29,9 @@ namespace ServiceControl.AcceptanceTests.Recoverability using ServiceControl.Infrastructure; using ServiceControl.MessageFailures; using ServiceControl.Operations; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.EFCore.DbContexts; using ServiceControl.Persistence.EFCore.Entities; - using ServiceControl.Persistence.EFCore.Infrastructure; - using ServiceControl.Recoverability; using ServiceControl.Transports; class When_hosting_error_ingestion_only : AcceptanceTest @@ -97,9 +96,36 @@ public async Task Should_default_the_concurrency_to_10_when_none_is_configured() "InternalCustomChecksHostedService", // reports this node's ingestion health to the database "MetricsReporterHostedService", "HealthCheckPublisherHostedService", // inert, no IHealthCheckPublisher is registered - "ExternalIntegrationRequestsDataStore" // its drain is inert here, nothing calls Subscribe + "ExternalIntegrationRequestsDataStore", // its drain is inert here, nothing calls Subscribe + "FinishedCopyBeforeAnIngestionNodeOpens" // keeps this node out of a database a copy has not finished filling ]; + [Test] + public async Task Should_refuse_to_start_while_a_copy_into_the_database_is_unfinished() + { + var settings = await CreateSettings(); + + await new SetupCommand().Execute(new HostArguments([]), settings); + + var host = ErrorIngestionOnlyCommand.BuildHost(settings); + + try + { + await host.Services.GetRequiredService().Upsert( + new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Halted, null, 0, 0, null, null, null, null, null, "The source stopped responding.")); + + var exception = Assert.ThrowsAsync(() => host.StartAsync()); + + Assert.That(exception.Message, Does.Contain("has not finished") + .And.Contain(MigrationCategoryIds.KnownEndpoints) + .And.Contain("--migration-abandon KnownEndpoints")); + } + finally + { + await host.DisposeAsync(); + } + } + [Test] public void Should_refuse_to_start_against_unsupported_storage() { @@ -204,6 +230,7 @@ public async Task Should_serve_health_over_https_with_the_configured_certificate await new SetupCommand().Execute(new HostArguments([]), settings); host = ErrorIngestionOnlyCommand.BuildHost(settings); + host.Urls.Clear(); host.Urls.Add("https://127.0.0.1:0"); await host.StartAsync(); diff --git a/src/ServiceControl.Migration.AcceptanceTests/.editorconfig b/src/ServiceControl.Migration.AcceptanceTests/.editorconfig new file mode 100644 index 0000000000..da44eb13fb --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/.editorconfig @@ -0,0 +1,6 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0013.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs new file mode 100644 index 0000000000..adf724538b --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationAcceptanceTest.cs @@ -0,0 +1,472 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using NServiceBus.Extensibility; +using NServiceBus.Transport; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore; +using Particular.ServiceControl.Hosting; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.AcceptanceTesting.InfrastructureConfig; +using ServiceControl.Hosting.Commands; +using ServiceControl.Infrastructure.WebApi; +using ServiceControl.Migration.Checks; +using ServiceControl.Operations; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; +using ServiceControl.Persistence.Infrastructure; +using ServiceControl.Persistence.Tests; +using ServiceControl.Persistence.UnitOfWork; +using TestHelper; +// Aliased rather than imported: Raven.Client.Documents carries LINQ extensions that collide with EF Core's. +using IDocumentStore = Raven.Client.Documents.IDocumentStore; + +// No base class: the persistence test bases sit in projects that cannot see RavenDB. +abstract class MigrationAcceptanceTest +{ + protected const string EndpointSettingsUrl = "api/endpointssettings"; + + readonly AcceptanceTestStorageConfiguration StorageConfiguration = new(); + readonly List setVariables = []; + CancellationTokenSource hostCancellation; + Task runningHost; + + ServiceProvider targetServices; + IMigrationCheckpointStore targetCheckpointStore; + + protected Settings Settings { get; private set; } + protected HttpClient HttpClient { get; private set; } + protected (string ServerUrl, string PrimaryDatabase, string ThroughputDatabase) Source { get; private set; } + protected IDocumentStore SourceStore { get; private set; } + + protected IMigrationTarget Target { get; private set; } + + protected InMemoryBodyStoragePersistence RecordedBodies { get; } = new(); + + [SetUp] + public async Task SetUp() + { + Source = await MigrationSourceServer.CreateDatabases(); + SourceStore = await (await MigrationSourceServer.GetInstance()).Connect(); + + SetSourceVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", Source.ServerUrl); + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", Source.PrimaryDatabase); + SetSourceVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", Source.ThroughputDatabase); + SetSourceVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); + + // Settings.Port has no public setter, so the port has to come in as the environment variable a customer would set. + SetSourceVariable("SERVICECONTROL_PORT", PortUtility.GetAssignedOrAvailablePort(33500).ToString(CultureInfo.InvariantCulture)); + + var transport = new ConfigureEndpointLearningTransport(); + + Settings = new Settings( + transportType: transport.TypeName, + persisterType: StorageConfiguration.PersistenceType, + forwardErrorMessages: false, + errorRetentionPeriod: TimeSpan.FromDays(10)) + { + TransportConnectionString = transport.ConnectionString + }; + + await StorageConfiguration.CustomizeSettings(Settings); + await new SetupCommand().Execute(new HostArguments([]), Settings); + + HttpClient = new HttpClient { BaseAddress = new Uri(Settings.RootUrl) }; + + var targetServiceCollection = new ServiceCollection(); + targetServiceCollection.AddLogging(); + targetServiceCollection.AddPersistence(Settings); + targetServiceCollection.AddSingleton(RecordedBodies); + targetServices = targetServiceCollection.BuildServiceProvider(); + + Target = targetServices.GetRequiredService(); + targetCheckpointStore = targetServices.GetRequiredService(); + await Target.Open(); + } + + // The variables go first because they are process wide: a failure in the cleanup below must not leave them set for the next test. + [TearDown] + public async Task TearDown() + { + foreach (var name in setVariables) + { + Environment.SetEnvironmentVariable(name, null); + } + + try + { + await StopHost(); + } + finally + { + HttpClient?.Dispose(); + SourceStore?.Dispose(); + + if (targetServices is not null) + { + await targetServices.DisposeAsync(); + } + + await StorageConfiguration.Cleanup(); + } + } + + protected void SetSourceVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + setVariables.Add(name); + } + + protected async Task SeedSource(string database, params (string Id, object Document)[] documents) + { + using var session = SourceStore.OpenAsyncSession(database); + + foreach (var (id, document) in documents) + { + await session.StoreAsync(document, id); + } + + await session.SaveChangesAsync(); + } + + protected Task SeedSource(params (string Id, object Document)[] documents) => + SeedSource(Source.PrimaryDatabase, documents); + + protected Task SeedSourceEndpointSettings(params (string Name, bool TrackInstances)[] settings) => + SeedSource([.. settings.Select(setting => ($"EndpointSettings/{DeterministicGuid.MakeId(setting.Name)}", (object)new EndpointSettings { Name = setting.Name, TrackInstances = setting.TrackInstances }))]); + + // The target copies an endpoint setting only when that endpoint is known, so seed the endpoint too. + protected Task SeedSourceKnownEndpoints(params string[] names) => + SeedSource([.. names + .Select(name => new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" } }) + .Select(endpoint => ($"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}", (object)endpoint))]); + + protected Task OpenSource(CancellationToken cancellationToken = default) => + PersistenceFactory.OpenMigrationSource(Settings, cancellationToken); + + static IReadOnlyCollection sourceSupportedCategoryIds; + + // SupportedCategoryIds answers before Open and cannot change across it, so one source built once for the + // whole run answers it for every test instead of loading the persister assembly again per test. + protected async Task> SourceSupportedCategoryIds() + { + if (sourceSupportedCategoryIds is null) + { + await using var source = PersistenceFactory.CreateMigrationSource(Settings); + sourceSupportedCategoryIds = source.SupportedCategoryIds; + } + + return sourceSupportedCategoryIds; + } + + + // No build carries the whole migration until its last phase ships, so without this marker every copy is refused. + protected static Action AllowingAnUnreleasedMigration(Action customize = null) => + builder => + { + builder.Services.AddSingleton(); + customize?.Invoke(builder); + }; + + protected async Task RunHostUntilTheApiAnswers(Action customize = null) + { + await StopHost(); + + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + hostCancellation = new CancellationTokenSource(); + runningHost = RunCommand.Run(Settings, AllowingAnUnreleasedMigration(SignallingOnceStarted(started, customize)), hostCancellation.Token); + + await WaitForEndpointSettingsResponse(TimeSpan.FromMinutes(2)); + + // The API answers before StartedAsync runs, and a stop before StartedAsync finishes cancels the start, which RunCommand rightly treats as a failure. + await await Task.WhenAny(started.Task, runningHost).WaitAsync(TimeSpan.FromMinutes(1)); + } + + // ApplicationStarted fires only after every hosted service's StartedAsync, which is where the marker is written. + protected static Action SignallingOnceStarted(TaskCompletionSource started, Action customize) => + builder => + { + builder.Services.AddHostedService(provider => new SignalWhenStarted(provider.GetRequiredService(), started)); + customize?.Invoke(builder); + }; + + sealed class SignalWhenStarted(IHostApplicationLifetime lifetime, TaskCompletionSource started) : IHostedService + { + public Task StartAsync(CancellationToken cancellationToken = default) + { + lifetime.ApplicationStarted.Register(() => started.TrySetResult()); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + protected Settings SettingsWithMigrationEnabled(bool copyEventLog = false, Action customize = null) + { + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", copyEventLog ? null : "0"); + SetSourceVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "0"); + customize?.Invoke(Settings); + return Settings; + } + + protected async Task RunRequiredCopy( + Action customize = null, + bool leaveOptionalIncomplete = false, + CancellationToken cancellationToken = default) + { + SettingsWithMigrationEnabled(copyEventLog: leaveOptionalIncomplete); + + await RunHostUntilTheApiAnswers(builder => customize?.Invoke(builder.Services)); + + var copyable = MigrationStartup.CopyableCategoryIds(await SourceSupportedCategoryIds(), Target.SupportedCategoryIds); + + var copied = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Required && copyable.Contains(category.Id)) + .Select(category => category.Id) + .ToArray(); + + await WaitUntil( + async () => (await targetCheckpointStore.ReadAll(cancellationToken)) + .Count(checkpoint => copied.Contains(checkpoint.CategoryId) && checkpoint.State.IsFinished()) == copied.Length, + "the required copy finished"); + + if (leaveOptionalIncomplete) + { + // The guarded services read IMigrationState.AnyCategoryIncomplete, which any selected category that has not finished holds true. Halted is one of those states. + await targetCheckpointStore.Upsert( + new MigrationCheckpoint("EventLog", MigrationCategoryState.Halted, null, 0, 0, null, null, null, null, null, + "Halted by the test fixture so the guarded services see an unfinished category."), + cancellationToken); + } + } + + protected async Task> WaitForEndpointSettingsResponse(TimeSpan timeout) + { + IReadOnlyList settings = null; + + await WaitUntil(async () => + { + // A host that refused to start surfaces its own exception here instead of waiting out the timeout. + if (runningHost is { IsFaulted: true }) + { + await runningHost; + } + + try + { + settings = await GetEndpointSettings(); + return true; + } + catch (HttpRequestException) + { + return false; + } + }, $"GET {EndpointSettingsUrl} answered", timeout); + + return settings; + } + + protected async Task> GetEndpointSettings() => + await HttpClient.GetFromJsonAsync>(EndpointSettingsUrl, SerializerOptions.Default); + + async Task StopHost() + { + if (runningHost is null) + { + return; + } + + await hostCancellation.CancelAsync(); + await runningHost; + hostCancellation.Dispose(); + runningHost = null; + } + + // Copied from PersistenceTestBase.WaitUntil, which this project cannot compile. + protected static async Task WaitUntil(Func> conditionChecker, string condition, TimeSpan timeout = default) + { + timeout = timeout == default ? TimeSpan.FromSeconds(10) : timeout; + + var start = DateTime.UtcNow; + + while (DateTime.UtcNow - start < timeout) + { + if (await conditionChecker()) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(500)); + } + + throw new Exception($"{condition} has not been meet in defined timespan: {timeout})"); + } + + // Copied from RavenMigrationSourceTestBase.CollectBatches, which this project cannot compile. + protected static async Task> CollectBatches(IMigrationSource source, MigrationCategory category, string resumeAfter = null, int batchSize = 100) + { + var batches = new List(); + + await foreach (var batch in source.Read(category, resumeAfter, batchSize, TestContext.CurrentContext.CancellationToken)) + { + batches.Add(batch); + } + + return batches; + } + + protected IEndpointSettingsStore EndpointSettingsStore => targetServices.GetRequiredService(); + protected IMonitoringDataStore MonitoringDataStore => targetServices.GetRequiredService(); + + protected async Task CopyCategory(string categoryId, CancellationToken cancellationToken = default) + { + var category = MigrationCategoryRegistry.Find(categoryId) ?? throw new ArgumentException($"'{categoryId}' is not a migration category.", nameof(categoryId)); + var before = await targetCheckpointStore.Read(categoryId, cancellationToken); + var target = new RecordingMigrationTarget(Target); + var options = new MigrationEngineOptions(TimeSpan.Zero, MigrationSettings.DefaultHaltThresholdPercent, MigrationSettings.DefaultHaltThresholdMinimum, []) { BodyRetryBackoff = TimeSpan.Zero }; + + await using var source = await OpenSource(cancellationToken); + var engine = new MigrationEngine(source, target, targetCheckpointStore, TimeProvider.System, options, NullLogger.Instance); + + var after = await engine.RunCategoryAsync(category, cancellationToken); + + // Both reached the end of the source, so a test can assert a Failed category's skips as well as a Done one's. + if (after.State is not (MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors)) + { + throw new InvalidOperationException($"Copying '{categoryId}' ended {after.State}: {after.LastError}"); + } + + return new MigrationWriteResult( + after, + (int)(after.CopiedCount - (before?.CopiedCount ?? 0)), + (int)(after.SkippedCount - (before?.SkippedCount ?? 0)), + target.SkippedIds, + (int)(after.AlreadyPresentCount - (before?.AlreadyPresentCount ?? 0))); + } + + + // Makes the target look like a database a migration has touched, which is the only state the host-opened marker is stamped in. + protected async Task SeedCheckpoint(string categoryId, MigrationCategoryState state = MigrationCategoryState.Complete) + { + await using var scope = targetServices.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await dbContext.UpsertCheckpoint(new MigrationCheckpoint(categoryId, state, null, 0, 0, null, null, null, null, null, null)); + } + + protected async Task DeleteCheckpoint(string categoryId) + { + await using var scope = targetServices.CreateAsyncScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await dbContext.MigrationCheckpoints.Where(checkpoint => checkpoint.CategoryId == categoryId).ExecuteDeleteAsync(); + } + + protected async Task QueryTarget(Func> query) + { + using var scope = targetServices.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + return await query(dbContext); + } + + protected async Task GetFailedMessage(Guid uniqueMessageId) + { + var row = await QueryTarget(dbContext => dbContext.FailedMessages.AsNoTracking().SingleOrDefaultAsync(m => m.UniqueMessageId == uniqueMessageId)); + + Assert.That(row, Is.Not.Null, $"No failed message row for {uniqueMessageId}"); + + return row; + } + + protected Task FindFailedMessage(Guid uniqueMessageId) => + QueryTarget(dbContext => dbContext.FailedMessages.AsNoTracking().SingleOrDefaultAsync(m => m.UniqueMessageId == uniqueMessageId)); + + protected EFPersisterSettings EFSettings => targetServices.GetRequiredService(); + + protected Task Ingest(params IngestedFailure[] failures) => + InBatch(async unitOfWork => + { + foreach (var failure in failures) + { + await unitOfWork.Recoverability.RecordFailedProcessingAttempt(failure.Context, failure.ProcessingAttempt, failure.Groups); + } + }); + + // Ingestion takes the body and its content type from the message context, so only the context changes. + protected Task IngestWithBody(IngestedFailure failure, byte[] body, string contentType) => + InBatch(unitOfWork => + { + var headers = new Dictionary(failure.Headers) { [NServiceBus.Headers.ContentType] = contentType }; + var context = new MessageContext(failure.MessageId, headers, body, new TransportTransaction(), "receiveAddress", new ContextBag()); + + return unitOfWork.Recoverability.RecordFailedProcessingAttempt(context, failure.ProcessingAttempt, failure.Groups); + }); + + async Task InBatch(Func record) + { + await using var unitOfWork = await targetServices.GetRequiredService().StartNew(); + + await record(unitOfWork); + + await unitOfWork.Complete(TestContext.CurrentContext.CancellationToken); + } + + protected Task ReadCheckpoint(string categoryId) => + QueryTarget(async dbContext => + { + var row = await dbContext.MigrationCheckpoints.AsNoTracking().SingleOrDefaultAsync(checkpoint => checkpoint.CategoryId == categoryId); + + return row is null ? null : new MigrationCheckpoint( + row.CategoryId, row.State, row.Cursor, + row.CopiedCount, row.SkippedCount, row.SourceTotal, row.SkipReasons, + row.StartedAt, row.LastProgressAt, row.SettledAt, row.LastError, + row.AlreadyPresentCount, row.Version); + }); + + protected Task ReadSetting(string key) => + QueryTarget(dbContext => dbContext.Settings.AsNoTracking() + .Where(setting => setting.Key == key) + .Select(setting => setting.Value) + .SingleOrDefaultAsync()); + + sealed class RecordingMigrationTarget(IMigrationTarget inner) : IMigrationTarget + { + public List SkippedIds { get; } = []; + + public Task Open(CancellationToken cancellationToken = default) => inner.Open(cancellationToken); + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => inner.BatchSizeFor(category, cancellationToken); + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var result = await inner.Write(category, batch, checkpointToExtend, cancellationToken); + SkippedIds.AddRange(result.SkippedIds); + return result; + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; + + public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs b/src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs new file mode 100644 index 0000000000..88a33b0003 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/MigrationSourceServer.cs @@ -0,0 +1,82 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting.Internal; +using Microsoft.Extensions.Logging.Abstractions; +using NUnit.Framework; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.RavenDB; +using TestHelper; + +// Its own copy rather than SharedEmbeddedServer: that one uses the RavenDB persister's own settings type, which this project cannot reference because it loads the persister from its manifest instead. +static class MigrationSourceServer +{ + public static async Task GetInstance(CancellationToken cancellationToken = default) + { + await startLock.WaitAsync(cancellationToken); + try + { + if (server != null) + { + return server; + } + + var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "MigrationSource"); + var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs", "MigrationSource"); + var port = PortUtility.GetAssignedOrAvailablePort(33350); + + var configuration = new EmbeddedDatabaseConfiguration($"http://localhost:{port}", "primary", dbPath, logPath, "Operations") { RunInMemory = true }; + + server = EmbeddedDatabase.Start(configuration, lifetime); + return server; + } + finally + { + startLock.Release(); + } + } + + public static async Task Stop() + { + await startLock.WaitAsync(); + try + { + if (server is null) + { + return; + } + + await server.Stop(CancellationToken.None); + server.Dispose(); + server = null; + lifetime.StopApplication(); + } + finally + { + startLock.Release(); + } + } + + public static async Task<(string ServerUrl, string PrimaryDatabase, string ThroughputDatabase)> CreateDatabases(CancellationToken cancellationToken = default) + { + var instance = await GetInstance(cancellationToken); + var primary = $"sc_src_{Guid.NewGuid():n}"; + var throughput = $"{primary}-throughput"; + + using var store = await instance.Connect(cancellationToken); + foreach (var name in new[] { primary, throughput }) + { + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(name)), cancellationToken); + } + + return (instance.ServerUrl, primary, throughput); + } + + static EmbeddedDatabase server; + static readonly ApplicationLifetime lifetime = new(new NullLogger()); + static readonly SemaphoreSlim startLock = new(1, 1); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs b/src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs new file mode 100644 index 0000000000..b42d2671f2 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/StopMigrationSourceServer.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Threading.Tasks; +using NUnit.Framework; + +[SetUpFixture] +public class StopMigrationSourceServer +{ + [OneTimeTearDown] + public Task Teardown() => MigrationSourceServer.Stop(); +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs new file mode 100644 index 0000000000..fd07ff535b --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/TestMigrationTargets.cs @@ -0,0 +1,121 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +static class TestMigrationTargets +{ + public static void ParkFirstMigrationWrite(this WebApplicationBuilder builder, TaskCompletionSource parked, Task release) => + builder.DecorateMigrationTarget(inner => new ParkingMigrationTarget(inner, parked, release)); + + public static void FailTheSecondMigrationWrite(this WebApplicationBuilder builder) => + builder.DecorateMigrationTarget(inner => new FaultingMigrationTarget(inner, failingWrite: 2)); + + public static void HaltTheEndpointSettingsCategory(this WebApplicationBuilder builder) => + builder.DecorateMigrationTarget(inner => new FaultingMigrationTarget(inner, failingWrite: 1)); + + public static void StopAfterTheFirstCategorySettles(this WebApplicationBuilder builder, CancellationTokenSource stopping) + { + var registered = builder.Services.Last(service => service.ServiceType == typeof(IMigrationCheckpointStore)); + + if (registered.ImplementationType is null) + { + throw new InvalidOperationException( + "IMigrationCheckpointStore is registered by a factory rather than by type, so the test decorator cannot rebuild the inner store. Register it as AddSingleton() or give the decorator a different seam."); + } + + builder.Services.AddSingleton(provider => + new StoppingAfterTheFirstSettleCheckpointStore((IMigrationCheckpointStore)ActivatorUtilities.CreateInstance(provider, registered.ImplementationType), stopping)); + } + + public static void DecorateMigrationTarget(this WebApplicationBuilder builder, Func decorate) + { + var registered = builder.Services.Last(service => service.ServiceType == typeof(IMigrationTarget)); + + if (registered.ImplementationType is null) + { + throw new InvalidOperationException( + "IMigrationTarget is registered by a factory rather than by type, so the test decorator cannot rebuild the inner target. Register it as AddSingleton() or give the decorator a different seam."); + } + + builder.Services.AddSingleton(provider => + decorate((IMigrationTarget)ActivatorUtilities.CreateInstance(provider, registered.ImplementationType))); + } +} + +// Throws on one EndpointSettings write, which is the only way to see what a copy does after a write has failed. +sealed class FaultingMigrationTarget(IMigrationTarget inner, int failingWrite) : IMigrationTarget +{ + int endpointSettingsWrites; + + public Task Open(CancellationToken cancellationToken = default) => inner.Open(cancellationToken); + + // Two rows a batch, so five settings take three writes and a failure on the second leaves exactly one batch committed. + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => + category.Id == MigrationCategoryIds.EndpointSettings ? Task.FromResult(2) : inner.BatchSizeFor(category, cancellationToken); + + public Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) => + category.Id == MigrationCategoryIds.EndpointSettings && ++endpointSettingsWrites == failingWrite + ? throw new Exception("injected mid-category failure") + : inner.Write(category, batch, checkpointToExtend, cancellationToken); + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; + + public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; +} + +// Stops the copy the moment the first category settles Done or Abandoned, which is the only way to stop it between two categories. +sealed class StoppingAfterTheFirstSettleCheckpointStore(IMigrationCheckpointStore inner, CancellationTokenSource stopping) : IMigrationCheckpointStore +{ + public Task> ReadAll(CancellationToken cancellationToken = default) => inner.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => inner.Read(categoryId, cancellationToken); + + public async Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + var stored = await inner.Upsert(checkpoint, cancellationToken); + + if (stored.State.IsFinished()) + { + await stopping.CancelAsync(); + throw new OperationCanceledException(stopping.Token); + } + + return stored; + } +} + +// Holds the copy open at a moment a test can observe, which is the only way to ask what the API answers mid-copy. +sealed class ParkingMigrationTarget(IMigrationTarget inner, TaskCompletionSource parked, Task release) : IMigrationTarget +{ + int writes; + + public Task Open(CancellationToken cancellationToken = default) => inner.Open(cancellationToken); + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => inner.BatchSizeFor(category, cancellationToken); + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + if (Interlocked.Exchange(ref writes, 1) == 0) + { + parked.SetResult(); + await release.WaitAsync(cancellationToken); + } + + return await inner.Write(category, batch, checkpointToExtend, cancellationToken); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => inner.Count(category, cancellationToken); + + public IReadOnlyCollection SupportedCategoryIds => inner.SupportedCategoryIds; + + public IReadOnlyDictionary DocumentTypes => inner.DocumentTypes; +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs new file mode 100644 index 0000000000..a55deaba69 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_copy_is_killed_mid_category.cs @@ -0,0 +1,182 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore; + +[TestFixture] +[NonParallelizable] +class When_a_copy_is_killed_mid_category : MigrationAcceptanceTest +{ + [Test] + public void A_migrating_host_stopped_while_starting_exits_cleanly() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.DoesNotThrowAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(StoppedWhileStarting), cancellation.Token)); + } + + [Test] + public void A_host_not_migrating_stopped_while_starting_still_fails() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.CatchAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(StoppedWhileStarting), cancellation.Token)); + } + + // Only a stop is a clean exit; a cancellation nobody asked for is still a failed start, migrating or not. + [Test] + public void A_migrating_host_whose_start_is_cancelled_without_a_stop_still_fails() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.CatchAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(CancelledWithoutAStop), cancellation.Token)); + } + + [Test] + public async Task It_resumes_rather_than_starting_over() + { + await SeedSourceKnownEndpoints("A", "B", "C", "D", "E"); + await SeedSourceEndpointSettings(("A", true), ("B", true), ("C", true), ("D", true), ("E", true)); + + // Bounded rather than None: a copy that did not fail would otherwise hold this call open forever. + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var killed = Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); + + Assert.That(killed, Is.Not.Null); + Assert.That(await TargetEndpointSettingsCount(), Is.EqualTo(2), "the first batch and its cursor should have committed together"); + + var refusedAgain = Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token)); + Assert.That(refusedAgain.Message, Is.EqualTo(killed.Message), "a start re-ran a Failed category the operator had not put back"); + + // The row a kill part way through the category leaves: still copying, with the first batch's cursor and counts. + var halted = await ReadCheckpoint("EndpointSettings"); + await QueryTarget(dbContext => dbContext.UpsertCheckpoint(halted with { State = MigrationCategoryState.InProgress, LastError = null, SettledAt = null })); + + await RunHostUntilTheApiAnswers(); + + var checkpoint = await ReadCheckpoint("EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await GetEndpointSettingsNames(), Is.EquivalentTo(new[] { "A", "B", "C", "D", "E" }), "no gaps"); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(5), "no row copied twice"); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.AlreadyPresentCount, Is.Zero, "a resumed run must not re-read rows the first run committed"); + } + } + + // Proves the already-present count in the test above can fail: both runs end with five correct rows, so that count is the only thing telling a resume from a fresh copy. + [Test] + public async Task Without_its_checkpoint_the_second_run_re_reads_what_the_first_already_wrote() + { + await SeedSourceKnownEndpoints("A", "B", "C", "D", "E"); + await SeedSourceEndpointSettings(("A", true), ("B", true), ("C", true), ("D", true), ("E", true)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.FailTheSecondMigrationWrite()), cancellation.Token)); + + await DeleteCheckpoint("EndpointSettings"); + await RunHostUntilTheApiAnswers(); + + var checkpoint = await ReadCheckpoint("EndpointSettings"); + + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(2)); + } + + // A copy stopped between two categories must still list the second, or every gate reading the table finds nothing outstanding. + [Test] + public async Task A_worker_started_after_a_copy_stopped_between_categories_names_the_one_it_never_reached() + { + await SeedSourceKnownEndpoints("A", "B"); + await SeedSourceEndpointSettings(("A", true), ("B", true)); + Settings.MaximumConcurrencyLevel = 1; + + using var stopping = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + Assert.CatchAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.StopAfterTheFirstCategorySettles(stopping)), stopping.Token)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var worker = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + try + { + using (Assert.EnterMultipleScope()) + { + Assert.That((await ReadCheckpoint(MigrationCategoryIds.KnownEndpoints))?.State, Is.EqualTo(MigrationCategoryState.Complete), "the stop has to land between the two categories for this test to mean anything"); + Assert.That(async () => await worker.StartAsync(cancellation.Token), + Throws.Exception.With.Message.Contain("EndpointSettings is Copying (NotStarted)").And.Message.Not.Contain("KnownEndpoints is"), + "a worker let into a database whose copy stopped after its first category"); + Assert.That((await ReadCheckpoint(MigrationCategoryIds.EndpointSettings))?.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + } + } + finally + { + await worker.DisposeAsync(); + } + } + + Task TargetEndpointSettingsCount() => + QueryTarget(dbContext => dbContext.EndpointSettings.CountAsync()); + + Task GetEndpointSettingsNames() => + QueryTarget(dbContext => dbContext.EndpointSettings.Select(settings => settings.Name).ToArrayAsync()); + + static void StoppedWhileStarting(WebApplicationBuilder builder) => + builder.Services.AddHostedService(provider => new StopsTheHostWhileStarting(provider.GetRequiredService())); + + static void CancelledWithoutAStop(WebApplicationBuilder builder) => + builder.Services.AddHostedService(_ => new CancelsWhileStarting()); + + sealed class CancelsWhileStarting : IHostedLifecycleService + { + public Task StartingAsync(CancellationToken cancellationToken = default) => throw new OperationCanceledException(); + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + // What a service stop or the critical-error handler does to a host that is still starting. + sealed class StopsTheHostWhileStarting(IHostApplicationLifetime lifetime) : IHostedLifecycleService + { + public Task StartingAsync(CancellationToken cancellationToken = default) + { + lifetime.StopApplication(); + throw new OperationCanceledException(lifetime.ApplicationStopping); + } + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs b/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs new file mode 100644 index 0000000000..86cd583c29 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs @@ -0,0 +1,266 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.Entities; + +[TestFixture] +[NonParallelizable] +class When_a_startup_check_fails : MigrationAcceptanceTest +{ + const string HostOpenedSetting = "Migration/HostOpenedOnTarget"; + + [Test] + public async Task An_optional_category_window_that_is_not_a_time_span_is_refused() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "a week"); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the optional category windows are valid").And.Contain("a week")); + } + + [Test] + public async Task A_retry_history_depth_that_empties_the_migrated_table_is_refused() + { + // Assigned rather than set as an environment variable: Settings reads RetryHistoryDepth once, in the constructor the fixture has already run. + Settings.RetryHistoryDepth = 0; + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the retry history depth will not empty a migrated table") + .And.Contain("RetryHistoryDepth") + .And.Contain("HistoricRetryOperations")); + } + + [Test] + public async Task A_target_whose_schema_migrations_were_never_applied_is_refused() + { + await QueryTarget(async dbContext => + { + // EF's own history repository, because it is the only thing that knows where the history table is once the persister is given a schema. + var history = dbContext.GetService(); + var applied = await history.GetAppliedMigrationsAsync(); + + foreach (var row in applied) + { + await dbContext.Database.ExecuteSqlRawAsync(history.GetDeleteScript(row.MigrationId)); + } + + return applied.Count; + }); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the target schema is current").And.Contain("--setup")); + } + + [Test] + public async Task A_target_that_already_holds_data_is_refused() + { + await QueryTarget(dbContext => + { + dbContext.Settings.Add(new SettingEntity { Key = "NotificationEmails", Value = "{}" }); + return dbContext.SaveChangesAsync(); + }); + var settingsTable = await QueryTarget(dbContext => Task.FromResult(dbContext.Model.FindEntityType(typeof(SettingEntity))!.GetTableName()!)); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the migration target holds no ServiceControl data").And.Contain(settingsTable).And.Contain("--setup")); + } + + [Test] + public async Task Message_body_storage_that_cannot_be_written_to_is_refused() + { + var parentThatIsAFile = Path.Combine(Path.GetTempPath(), $"sc-not-a-directory-{Guid.NewGuid():n}"); + File.WriteAllText(parentThatIsAFile, string.Empty); + + try + { + ((FileSystemBodyStorageSettings)EFSettings.BodyStorage).StoragePath = Path.Combine(parentThatIsAFile, "bodies"); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("message body storage is writable")); + } + finally + { + File.Delete(parentThatIsAFile); + } + } + + [Test] + public async Task A_client_certificate_that_has_expired_is_refused() + { + var notBefore = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero); + var notAfter = new DateTimeOffset(2021, 1, 1, 0, 0, 0, TimeSpan.Zero); + + SetSourceVariable("SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEBASE64", ExpiredClientCertificate(notBefore, notAfter)); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the migration source opens") + .And.Contain($"{notBefore.UtcDateTime:u} to {notAfter.UtcDateTime:u}")); + } + + [Test] + public async Task A_throughput_database_that_does_not_exist_is_refused() + { + var missing = $"{Source.ThroughputDatabase}-does-not-exist"; + SetSourceVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", missing); + + var refusal = await RefusedStartup(); + + Assert.That(refusal.Message, Does.Contain("the migration source opens") + .And.Contain(missing) + .And.Contain("LicensingComponent/RavenDB/ThroughputDatabaseName")); + } + + // Without this check a build that copies the required set commits a customer to SQL before the rest of the migration exists. + [Test] + public async Task A_build_without_the_whole_migration_is_refused() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "a week"); + + var refusal = await RefusedStartup(allowUnreleasedMigration: false); + + Assert.That(refusal.Message, Does.Contain("this build carries the whole migration") + .And.Not.Contain("the optional category windows are valid"), + "this refusal comes first, so no other check is consulted on an unreleased build"); + } + + [Test] + public async Task A_halted_required_category_stops_the_host_rather_than_opening_it() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var exception = Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(builder => builder.HaltTheEndpointSettingsCategory()), cancellation.Token)); + + Assert.Multiple(() => + { + Assert.That(exception.Message, Does.Contain("EndpointSettings").And.Contain("Halted")); + Assert.That(exception.Message, Does.Contain("nothing has been lost")); + Assert.That(exception.Message, Does.Contain("--migration-retry EndpointSettings").And.Contain("--migration-abandon EndpointSettings"), "a Failed category waits for the operator, so the refusal has to name what moves it on"); + Assert.That(async () => await HttpClient.GetAsync(EndpointSettingsUrl), Throws.InstanceOf(), "Kestrel bound its socket behind a halted required category"); + }); + } + + [Test] + public async Task A_source_that_will_not_open_once_the_required_copy_settled_still_lets_the_host_open() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + await RunHostUntilTheApiAnswers(); + + var missing = $"{Source.PrimaryDatabase}-does-not-exist"; + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", missing); + + await RunHostUntilTheApiAnswers(); + + var endpointSettings = await GetEndpointSettings(); + var eventLog = await ReadCheckpoint(MigrationCategoryIds.EventLog); + var archived = await ReadCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpointSettings.Select(setting => setting.Name), Does.Contain("Sales")); + Assert.That(eventLog?.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(eventLog?.LastError, Does.Contain(missing).And.Not.Contain("will not start"), "status reads the outage from the row, on a start that did open"); + Assert.That(archived?.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(archived?.LastError, Does.Contain(missing)); + } + } + + [Test] + public async Task A_source_that_will_not_open_before_the_required_copy_settled_still_refuses() + { + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + await SeedCheckpoint(MigrationCategoryIds.EndpointSettings, MigrationCategoryState.InProgress); + + var missing = $"{Source.PrimaryDatabase}-does-not-exist"; + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", missing); + + var refusal = await RefusedStartup(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refusal.Message, Does.Contain("the migration source opens").And.Contain(missing)); + Assert.That(refusal.Message, Does.Contain("--migration-abandon"), "a required category that never started has no exit once the source is gone, and the refusal has to say so"); + Assert.That(await ReadCheckpoint(MigrationCategoryIds.EventLog), Is.Null, "a refusal writes nothing"); + } + } + + [Test] + public async Task A_failed_optional_row_does_not_stop_the_host_opening() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + await RunHostUntilTheApiAnswers(); + + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.CompleteWithErrors); + SetSourceVariable("SERVICECONTROL_RAVENDB_DATABASENAME", $"{Source.PrimaryDatabase}-does-not-exist"); + + await RunHostUntilTheApiAnswers(); + + var eventLog = await ReadCheckpoint(MigrationCategoryIds.EventLog); + + using (Assert.EnterMultipleScope()) + { + Assert.That(eventLog?.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(eventLog?.LastError, Is.Null, "the outage leaves a Failed row alone"); + } + } + + // Every refusal is asserted against a source that has rows to copy: an empty target proves nothing otherwise. + // Only the release gate's own test withholds the marker; every other refusal has to get past it. + async Task RefusedStartup(bool allowUnreleasedMigration = true) + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + var refusal = Assert.ThrowsAsync(async () => + await RunCommand.Run(Settings, allowUnreleasedMigration ? AllowingAnUnreleasedMigration() : null, cancellation.Token)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(refusal.Message, Does.Contain("this start has copied nothing")); + Assert.That(await TargetEndpointSettingsCount(), Is.Zero, "a check that fires after rows have moved is worse than no check"); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "a refused startup has opened on nothing, so the abort is still free"); + } + + return refusal; + } + + Task TargetEndpointSettingsCount() => + QueryTarget(dbContext => dbContext.EndpointSettings.CountAsync()); + + static string ExpiredClientCertificate(DateTimeOffset notBefore, DateTimeOffset notAfter) + { + using var key = RSA.Create(2048); + var request = new CertificateRequest("CN=sc-migration-source-expired", key, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + using var certificate = request.CreateSelfSigned(notBefore, notAfter); + + return Convert.ToBase64String(certificate.Export(X509ContentType.Pkcs12)); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs b/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs new file mode 100644 index 0000000000..07d17190d4 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_categories_change_between_restarts.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System.Threading.Tasks; +using NUnit.Framework; + +[TestFixture] +[NonParallelizable] +class When_categories_change_between_restarts : MigrationAcceptanceTest +{ + [Test] + public async Task Changing_the_windows_between_restarts_deletes_nothing() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "0"); + await RunHostUntilTheApiAnswers(); + Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "a category no run has copied has no row, and no column anywhere says it was selected"); + var firstRun = await ReadCheckpoint("EndpointSettings"); + + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + await RunHostUntilTheApiAnswers(); + Assert.That(await ReadCheckpoint("EventLog"), Is.Null, "the required copy runs required categories only, so turning an optional one on starts nothing here"); + + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "0"); + await RunHostUntilTheApiAnswers(); + + var endpointSettings = await ReadCheckpoint("EndpointSettings"); + + Assert.Multiple(() => + { + Assert.That(endpointSettings.CopiedCount, Is.EqualTo(1), "a finished category must not be copied again when the selection changes around it"); + Assert.That(endpointSettings.Cursor, Is.Not.Null, "changing the selection must not delete or reset what an earlier run recorded"); + Assert.That(endpointSettings.Version, Is.EqualTo(firstRun.Version), "a row saved again, even unchanged, would have moved its version"); + Assert.That(endpointSettings.StartedAt, Is.EqualTo(firstRun.StartedAt), "a copy run again from scratch would have a new start"); + }); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs b/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs new file mode 100644 index 0000000000..29d5b8b432 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_endpoint_settings_are_migrated.cs @@ -0,0 +1,43 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; + +[TestFixture] +[NonParallelizable] +class When_endpoint_settings_are_migrated : MigrationAcceptanceTest +{ + [Test] + public async Task The_source_server_starts_and_creates_both_databases() + { + var source = await MigrationSourceServer.CreateDatabases(); + + Assert.That(source.ServerUrl, Does.StartWith("http://localhost:")); + Assert.That(source.PrimaryDatabase, Is.Not.Empty); + Assert.That(source.ThroughputDatabase, Is.EqualTo($"{source.PrimaryDatabase}-throughput")); + } + + [Test] + public async Task They_are_readable_through_the_product_read_api() + { + await SeedSourceKnownEndpoints("Sales", "Billing"); + await SeedSourceEndpointSettings(("Sales", true), ("Billing", false), ("", true)); + + // A clock that never moves keeps HeartbeatEndpointSettingsSyncHostedService's twenty second + // delay from elapsing and rewriting these rows while the assertions read them. + await RunHostUntilTheApiAnswers(builder => builder.Services.AddSingleton(new FakeTimeProvider())); + + var settings = await GetEndpointSettings(); + + Assert.Multiple(() => + { + Assert.That(settings.Single(row => row.Name == "Sales").TrackInstances, Is.True); + Assert.That(settings.Single(row => row.Name == "Billing").TrackInstances, Is.False); + Assert.That(settings.Single(row => row.Name == string.Empty).TrackInstances, Is.True); + }); + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs new file mode 100644 index 0000000000..a57aa10426 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_api_is_called_during_the_required_copy.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Hosting.Commands; + +[TestFixture] +// Mandatory, not stylistic: this assembly is Parallelizable(ParallelScope.All) and these fixtures set +// process-global environment variables. One fixture added without it makes the whole suite intermittent. +[NonParallelizable] +class When_the_api_is_called_during_the_required_copy : MigrationAcceptanceTest +{ + [Test] + public async Task The_api_does_not_answer_until_the_required_copy_has_finished() + { + await SeedSourceKnownEndpoints("Sales", "Billing", "Shipping"); + await SeedSourceEndpointSettings(("Sales", true), ("Billing", true), ("Shipping", true)); + + var copyIsParked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseTheCopy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var host = RunCommand.Run(Settings, AllowingAnUnreleasedMigration(SignallingOnceStarted(started, builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task))), cancellation.Token); + + await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); + Assert.That(releaseTheCopy.Task.IsCompleted, Is.False, "the copy must still be parked for this assertion to mean anything"); + Assert.That(async () => await HttpClient.GetAsync(EndpointSettingsUrl), Throws.InstanceOf(), + "Kestrel had already bound its socket while the required copy was still running"); + + releaseTheCopy.SetResult(); + var settings = await WaitForEndpointSettingsResponse(TimeSpan.FromMinutes(2)); + + Assert.That(settings.Select(row => row.Name), Is.SupersetOf(new[] { "Sales", "Billing", "Shipping" })); + + await started.Task.WaitAsync(TimeSpan.FromMinutes(1)); + await cancellation.CancelAsync(); + await host; + } +} diff --git a/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs b/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs new file mode 100644 index 0000000000..7620e21618 --- /dev/null +++ b/src/ServiceControl.Migration.AcceptanceTests/When_the_host_opens_on_the_target.cs @@ -0,0 +1,258 @@ +namespace ServiceControl.Migration.AcceptanceTests; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Hosting.Commands; + +[TestFixture] +// Mandatory, not stylistic: this assembly is Parallelizable(ParallelScope.All) and these fixtures set +// process-global environment variables. One fixture added without it makes the whole suite intermittent. +[NonParallelizable] +class When_the_host_opens_on_the_target : MigrationAcceptanceTest +{ + const string HostOpenedSetting = "Migration/HostOpenedOnTarget"; + const string CategoryFromANewerBuild = "CategoryFromANewerBuild"; + + [Test] + public async Task The_row_does_not_exist_while_the_required_copy_is_still_running() + { + await SeedSourceKnownEndpoints("Sales"); + await SeedSourceEndpointSettings(("Sales", true)); + + var copyIsParked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releaseTheCopy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var host = RunCommand.Run(Settings, AllowingAnUnreleasedMigration(SignallingOnceStarted(started, builder => builder.ParkFirstMigrationWrite(copyIsParked, releaseTheCopy.Task))), cancellation.Token); + + await copyIsParked.Task.WaitAsync(TimeSpan.FromMinutes(2)); + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "the abort is still free at this moment, and this row is what says it is not"); + + releaseTheCopy.SetResult(); + await WaitForEndpointSettingsResponse(TimeSpan.FromMinutes(2)); + await started.Task.WaitAsync(TimeSpan.FromMinutes(1)); + + Assert.That(await ReadSetting(HostOpenedSetting), Is.Not.Null); + + await cancellation.CancelAsync(); + await host; + } + + [Test] + public async Task A_start_with_migration_off_does_not_stamp_the_target() + { + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + + await RunHostUntilTheApiAnswers(); + + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "only a start with the migration on may write the marker"); + } + + [Test] + public async Task A_start_with_migration_off_does_not_read_the_checkpoint_table() + { + await DropTheCheckpointTable(); + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + + Assert.That(async () => await RunHostUntilTheApiAnswers(), Throws.Nothing, "every hosted service, StartedAsync included, ran without touching the missing table"); + } + + [Test] + public async Task An_ingestion_only_host_is_refused_while_a_required_category_is_failed() + { + Settings.MaximumConcurrencyLevel = 1; + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedRequiredCategoriesComplete(except: MigrationCategoryIds.MessageRedirects); + await SeedCheckpoint(MigrationCategoryIds.MessageRedirects, MigrationCategoryState.CompleteWithErrors); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + try + { + var refusal = Assert.ThrowsAsync(async () => await app.StartAsync(cancellation.Token)); + + Assert.That(refusal.Message, Does.Contain("MessageRedirects is Failed (CompleteWithErrors)") + .And.Contain("--migration-retry MessageRedirects") + .And.Contain("--migration-abandon MessageRedirects")); + } + finally + { + await app.DisposeAsync(); + } + } + + [Test] + public async Task An_ingestion_only_host_starts_while_an_optional_category_is_still_copying_or_failed() + { + Settings.MaximumConcurrencyLevel = 1; + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedRequiredCategoriesComplete(); + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress); + await SeedCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + try + { + Assert.That(async () => await app.StartAsync(cancellation.Token), Throws.Nothing, "an optional category copies in the background after the host opens, so it never holds a worker back"); + await app.StopAsync(cancellation.Token); + } + finally + { + await app.DisposeAsync(); + } + } + + [Test] + public async Task An_ingestion_only_host_is_refused_over_a_category_this_build_does_not_know() + { + Settings.MaximumConcurrencyLevel = 1; + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedRequiredCategoriesComplete(); + await SeedCheckpoint(CategoryFromANewerBuild, MigrationCategoryState.InProgress); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + try + { + var refusal = Assert.ThrowsAsync(async () => await app.StartAsync(cancellation.Token)); + + Assert.That(refusal.Message, Does.Contain($"{CategoryFromANewerBuild} is Copying (InProgress)"), "nobody here can say whether an id this build does not hold is optional, so it holds the worker back"); + } + finally + { + await app.DisposeAsync(); + } + } + + [Test] + public async Task Importing_failed_errors_is_refused_while_a_copy_is_unfinished() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + using var host = ImportFailedErrorsCommand.BuildHost(Settings); + + var refusal = Assert.ThrowsAsync(async () => await host.StartAsync(cancellation.Token)); + + Assert.That(refusal.Message, Does.Contain("--import-failed-errors will not start").And.Contain("KnownEndpoints is Copying (InProgress)")); + } + + [Test] + public async Task Importing_failed_errors_starts_while_an_optional_category_is_still_copying_or_failed() + { + SetSourceVariable("SERVICECONTROL_MIGRATION_ENABLED", "false"); + await SeedRequiredCategoriesComplete(); + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress); + await SeedCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + using var host = ImportFailedErrorsCommand.BuildHost(Settings); + + Assert.That(async () => await host.StartAsync(cancellation.Token), Throws.Nothing, "an optional category copies in the background after the host opens, so it never holds the import back"); + await host.StopAsync(cancellation.Token); + } + + // A customer whose migration never started must not be told they have passed the point of no return. + [Test] + public async Task The_row_does_not_exist_after_a_startup_check_refused() + { + // Seeded so the marker would be written if anything stamped it, which makes the refusal the only reason it is not. + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + SetSourceVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "NoSuchWindow"); + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + Assert.That(async () => await RunCommand.Run(Settings, AllowingAnUnreleasedMigration(), cancellation.Token), + Throws.Exception.With.Message.Contains("NoSuchWindow")); + + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "a refused startup has opened on nothing"); + } + + // Ingestion-only hosts write failed messages into the same database a migration targets, so a rollback + // gate that had not seen them would discard everything they ingested. + [Test] + public async Task An_ingestion_only_host_records_the_row_too() + { + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + + // The ingestion-only host runs a real transport, which the migration fixture does not otherwise configure. + Settings.MaximumConcurrencyLevel = 1; + + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints); + + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + await app.StartAsync(cancellation.Token); + + try + { + Assert.That(await ReadSetting(HostOpenedSetting), Is.Not.Null, "only RunCommand used to record this, so a scaled-out ingestion host wrote to the target and left no trace of having opened on it"); + } + finally + { + await app.StopAsync(cancellation.Token); + await app.DisposeAsync(); + } + } + + + // A customer running on SQL who has never migrated must not be told the way back is gone. + [Test] + public async Task A_database_no_migration_has_touched_is_not_stamped() + { + Settings.MaximumConcurrencyLevel = 1; + + using var cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(3)); + var app = ErrorIngestionOnlyCommand.BuildHost(Settings, AllowingAnUnreleasedMigration()); + + await app.StartAsync(cancellation.Token); + + try + { + Assert.That(await ReadSetting(HostOpenedSetting), Is.Null, "no checkpoint row exists, so no migration has ever run against this database"); + } + finally + { + await app.StopAsync(cancellation.Token); + await app.DisposeAsync(); + } + } + + async Task SeedRequiredCategoriesComplete(string except = null) + { + foreach (var category in MigrationCategoryRegistry.All.Where(category => category.Kind == MigrationCategoryKind.Required && category.Id != except)) + { + await SeedCheckpoint(category.Id); + } + } + + Task DropTheCheckpointTable() => + QueryTarget(dbContext => + { + var table = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!; + var name = dbContext.GetService().DelimitIdentifier(table.GetTableName()!, table.GetSchema()); + + // The name comes from the EF model and the provider delimits it, so no outside input reaches this SQL. +#pragma warning disable EF1003 + return dbContext.Database.ExecuteSqlRawAsync("DROP TABLE " + name); +#pragma warning restore EF1003 + }); +} diff --git a/src/ServiceControl.Migration.Tests/.editorconfig b/src/ServiceControl.Migration.Tests/.editorconfig new file mode 100644 index 0000000000..ca5ad8bd2e --- /dev/null +++ b/src/ServiceControl.Migration.Tests/.editorconfig @@ -0,0 +1,5 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs b/src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs new file mode 100644 index 0000000000..1bdc4ed89f --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationCategoryCoverageTests.cs @@ -0,0 +1,100 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationCategoryCoverageTests +{ + Settings settings; + + [SetUp] + public void SetUp() + { + // Both persistence configurations read ErrorRetentionPeriod through the settings reader rather + // than off the Settings object, so the constructor argument below does not satisfy either of them. + SetVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + // Registering the SQL Server persistence never connects, so any connection string satisfies it. + SetVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING", "Server=localhost;Database=ServiceControl;Trusted_Connection=True;TrustServerCertificate=True"); + SetVariable("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem"); + SetVariable("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", + Path.Combine(TestContext.CurrentContext.WorkDirectory, "Bodies", Guid.NewGuid().ToString("n"))); + + // SQL Server stands in for PostgreSQL too, because both register the same EFCoreMigrationTarget and its writer list does not depend on the provider. + settings = new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + } + + [TearDown] + public void TearDown() + { + foreach (var name in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + + variables.Clear(); + } + + // A reader with no writer, or the reverse, is a category the intersection would otherwise drop in silence. + [Test] + public async Task Every_reader_has_a_writer_and_the_reverse() + { + await using var source = PersistenceFactory.CreateMigrationSource(settings); + var sourceCategoryIds = source.SupportedCategoryIds; + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddPersistence(settings); + await using var provider = services.BuildServiceProvider(); + var target = provider.GetRequiredService(); + + var readerOnly = sourceCategoryIds.Except(target.SupportedCategoryIds).ToArray(); + var writerOnly = target.SupportedCategoryIds.Except(sourceCategoryIds).ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(readerOnly, Is.Empty, $"the reader claims a category the writer does not: {string.Join(", ", readerOnly)}"); + Assert.That(writerOnly, Is.Empty, $"the writer claims a category the reader does not: {string.Join(", ", writerOnly)}"); + } + } + + [Test] + public async Task Every_reader_and_its_writer_agree_on_the_document_type() + { + await using var source = PersistenceFactory.CreateMigrationSource(settings); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddPersistence(settings); + await using var provider = services.BuildServiceProvider(); + var target = provider.GetRequiredService(); + + var shared = source.DocumentTypes.Keys.Intersect(target.DocumentTypes.Keys).ToArray(); + Assert.That(shared, Is.Not.Empty, "the test proves nothing if no category has both a reader and a writer"); + + using (Assert.EnterMultipleScope()) + { + foreach (var categoryId in shared) + { + Assert.That(target.DocumentTypes[categoryId], Is.EqualTo(source.DocumentTypes[categoryId]), $"the {categoryId} reader yields a document type its writer does not take"); + } + } + } + + void SetVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + variables.Add(name); + } + + readonly List variables = []; +} diff --git a/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs new file mode 100644 index 0000000000..737a26bd4f --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.IO; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.ServiceControl.Hosting; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Hosting.Commands; + +[TestFixture] +[NonParallelizable] +class MigrationSourceReportCommandTests +{ + [SetUp] + public void SetUp() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", MigrationSourceServer.ServerUrl); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_DATABASENAME", MigrationSourceServer.PrimaryDatabase); + Environment.SetEnvironmentVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", MigrationSourceServer.ThroughputDatabase); + } + + [TearDown] + public void TearDown() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_DATABASENAME", null); + Environment.SetEnvironmentVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", null); + } + + [Test] + public async Task The_report_names_the_databases_the_settings_and_the_collections() + { + var report = await RunReport(); + + Assert.Multiple(() => + { + Assert.That(report, Does.Match(@"Mode\s+: external")); + Assert.That(report, Does.Contain(MigrationSourceServer.ServerUrl)); + Assert.That(report, Does.Contain("from ServiceControl/RavenDB/DatabaseName")); + Assert.That(report, Does.Contain("from LicensingComponent/RavenDB/ThroughputDatabaseName")); + Assert.That(report, Does.Match(@"EndpointSettings\s+1"), "The report has to render a count, not just name the collection."); + }); + } + + static async Task RunReport() + { + var settings = new Settings(persisterType: "RavenDB", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + + var writer = new StringWriter(); + var original = Console.Out; + Console.SetOut(writer); + + try + { + await new MigrationSourceReportCommand().Execute(new HostArguments([]), settings); + } + finally + { + Console.SetOut(original); + } + + return writer.ToString(); + } +} diff --git a/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs b/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs new file mode 100644 index 0000000000..9b6e933c03 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Persistence; +using ServiceControl.RavenDB; +using TestHelper; + +/// +/// One embedded RavenDB for the whole assembly, holding a seeded database that stands in for a +/// customer's old instance. EmbeddedServer.Instance is process wide, so there can only be one. +/// +static class MigrationSourceServer +{ + public static string ServerUrl { get; private set; } + + public static string PrimaryDatabase { get; private set; } + + public static string ThroughputDatabase { get; private set; } + + public static async Task Start(CancellationToken cancellationToken = default) + { + var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "MigrationSource", "Data"); + var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "MigrationSource", "Logs"); + var port = PortUtility.GetAssignedOrAvailablePort(33377); + + ServerUrl = $"http://localhost:{port}"; + PrimaryDatabase = "primary"; + ThroughputDatabase = "throughput"; + + var configuration = new EmbeddedDatabaseConfiguration(ServerUrl, PrimaryDatabase, dbPath, logPath, "Operations") { RunInMemory = true }; + database = EmbeddedDatabase.Start(configuration, lifetime); + + using var store = await database.Connect(cancellationToken); + + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(PrimaryDatabase)), cancellationToken); + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(ThroughputDatabase)), cancellationToken); + + using var seeder = new DocumentStore + { + Urls = [ServerUrl], + Database = PrimaryDatabase, + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }.Initialize(); + + using var session = seeder.OpenAsyncSession(); + await session.StoreAsync(new EndpointSettings { Name = "Sales", TrackInstances = true }, "EndpointSettings/1", cancellationToken); + await session.SaveChangesAsync(cancellationToken); + } + + public static async Task Stop() + { + if (database is null) + { + return; + } + + await database.Stop(CancellationToken.None); + database.Dispose(); + database = null; + lifetime.StopApplication(); + } + + static EmbeddedDatabase database; + static readonly TestLifetime lifetime = new(); + + sealed class TestLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => stopping.Token; + public CancellationToken ApplicationStopped => CancellationToken.None; + public void StopApplication() => stopping.Cancel(); + + readonly CancellationTokenSource stopping = new(); + } +} + +[SetUpFixture] +public class MigrationSourceServerFixture +{ + [OneTimeSetUp] + public Task StartSource() => MigrationSourceServer.Start(); + + [OneTimeTearDown] + public Task StopSource() => MigrationSourceServer.Stop(); +} diff --git a/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj new file mode 100644 index 0000000000..da4cda3938 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + Migration + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs new file mode 100644 index 0000000000..b246735641 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs @@ -0,0 +1,103 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Loader; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure; +using ServiceControl.Persistence; + +[TestFixture] +[NonParallelizable] +class TwoPersistersInOneProcessTests +{ + Settings settings; + string bodyStoragePath; + + [SetUp] + public void SetUp() + { + bodyStoragePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Bodies", Guid.NewGuid().ToString("n")); + + // Both persistence configurations read ErrorRetentionPeriod through the settings reader rather + // than off the Settings object, so the constructor argument below does not satisfy either of them. + SetVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + // Registering the SQL Server persistence never connects, so any connection string satisfies it. + SetVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING", + Environment.GetEnvironmentVariable("ServiceControl_Persistence_SqlServer_ConnectionString") + ?? "Server=localhost;Database=ServiceControl;Trusted_Connection=True;TrustServerCertificate=True"); + SetVariable("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem"); + SetVariable("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", bodyStoragePath); + SetVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", MigrationSourceServer.ServerUrl); + SetVariable("SERVICECONTROL_RAVENDB_DATABASENAME", MigrationSourceServer.PrimaryDatabase); + SetVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", MigrationSourceServer.ThroughputDatabase); + + settings = new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + } + + [TearDown] + public void TearDown() + { + foreach (var name in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + + variables.Clear(); + } + + [Test] + public async Task A_source_and_a_target_load_side_by_side_and_share_one_type_identity() + { + Assert.That(settings.AssemblyLoadContextResolver(typeof(Settings).Assembly.Location), Is.InstanceOf(), + "This test is meaningless if the resolver is pinned to AssemblyLoadContext.Default the way every acceptance test pins it."); + + var services = new ServiceCollection(); + services.AddPersistence(settings); + var targetSettings = settings.PersisterSpecificSettings; + + await using var source = await PersistenceFactory.OpenMigrationSource(settings); + + var sourceContext = AssemblyLoadContext.GetLoadContext(source.GetType().Assembly); + var targetContext = AssemblyLoadContext.GetLoadContext(settings.PersisterSpecificSettings.GetType().Assembly); + + Assert.Multiple(() => + { + Assert.That(sourceContext, Is.Not.SameAs(AssemblyLoadContext.Default), + "The RavenDB persister must load into its own plugin context, or this proves nothing."); + Assert.That(targetContext, Is.Not.SameAs(AssemblyLoadContext.Default), + "The target persister must be isolated too, otherwise only one half of the pairing is being tested."); + Assert.That(sourceContext, Is.Not.SameAs(targetContext), + "Source and target must land in separate contexts. That separation is the whole feature, and nothing else here asserts it."); + Assert.That(settings.PersisterSpecificSettings, Is.SameAs(targetSettings), + "Opening a source must leave the target's settings object exactly where it was."); + }); + + var description = await source.Describe(); + + Assert.That(description.Facts.Single(fact => fact.Label == "Primary database").Value, Is.EqualTo(MigrationSourceServer.PrimaryDatabase), + "The source read its own RavenDB settings rather than the target's."); + } + + [Test] + public void Asking_a_SQL_persister_for_maintenance_mode_is_still_refused() + { + var refusal = Assert.Throws(() => PersistenceFactory.Create(settings, maintenanceMode: true)); + + Assert.That(refusal.Message, Does.Contain("Maintenance mode is not supported").And.Contain("SQLServer"), + "PersistenceFactory's SupportsMaintenanceMode guard is what turns a persister mismatch into a sentence instead of a cast exception. A restructure that drops it compiles and passes every other test."); + } + + void SetVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + variables.Add(name); + } + + readonly List variables = []; +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs new file mode 100644 index 0000000000..f9b6c71345 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.Designer.cs @@ -0,0 +1,991 @@ +// +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Persistence.EFCore.PostgreSql; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + [DbContext(typeof(PostgreSqlServiceControlDbContext))] + [Migration("20261007000244_AddMigrationCheckpoints")] + partial class AddMigrationCheckpoints + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("request_id"); + + b.Property("ArchiveType") + .HasColumnType("integer") + .HasColumnName("archive_type"); + + b.Property("OperationType") + .HasColumnType("integer") + .HasColumnName("operation_type"); + + b.Property("CurrentBatch") + .HasColumnType("integer") + .HasColumnName("current_batch"); + + b.Property("GroupName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("group_name"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_name"); + + b.Property("NumberOfBatches") + .HasColumnType("integer") + .HasColumnName("number_of_batches"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property("Started") + .HasColumnType("timestamp with time zone") + .HasColumnName("started"); + + b.Property("TotalNumberOfMessages") + .HasColumnType("integer") + .HasColumnName("total_number_of_messages"); + + b.HasKey("RequestId", "ArchiveType", "OperationType") + .HasName("pk_archive_operations"); + + b.HasIndex("Started") + .HasDatabaseName("ix_archive_operations_started"); + + b.ToTable("archive_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Category") + .IsRequired() + .HasColumnType("text") + .HasColumnName("category"); + + b.Property("CustomCheckId") + .IsRequired() + .HasColumnType("text") + .HasColumnName("custom_check_id"); + + b.Property("FailureReason") + .HasColumnType("text") + .HasColumnName("failure_reason"); + + b.Property("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_host"); + + b.Property("OriginatingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("originating_endpoint_host_id"); + + b.Property("OriginatingEndpointName") + .IsRequired() + .HasColumnType("text") + .HasColumnName("originating_endpoint_name"); + + b.Property("ReportedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("reported_at"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_custom_checks"); + + b.HasIndex("ReportedAt") + .HasDatabaseName("ix_custom_checks_reported_at"); + + b.HasIndex("Status", "ReportedAt") + .HasDatabaseName("ix_custom_checks_status_reported_at"); + + b.ToTable("custom_checks", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.Property("TrackInstances") + .HasColumnType("boolean") + .HasColumnName("track_instances"); + + b.HasKey("Name") + .HasName("pk_endpoint_settings"); + + b.ToTable("endpoint_settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category"); + + b.Property("Description") + .IsRequired() + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("event_type"); + + b.Property("RaisedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("raised_at"); + + b.PrimitiveCollection>("RelatedTo") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("related_to"); + + b.Property("Severity") + .HasColumnType("integer") + .HasColumnName("severity"); + + b.HasKey("Id") + .HasName("pk_event_log_items"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending() + .HasDatabaseName("ix_event_log_items_raised_at_id"); + + b.ToTable("event_log_items", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("DispatchContextJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("dispatch_context_json"); + + b.Property("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("dispatch_context_type_name"); + + b.HasKey("Id") + .HasName("pk_external_integration_dispatch_requests"); + + b.ToTable("external_integration_dispatch_requests", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("Body") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("body"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("FailedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("failed_at"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("MessageId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_error_imports"); + + b.HasIndex("FailedAt") + .HasDatabaseName("ix_failed_error_imports_failed_at"); + + b.ToTable("failed_error_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("edit_id"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_edits"); + + b.HasIndex("EditId") + .HasDatabaseName("ix_failed_message_edits_edit_id"); + + b.ToTable("failed_message_edits", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("body_content_type"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyStoredExternally") + .HasColumnType("boolean") + .HasColumnName("body_stored_externally"); + + b.Property("BodyText") + .HasColumnType("text") + .HasColumnName("body_text"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("conversation_id"); + + b.Property("ExceptionMessage") + .HasColumnType("text") + .HasColumnName("exception_message"); + + b.Property("ExceptionType") + .HasColumnType("text") + .HasColumnName("exception_type"); + + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("failing_endpoint_address"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_time_of_failure"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("LastAttemptedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_attempted_at"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("LastTimeOfFailure") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_time_of_failure"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("message_type"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("integer") + .HasColumnName("number_of_processing_attempts"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_host"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("receiving_endpoint_host_id"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_host"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uuid") + .HasColumnName("sending_endpoint_host_id"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sending_endpoint_name"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("StatusChangedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("status_changed_at"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_messages"); + + b.HasIndex("ConversationId") + .HasDatabaseName("ix_failed_messages_conversation_id"); + + b.HasIndex("FailingEndpointAddress") + .HasDatabaseName("ix_failed_messages_failing_endpoint_address"); + + b.HasIndex("ReceivingEndpointName") + .HasDatabaseName("ix_failed_messages_receiving_endpoint_name"); + + b.HasIndex("StatusChangedAt") + .HasDatabaseName("ix_failed_messages_status_changed_at") + .HasFilter("status IN (2, 4)"); + + b.HasIndex("TimeSent") + .HasDatabaseName("ix_failed_messages_time_sent"); + + b.HasIndex("Status", "LastTimeOfFailure") + .HasDatabaseName("ix_failed_messages_status_last_time_of_failure"); + + b.HasIndex("Status", "LastModified", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_last_modified_unique_message_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified", "UniqueMessageId"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "MessageType", "UniqueMessageId") + .HasDatabaseName("ix_failed_messages_status_message_type_unique_message_id"); + + b.ToTable("failed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uuid") + .HasColumnName("failed_message_unique_id"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Title") + .IsRequired() + .HasColumnType("text") + .HasColumnName("title"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("type"); + + b.HasKey("FailedMessageUniqueId", "GroupId") + .HasName("pk_failed_message_groups"); + + b.HasIndex("GroupId") + .HasDatabaseName("ix_failed_message_groups_group_id"); + + b.HasIndex("Type", "GroupId") + .HasDatabaseName("ix_failed_message_groups_type_group_id"); + + NpgsqlIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "FailedMessageUniqueId", "Title" }); + + b.ToTable("failed_message_groups", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uuid") + .HasColumnName("unique_message_id"); + + b.Property("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.Property("StageAttempts") + .HasColumnType("integer") + .HasColumnName("stage_attempts"); + + b.HasKey("UniqueMessageId") + .HasName("pk_failed_message_retries"); + + b.HasIndex("RetryBatchId") + .HasDatabaseName("ix_failed_message_retries_retry_batch_id"); + + b.ToTable("failed_message_retries", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("group_id"); + + b.Property("Comment") + .IsRequired() + .HasColumnType("text") + .HasColumnName("comment"); + + b.HasKey("GroupId") + .HasName("pk_group_comments"); + + b.ToTable("group_comments", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("Id") + .HasName("pk_historic_retry_operations"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending() + .HasDatabaseName("ix_historic_retry_operations_completion_time_id"); + + b.ToTable("historic_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("Monitored") + .HasColumnType("boolean") + .HasColumnName("monitored"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("name"); + + b.HasKey("Id") + .HasName("pk_known_endpoints"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.PrimitiveCollection>("EndpointIndicators") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("endpoint_indicators"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("name"); + + b.Property("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("normalized_sanitized_name"); + + b.Property("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("sanitized_name"); + + b.Property("Scope") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("scope"); + + b.Property("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("user_indicator"); + + b.HasKey("NormalizedName", "ThroughputSource") + .HasName("pk_licensing_endpoints"); + + b.HasIndex("NormalizedSanitizedName") + .HasDatabaseName("ix_licensing_endpoints_normalized_sanitized_name"); + + b.ToTable("licensing_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("character varying(300)") + .HasColumnName("normalized_name"); + + b.Property("ThroughputSource") + .HasColumnType("integer") + .HasColumnName("throughput_source"); + + b.Property("DateUtc") + .HasColumnType("date") + .HasColumnName("date_utc"); + + b.Property("MessageCount") + .HasColumnType("bigint") + .HasColumnName("message_count"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc") + .HasName("pk_licensing_endpoint_throughput"); + + b.HasIndex("DateUtc") + .HasDatabaseName("ix_licensing_endpoint_throughput_date_utc"); + + b.ToTable("licensing_endpoint_throughput", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("from_physical_address"); + + b.Property("LastModified") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_modified"); + + b.Property("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("to_physical_address"); + + b.HasKey("FromPhysicalAddress") + .HasName("pk_message_redirects"); + + b.ToTable("message_redirects", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property("CategoryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category_id"); + + b.Property("AlreadyPresentCount") + .HasColumnType("bigint") + .HasColumnName("already_present_count"); + + b.Property("CopiedCount") + .HasColumnType("bigint") + .HasColumnName("copied_count"); + + b.Property("Cursor") + .HasColumnType("text") + .HasColumnName("cursor"); + + b.Property("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property("LastProgressAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_progress_at"); + + b.Property("SettledAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("settled_at"); + + b.Property("SkipReasons") + .HasColumnType("text") + .HasColumnName("skip_reasons"); + + b.Property("SkippedCount") + .HasColumnType("bigint") + .HasColumnName("skipped_count"); + + b.Property("SourceTotal") + .HasColumnType("bigint") + .HasColumnName("source_total"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("StartedWindowSeconds") + .HasColumnType("bigint") + .HasColumnName("started_window_seconds"); + + b.Property("State") + .HasColumnType("integer") + .HasColumnName("state"); + + b.Property("Version") + .IsConcurrencyToken() + .HasColumnType("bigint") + .HasColumnName("version"); + + b.HasKey("CategoryId") + .HasName("pk_migration_checkpoints"); + + b.ToTable("migration_checkpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Classifier") + .HasColumnType("text") + .HasColumnName("classifier"); + + b.Property("Context") + .HasColumnType("text") + .HasColumnName("context"); + + b.Property("InitialBatchSize") + .HasColumnType("integer") + .HasColumnName("initial_batch_size"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("initiated_by_id"); + + b.Property("InitiatedByName") + .HasColumnType("text") + .HasColumnName("initiated_by_name"); + + b.Property("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("operation_id"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("request_id"); + + b.Property("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("retry_session_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("StagingId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("staging_id"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id") + .HasName("pk_retry_batches"); + + b.HasIndex("Status", "RetrySessionId") + .HasDatabaseName("ix_retry_batches_status_retry_session_id"); + + b.ToTable("retry_batches", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property("Id") + .HasColumnType("integer") + .HasColumnName("id"); + + b.Property("RetryBatchId") + .HasColumnType("uuid") + .HasColumnName("retry_batch_id"); + + b.HasKey("Id") + .HasName("pk_retry_batch_now_forwarding"); + + b.ToTable("retry_batch_now_forwarding", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property("Key") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("key"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text") + .HasColumnName("value"); + + b.HasKey("Key") + .HasName("pk_settings"); + + b.ToTable("settings", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_type"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("transport_address"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("endpoint"); + + b.HasKey("MessageType", "TransportAddress") + .HasName("pk_subscriptions"); + + b.ToTable("subscriptions", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(400) + .HasColumnType("character varying(400)") + .HasColumnName("request_id"); + + b.Property("RetryType") + .HasColumnType("integer") + .HasColumnName("retry_type"); + + b.Property("Classifier") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("classifier"); + + b.Property("CompletionTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("completion_time"); + + b.Property("Failed") + .HasColumnType("boolean") + .HasColumnName("failed"); + + b.Property("Last") + .HasColumnType("timestamp with time zone") + .HasColumnName("last"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("integer") + .HasColumnName("number_of_messages_processed"); + + b.Property("Originator") + .HasColumnType("text") + .HasColumnName("originator"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.HasKey("RequestId", "RetryType") + .HasName("pk_unacknowledged_retry_operations"); + + b.ToTable("unacknowledged_retry_operations", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_failed_message_groups_failed_messages_failed_message_unique"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_licensing_endpoint_throughput_licensing_endpoints_normalize"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs new file mode 100644 index 0000000000..6eaf796b17 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/20261007000244_AddMigrationCheckpoints.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.PostgreSql.Migrations +{ + /// + public partial class AddMigrationCheckpoints : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "migration_checkpoints", + columns: table => new + { + category_id = table.Column(type: "character varying(450)", maxLength: 450, nullable: false), + state = table.Column(type: "integer", nullable: false), + cursor = table.Column(type: "text", nullable: true), + copied_count = table.Column(type: "bigint", nullable: false), + skipped_count = table.Column(type: "bigint", nullable: false), + source_total = table.Column(type: "bigint", nullable: true), + skip_reasons = table.Column(type: "text", nullable: true), + started_at = table.Column(type: "timestamp with time zone", nullable: true), + last_progress_at = table.Column(type: "timestamp with time zone", nullable: true), + settled_at = table.Column(type: "timestamp with time zone", nullable: true), + last_error = table.Column(type: "text", nullable: true), + already_present_count = table.Column(type: "bigint", nullable: false), + version = table.Column(type: "bigint", nullable: false), + started_window_seconds = table.Column(type: "bigint", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_migration_checkpoints", x => x.category_id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "migration_checkpoints"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs index 06b1a9b639..bdfd8aebaa 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/Migrations/PostgreSqlServiceControlDbContextModelSnapshot.cs @@ -715,6 +715,72 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("message_redirects", (string)null); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property("CategoryId") + .HasMaxLength(450) + .HasColumnType("character varying(450)") + .HasColumnName("category_id"); + + b.Property("AlreadyPresentCount") + .HasColumnType("bigint") + .HasColumnName("already_present_count"); + + b.Property("CopiedCount") + .HasColumnType("bigint") + .HasColumnName("copied_count"); + + b.Property("Cursor") + .HasColumnType("text") + .HasColumnName("cursor"); + + b.Property("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property("LastProgressAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_progress_at"); + + b.Property("SettledAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("settled_at"); + + b.Property("SkipReasons") + .HasColumnType("text") + .HasColumnName("skip_reasons"); + + b.Property("SkippedCount") + .HasColumnType("bigint") + .HasColumnName("skipped_count"); + + b.Property("SourceTotal") + .HasColumnType("bigint") + .HasColumnName("source_total"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("StartedWindowSeconds") + .HasColumnType("bigint") + .HasColumnName("started_window_seconds"); + + b.Property("State") + .HasColumnType("integer") + .HasColumnName("state"); + + b.Property("Version") + .IsConcurrencyToken() + .HasColumnType("bigint") + .HasColumnName("version"); + + b.HasKey("CategoryId") + .HasName("pk_migration_checkpoints"); + + b.ToTable("migration_checkpoints", (string)null); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => { b.Property("Id") diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs new file mode 100644 index 0000000000..fdac0674c0 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlMigrationSqlDialect.cs @@ -0,0 +1,43 @@ +namespace ServiceControl.Persistence.EFCore.PostgreSql; + +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Infrastructure; + +/// +/// The migration's PostgreSQL statements. An insert that does nothing on a conflicting key is all it takes to +/// add only what is absent, and the keys it inserted come back from the same statement. +/// +class PostgreSqlMigrationSqlDialect : PostgreSqlDialect, IMigrationSqlDialect +{ + // Deterministic collations compare keys byte for byte, so there is no schema fact to read and none to name in the statement. + public Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public IEqualityComparer KeyComparer(Type entityType, string propertyName) => StringComparer.Ordinal; + + // A fixed chunk, because PostgreSQL's parameter limit is far away and the row width does not bring it closer. + public int RowsPerStatement(int parametersPerRow) => MaxRowsPerStatement; + + public async Task> InsertMissing(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) where TEntity : class + { + var insert = MigrationInsert.For(dbContext); + var keyColumns = string.Join(", ", insert.KeyColumns); + var inserted = new List(rows.Count); + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + inserted.AddRange(await insert.Execute( + dbContext, + $""" + INSERT INTO {insert.Table} ({string.Join(", ", insert.Columns)}) + VALUES + {ParameterRows(chunk.Length, insert.Columns.Count)} + ON CONFLICT ({keyColumns}) DO NOTHING + RETURNING {keyColumns} + """, + chunk, + cancellationToken)); + } + + return inserted; + } +} diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs index 3cbe6c1549..898a4a6bd4 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs @@ -18,6 +18,7 @@ public void AddPersistence(IServiceCollection services) RegisterDataStores(services, settings); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs new file mode 100644 index 0000000000..2f862bcb80 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.Designer.cs @@ -0,0 +1,794 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Persistence.EFCore.SqlServer; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerServiceControlDbContext))] + [Migration("20261007000240_AddMigrationCheckpoints")] + partial class AddMigrationCheckpoints + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.11") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ArchiveOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ArchiveType") + .HasColumnType("int"); + + b.Property("OperationType") + .HasColumnType("int"); + + b.Property("CurrentBatch") + .HasColumnType("int"); + + b.Property("GroupName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("InitiatedByName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("NumberOfBatches") + .HasColumnType("int"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Started") + .HasColumnType("datetime2"); + + b.Property("TotalNumberOfMessages") + .HasColumnType("int"); + + b.HasKey("RequestId", "ArchiveType", "OperationType"); + + b.HasIndex("Started"); + + b.ToTable("ArchiveOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.CustomCheckEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Category") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("CustomCheckId") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailureReason") + .HasColumnType("nvarchar(max)"); + + b.Property("OriginatingEndpointHost") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OriginatingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("OriginatingEndpointName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("ReportedAt") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("ReportedAt"); + + b.HasIndex("Status", "ReportedAt"); + + b.ToTable("CustomChecks"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EndpointSettingsEntity", b => + { + b.Property("Name") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("TrackInstances") + .HasColumnType("bit"); + + b.HasKey("Name"); + + b.ToTable("EndpointSettings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.EventLogItemEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Description") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RaisedAt") + .HasColumnType("datetime2"); + + b.PrimitiveCollection("RelatedTo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Severity") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("RaisedAt", "Id") + .IsDescending(); + + b.ToTable("EventLogItems"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.ExternalIntegrationDispatchRequestEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("DispatchContextJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DispatchContextTypeName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("ExternalIntegrationDispatchRequests"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedErrorImportEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("Body") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("ExceptionInfo") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FailedAt") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("MessageId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("FailedAt"); + + b.ToTable("FailedErrorImports"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEditEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("EditId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("EditId"); + + b.ToTable("FailedMessageEdits"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("BodyContentType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyStoredExternally") + .HasColumnType("bit"); + + b.Property("BodyText") + .HasColumnType("nvarchar(max)"); + + b.Property("ConversationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ExceptionMessage") + .HasColumnType("nvarchar(max)"); + + b.Property("ExceptionType") + .HasColumnType("nvarchar(max)"); + + b.Property("FailingEndpointAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("FirstTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("LastAttemptedAt") + .HasColumnType("datetime2"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("LastTimeOfFailure") + .HasColumnType("datetime2"); + + b.Property("MessageId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("MessageType") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("NumberOfProcessingAttempts") + .HasColumnType("int"); + + b.Property("ReceivingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("ReceivingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHost") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SendingEndpointHostId") + .HasColumnType("uniqueidentifier"); + + b.Property("SendingEndpointName") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("StatusChangedAt") + .HasColumnType("datetime2"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("ConversationId"); + + b.HasIndex("FailingEndpointAddress"); + + b.HasIndex("ReceivingEndpointName"); + + b.HasIndex("StatusChangedAt") + .HasFilter("[Status] IN (2, 4)"); + + b.HasIndex("TimeSent"); + + b.HasIndex("Status", "LastModified"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Status", "LastModified"), new[] { "FirstTimeOfFailure", "LastTimeOfFailure" }); + + b.HasIndex("Status", "LastTimeOfFailure"); + + b.HasIndex("Status", "MessageType", "UniqueMessageId"); + + b.ToTable("FailedMessages"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.Property("FailedMessageUniqueId") + .HasColumnType("uniqueidentifier"); + + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Title") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("FailedMessageUniqueId", "GroupId"); + + b.HasIndex("GroupId"); + + b.HasIndex("Type", "GroupId"); + + SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("Type", "GroupId"), new[] { "Title" }); + + b.ToTable("FailedMessageGroups"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageRetryEntity", b => + { + b.Property("UniqueMessageId") + .HasColumnType("uniqueidentifier"); + + b.Property("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.Property("StageAttempts") + .HasColumnType("int"); + + b.HasKey("UniqueMessageId"); + + b.HasIndex("RetryBatchId"); + + b.ToTable("FailedMessageRetries"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.GroupCommentEntity", b => + { + b.Property("GroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Comment") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("GroupId"); + + b.ToTable("GroupComments"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.HistoricRetryOperationEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CompletionTime") + .HasColumnType("datetime2"); + + b.Property("Failed") + .HasColumnType("bit"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("CompletionTime", "Id") + .IsDescending(); + + b.ToTable("HistoricRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("Monitored") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.ToTable("KnownEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("ThroughputSource") + .HasColumnType("int"); + + b.PrimitiveCollection("EndpointIndicators") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("NormalizedSanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("SanitizedName") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Scope") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("UserIndicator") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("NormalizedName", "ThroughputSource"); + + b.HasIndex("NormalizedSanitizedName"); + + b.ToTable("LicensingEndpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.Property("NormalizedName") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("ThroughputSource") + .HasColumnType("int"); + + b.Property("DateUtc") + .HasColumnType("date"); + + b.Property("MessageCount") + .HasColumnType("bigint"); + + b.HasKey("NormalizedName", "ThroughputSource", "DateUtc"); + + b.HasIndex("DateUtc"); + + b.ToTable("LicensingEndpointThroughput"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MessageRedirectEntity", b => + { + b.Property("FromPhysicalAddress") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("LastModified") + .HasColumnType("datetime2"); + + b.Property("ToPhysicalAddress") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("FromPhysicalAddress"); + + b.ToTable("MessageRedirects"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property("CategoryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("AlreadyPresentCount") + .HasColumnType("bigint"); + + b.Property("CopiedCount") + .HasColumnType("bigint"); + + b.Property("Cursor") + .HasColumnType("nvarchar(max)"); + + b.Property("LastError") + .HasColumnType("nvarchar(max)"); + + b.Property("LastProgressAt") + .HasColumnType("datetime2"); + + b.Property("SettledAt") + .HasColumnType("datetime2"); + + b.Property("SkipReasons") + .HasColumnType("nvarchar(max)"); + + b.Property("SkippedCount") + .HasColumnType("bigint"); + + b.Property("SourceTotal") + .HasColumnType("bigint"); + + b.Property("StartedAt") + .HasColumnType("datetime2"); + + b.Property("StartedWindowSeconds") + .HasColumnType("bigint"); + + b.Property("State") + .HasColumnType("int"); + + b.Property("Version") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.HasKey("CategoryId"); + + b.ToTable("MigrationCheckpoints"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Classifier") + .HasColumnType("nvarchar(max)"); + + b.Property("Context") + .HasColumnType("nvarchar(max)"); + + b.Property("InitialBatchSize") + .HasColumnType("int"); + + b.Property("InitiatedById") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("InitiatedByName") + .HasColumnType("nvarchar(max)"); + + b.Property("Last") + .HasColumnType("datetime2"); + + b.Property("OperationId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("RequestId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RetrySessionId") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("StagingId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("Status", "RetrySessionId"); + + b.ToTable("RetryBatches"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchNowForwardingEntity", b => + { + b.Property("Id") + .HasColumnType("int"); + + b.Property("RetryBatchId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("Id"); + + b.ToTable("RetryBatchNowForwarding"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SettingEntity", b => + { + b.Property("Key") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("Value") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.SubscriptionEntity", b => + { + b.Property("MessageType") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("TransportAddress") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.HasKey("MessageType", "TransportAddress"); + + b.ToTable("Subscriptions"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.UnacknowledgedRetryOperationEntity", b => + { + b.Property("RequestId") + .HasMaxLength(400) + .HasColumnType("nvarchar(400)"); + + b.Property("RetryType") + .HasColumnType("int"); + + b.Property("Classifier") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("CompletionTime") + .HasColumnType("datetime2"); + + b.Property("Failed") + .HasColumnType("bit"); + + b.Property("Last") + .HasColumnType("datetime2"); + + b.Property("NumberOfMessagesProcessed") + .HasColumnType("int"); + + b.Property("Originator") + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.HasKey("RequestId", "RetryType"); + + b.ToTable("UnacknowledgedRetryOperations"); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.FailedMessageGroupEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.FailedMessageEntity", null) + .WithMany() + .HasForeignKey("FailedMessageUniqueId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointThroughputEntity", b => + { + b.HasOne("ServiceControl.Persistence.EFCore.Entities.LicensingEndpointEntity", null) + .WithMany() + .HasForeignKey("NormalizedName", "ThroughputSource") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs new file mode 100644 index 0000000000..c3dab88c81 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/20261007000240_AddMigrationCheckpoints.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Persistence.EFCore.SqlServer.Migrations +{ + /// + public partial class AddMigrationCheckpoints : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "MigrationCheckpoints", + columns: table => new + { + CategoryId = table.Column(type: "nvarchar(450)", maxLength: 450, nullable: false), + State = table.Column(type: "int", nullable: false), + Cursor = table.Column(type: "nvarchar(max)", nullable: true), + CopiedCount = table.Column(type: "bigint", nullable: false), + SkippedCount = table.Column(type: "bigint", nullable: false), + SourceTotal = table.Column(type: "bigint", nullable: true), + SkipReasons = table.Column(type: "nvarchar(max)", nullable: true), + StartedAt = table.Column(type: "datetime2", nullable: true), + LastProgressAt = table.Column(type: "datetime2", nullable: true), + SettledAt = table.Column(type: "datetime2", nullable: true), + LastError = table.Column(type: "nvarchar(max)", nullable: true), + AlreadyPresentCount = table.Column(type: "bigint", nullable: false), + Version = table.Column(type: "bigint", nullable: false), + StartedWindowSeconds = table.Column(type: "bigint", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_MigrationCheckpoints", x => x.CategoryId); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "MigrationCheckpoints"); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs index b9e41a3681..42b194da3a 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/Migrations/SqlServerServiceControlDbContextModelSnapshot.cs @@ -572,6 +572,57 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("MessageRedirects"); }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.MigrationCheckpointEntity", b => + { + b.Property("CategoryId") + .HasMaxLength(450) + .HasColumnType("nvarchar(450)"); + + b.Property("AlreadyPresentCount") + .HasColumnType("bigint"); + + b.Property("CopiedCount") + .HasColumnType("bigint"); + + b.Property("Cursor") + .HasColumnType("nvarchar(max)"); + + b.Property("LastError") + .HasColumnType("nvarchar(max)"); + + b.Property("LastProgressAt") + .HasColumnType("datetime2"); + + b.Property("SettledAt") + .HasColumnType("datetime2"); + + b.Property("SkipReasons") + .HasColumnType("nvarchar(max)"); + + b.Property("SkippedCount") + .HasColumnType("bigint"); + + b.Property("SourceTotal") + .HasColumnType("bigint"); + + b.Property("StartedAt") + .HasColumnType("datetime2"); + + b.Property("StartedWindowSeconds") + .HasColumnType("bigint"); + + b.Property("State") + .HasColumnType("int"); + + b.Property("Version") + .IsConcurrencyToken() + .HasColumnType("bigint"); + + b.HasKey("CategoryId"); + + b.ToTable("MigrationCheckpoints"); + }); + modelBuilder.Entity("ServiceControl.Persistence.EFCore.Entities.RetryBatchEntity", b => { b.Property("Id") diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs index 5835c1f39c..acae4e5f55 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerDialect.cs @@ -69,6 +69,11 @@ protected static string ParameterRows(int rowCount, int columnCount) return sql.ToString(); } + // The same rows as ParameterRows, each ending with its position in the batch, so a statement can keep the first of two duplicate keys. + protected static string ParameterRowsWithOrdinal(int rowCount, int columnCount) => + string.Join(",\n", Enumerable.Range(0, rowCount).Select(row => + $"({string.Join(", ", Enumerable.Range(0, columnCount).Select(column => $"@p{(row * columnCount) + column}"))}, {row})")); + protected static int MaxRowsPerStatement(int columns) => MaxParametersPerStatement / columns; // SQL Server's ceiling is 2100 (https://learn.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server). diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs new file mode 100644 index 0000000000..45d8ea8454 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerMigrationSqlDialect.cs @@ -0,0 +1,114 @@ +namespace ServiceControl.Persistence.EFCore.SqlServer; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Infrastructure; + +/// +/// The migration's SQL Server statements. Two keys that differ only in case can be one row here, because a +/// column's collation decides what counts as equal, so the statements compare keys under the collation the +/// schema gives them and the batch is de-duplicated the same way before it is offered to the table. +/// +class SqlServerMigrationSqlDialect : SqlServerDialect, IMigrationSqlDialect +{ + const int IgnoreCaseStyle = 1; + + OpenedKeys? OpenedState { get; set; } + + public async Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) + { + var sql = dbContext.GetService(); + var collations = new Dictionary<(string Table, string Column), string>(); + var comparers = new Dictionary<(Type EntityType, string Property), IEqualityComparer>(); + + foreach (var entityType in dbContext.Model.GetEntityTypes()) + { + if (entityType.GetTableName() is not { } tableName || entityType.FindPrimaryKey() is not { } key) + { + continue; + } + + var storeObject = StoreObjectIdentifier.Table(tableName, entityType.GetSchema()); + var table = sql.DelimitIdentifier(tableName, entityType.GetSchema()); + + foreach (var property in key.Properties.Where(property => property.ClrType == typeof(string))) + { + // A key property with no column of its own is refused by name when MigrationInsert builds the statement. + if (property.GetColumnName(storeObject) is not { } column) + { + continue; + } + + var collation = await dbContext.Database + .SqlQuery($""" + SELECT c.collation_name AS [Name], + CONVERT(int, COLLATIONPROPERTY(c.collation_name, 'ComparisonStyle')) AS [ComparisonStyle] + FROM sys.columns AS c + WHERE c.object_id = OBJECT_ID({table}) AND c.name = {column} AND c.collation_name IS NOT NULL + """) + .SingleOrDefaultAsync(cancellationToken) + ?? throw new InvalidOperationException($"The migration target found no collation for {tableName}.{column}, so the SQL Server schema is missing or not current. Run ServiceControl with --setup against this database first."); + + collations[(table, sql.DelimitIdentifier(column))] = collation.Name; + comparers[(entityType.ClrType, property.Name)] = (collation.ComparisonStyle & IgnoreCaseStyle) == 0 ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase; + } + } + + OpenedState = new OpenedKeys(collations, comparers); + } + + public IEqualityComparer KeyComparer(Type entityType, string propertyName) => + Keys.Comparers.TryGetValue((entityType, propertyName), out var comparer) + ? comparer + : throw new ArgumentException($"{entityType.Name}.{propertyName} is not a string primary-key column, so it has no collation to compare under."); + + public int RowsPerStatement(int parametersPerRow) => MaxRowsPerStatement(parametersPerRow); + + public async Task> InsertMissing(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) where TEntity : class + { + var insert = MigrationInsert.For(dbContext); + var columns = string.Join(", ", insert.Columns); + // The collation is an identifier sys.columns returned, and COLLATE takes no parameter. + var partitionBy = string.Join(", ", insert.KeyColumns.Select(column => + Keys.Collations.TryGetValue((insert.Table, column), out var collation) ? $"{column} COLLATE {collation}" : column)); + var inserted = new List(rows.Count); + + foreach (var chunk in rows.Chunk(RowsPerStatement(insert.Columns.Count))) + { + // HOLDLOCK, or two writers can both find a key missing and both insert it. + inserted.AddRange(await insert.Execute( + dbContext, + $""" + MERGE {insert.Table} WITH (HOLDLOCK) AS t + USING ( + SELECT {columns} + FROM ( + SELECT {columns}, ROW_NUMBER() OVER (PARTITION BY {partitionBy} ORDER BY [MigrationOrdinal]) AS [MigrationDuplicate] + FROM (VALUES + {ParameterRowsWithOrdinal(chunk.Length, insert.Columns.Count)} + ) AS v ({columns}, [MigrationOrdinal]) + ) AS deduplicated + WHERE [MigrationDuplicate] = 1 + ) AS s ({columns}) + ON {string.Join(" AND ", insert.KeyColumns.Select(column => $"t.{column} = s.{column}"))} + WHEN NOT MATCHED THEN INSERT ({columns}) VALUES ({string.Join(", ", insert.Columns.Select(column => $"s.{column}"))}) + OUTPUT {string.Join(", ", insert.KeyColumns.Select(column => $"inserted.{column}"))}; + """, + chunk, + cancellationToken)); + } + + return inserted; + } + + OpenedKeys Keys => OpenedState ?? throw new InvalidOperationException($"The SQL Server migration dialect is not open. Call {nameof(Open)} first."); + + sealed record OpenedKeys( + IReadOnlyDictionary<(string Table, string Column), string> Collations, + IReadOnlyDictionary<(Type EntityType, string Property), IEqualityComparer> Comparers); + + sealed record KeyColumnCollation(string Name, int ComparisonStyle); +} diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs index dead62d3cc..b508cfd7d1 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs @@ -18,6 +18,7 @@ public void AddPersistence(IServiceCollection services) RegisterDataStores(services, settings); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs index 4b76e5add7..46cb05d67e 100644 --- a/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs +++ b/src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs @@ -7,12 +7,14 @@ namespace ServiceControl.Persistence.EFCore.Abstractions; using Particular.LicensingComponent.Persistence; using ServiceControl.CustomChecks; using ServiceControl.Operations.BodyStorage; +using ServiceControl.Persistence.EFCore.DataMigration; using ServiceControl.Persistence.EFCore.Implementation; using ServiceControl.Persistence.EFCore.Implementation.BodyStorage; using ServiceControl.Persistence.EFCore.Implementation.Recoverability; using ServiceControl.Persistence.EFCore.Implementation.UnitOfWork; using ServiceControl.Persistence.EFCore.Infrastructure; using ServiceControl.Persistence.EFCore.Infrastructure.Metrics; +using ServiceControl.Persistence.DataMigration; using ServiceControl.Persistence.MessageRedirects; using ServiceControl.Persistence.Recoverability; using ServiceControl.Persistence.UnitOfWork; @@ -38,6 +40,10 @@ protected static void RegisterDataStores(IServiceCollection services, EFPersiste services.AddSingleton(p => p.GetRequiredService()); services.AddHostedService(p => p.GetRequiredService()); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + if (settings.RunRetentionSweep) { services.AddSingleton(); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs new file mode 100644 index 0000000000..5a5c116bdf --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/BodyStorageIsWritableCheck.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Text; +using Infrastructure; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes, reads and deletes one probe body, so a body store that is misconfigured stops the copy before it +/// starts. Bodies can live on a file share, in Azure Blob Storage or in S3, and without this the first category +/// that carries bodies would find out part way through and skip every message it could not write. +/// +sealed class BodyStorageIsWritableCheck(IBodyStoragePersistence bodyStorage) : IMigrationStartupCheck +{ + const string ProbeBodyId = "migration-writable-probe"; + + public string Name => "message body storage is writable"; + + public async Task Run(CancellationToken cancellationToken = default) + { + await bodyStorage.WriteBody(ProbeBodyId, Encoding.UTF8.GetBytes(ProbeBodyId), "text/plain", cancellationToken); + + try + { + var probe = await bodyStorage.ReadBody(ProbeBodyId, cancellationToken) + ?? throw new Exception($"Message body storage accepted the probe body '{ProbeBodyId}' and then did not return it."); + + await probe.Stream.DisposeAsync(); + } + finally + { + // The probe is not migrated data, so it goes even when the read fails: a probe left behind is a body the source never had. + await bodyStorage.DeleteBodyIfExists(ProbeBodyId, cancellationToken); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs new file mode 100644 index 0000000000..09510292ff --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTarget.cs @@ -0,0 +1,113 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using System.Collections.Frozen; +using Implementation; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.DataMigration; +using Writers; + +/// +/// Writes a migration into SQL Server or PostgreSQL. It holds one writer per category and knows nothing about +/// any of them beyond that, so a category this build cannot write is simply absent from +/// and never reaches the engine. +/// +sealed class EFCoreMigrationTarget(IServiceScopeFactory scopeFactory, IMigrationSqlDialect migrationDialect, ILogger logger) : DataStoreBase(scopeFactory), IMigrationTarget +{ + readonly FrozenDictionary writers = new IMigrationCategoryWriter[] + { + new KnownEndpointsWriter(migrationDialect), + new EndpointSettingsWriter(migrationDialect) + }.ToFrozenDictionary(writer => writer.CategoryId, StringComparer.Ordinal); + + public Task Open(CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => migrationDialect.Open(dbContext, token), cancellationToken); + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, _) => Task.FromResult(WriterFor(category).BatchSize(dbContext)), cancellationToken); + + public Task Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, writeToken) => + { + var prepared = await WriterFor(category).Prepare(dbContext, batch, writeToken); + + AccountForEveryRow(category, batch, prepared); + + var skipReasons = prepared.Skips.Count == 0 ? null : prepared.Skips.GroupBy(skip => skip.Reason).ToDictionary(group => group.Key, group => group.LongCount()); + + var (copied, alreadyPresent, saved) = await dbContext.Database.CreateExecutionStrategy().ExecuteAsync(async token => + { + // This block is retried, and an attempt that failed leaves the checkpoint it added still tracked, so the next attempt would insert it a second time. + dbContext.ChangeTracker.Clear(); + + await using var transaction = await dbContext.Database.BeginTransactionAsync(token); + + var inserted = await prepared.Insert(dbContext, token); + var present = AlreadyPresentIn(category, batch, inserted, prepared.Skips.Count); + var stored = await dbContext.UpsertCheckpoint( + checkpointToExtend.Extend(inserted, prepared.Skips.Count, present, skipReasons), + token); + await transaction.CommitAsync(token); + + return (inserted, present, stored); + }, writeToken); + + foreach (var (sourceId, reason, detail) in prepared.Skips) + { + logger.LogWarning("Skipped {SourceId} in category {CategoryId} as {SkipReason}: {Detail}", sourceId, category.Id, reason, detail); + } + + foreach (var (sourceId, detail) in prepared.Merges) + { + logger.LogWarning("Merged {SourceId} in category {CategoryId} into a row with the same key: {Detail}", sourceId, category.Id, detail); + } + + return new MigrationWriteResult( + saved, + copied, + prepared.Skips.Count, + [.. prepared.Skips.Select(skip => skip.SourceId)], + AlreadyPresent: alreadyPresent, + SkipReasons: skipReasons); + }, cancellationToken); + + // Checked before the insert, because the subtraction below can only catch a writer that over-reports. + internal static void AccountForEveryRow(MigrationCategory category, MigrationBatch batch, PreparedBatch prepared) + { + if (prepared.PreparedRowCount + prepared.Skips.Count != batch.Rows.Count) + { + throw new InvalidOperationException($"The {category.Id} writer prepared {prepared.PreparedRowCount} rows and skipped {prepared.Skips.Count} of the {batch.Rows.Count} rows in the batch. Every row must be one or the other, or the rows it dropped would be counted as rows the target already held."); + } + } + + // Without the throw, a miscount would quietly shrink the halt threshold's denominator instead of failing. + internal static int AlreadyPresentIn(MigrationCategory category, MigrationBatch batch, int copied, int skipped) + { + var alreadyPresent = batch.Rows.Count - copied - skipped; + + if (alreadyPresent < 0) + { + throw new InvalidOperationException($"The target copied {copied} and skipped {skipped} of the {batch.Rows.Count} rows in category {category.Id}, leaving {alreadyPresent} already present. Every row is copied, skipped or already present."); + } + + return alreadyPresent; + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => WriterFor(category).Count(dbContext, token), cancellationToken); + + public IReadOnlyCollection SupportedCategoryIds => writers.Keys; + + public IReadOnlyDictionary DocumentTypes => writers.ToFrozenDictionary(pair => pair.Key, pair => pair.Value.DocumentType, StringComparer.Ordinal); + + IMigrationCategoryWriter WriterFor(MigrationCategory category) => + writers.TryGetValue(category.Id, out var writer) + ? writer + : throw new NotSupportedException($"The migration target cannot yet write the '{category.Id}' category."); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs new file mode 100644 index 0000000000..9988718ea8 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/EFCoreMigrationTargetReadiness.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Implementation; +using Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +/// +/// What a SQL Server or PostgreSQL database has to be before a copy writes to it, and the stamp saying a host +/// has opened on it since. The stamp is a row in the settings table, so it survives a restart and belongs to +/// the database rather than to the instance. +/// +public class EFCoreMigrationTargetReadiness( + IServiceScopeFactory scopeFactory, + IBodyStoragePersistence bodyStorage, + TimeProvider timeProvider) : DataStoreBase(scopeFactory), IMigrationTargetReadiness +{ + // Cheapest first: the schema costs one query, and the body store costs a round trip to a file share or a + // cloud service. + public IReadOnlyList ContributedChecks() => + [ + new SchemaIsCurrentCheck(scopeFactory), + new TargetHoldsNoServiceControlDataCheck(scopeFactory), + new BodyStorageIsWritableCheck(bodyStorage) + ]; + + public Task RecordHostOpened(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + if (await dbContext.GetSetting(SettingKeys.MigrationHostOpenedOnTarget, token) is null) + { + await dbContext.StoreSetting(SettingKeys.MigrationHostOpenedOnTarget, timeProvider.GetUtcNow().UtcDateTime, token); + } + }, cancellationToken); + + public Task HasHostOpened(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + await dbContext.GetSetting(SettingKeys.MigrationHostOpenedOnTarget, token) is not null, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs new file mode 100644 index 0000000000..3349a45a38 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/IMigrationCategoryWriter.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using ServiceControl.Persistence.DataMigration; + +/// +/// Everything the target needs to know about one category: how big a batch is, how its documents +/// become rows, and how many rows the target holds for it. +/// +interface IMigrationCategoryWriter +{ + /// The category this writer handles, named as in . + string CategoryId { get; } + + /// + /// The type of takes. A row holding any other type + /// fails the batch with an . + /// + Type DocumentType { get; } + + /// + /// The most rows the target will take from the source in one batch. It comes from how many values each row + /// carries and how many parameters one statement can hold, so a wider table takes fewer rows. The context is read for its model only. + /// + int BatchSize(ServiceControlDbContext dbContext); + + /// + /// Turns one batch of source documents into the rows to insert and the rows to skip, writing nothing: the insert it returns runs later, inside the target's transaction. + /// Every batch row produces exactly one row or exactly one skip, never both and never neither, because the target works out how many were already there by subtracting both from the batch size. + /// + Task Prepare(ServiceControlDbContext dbContext, MigrationBatch batch, CancellationToken cancellationToken = default); + + /// + /// How many rows the target holds for this category. It counts only this category, even where two of them + /// share a table. + /// + Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs new file mode 100644 index 0000000000..4550acc0bb --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/MigrationCategoryWriter.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using ServiceControl.Persistence.DataMigration; + +/// +/// A writer that takes rows carrying a . It casts every row once, before +/// sees it, so a row from a reader that yields another type fails the batch by name. +/// +abstract class MigrationCategoryWriter : IMigrationCategoryWriter where TDocument : class +{ + public abstract string CategoryId { get; } + + public Type DocumentType => typeof(TDocument); + + public abstract int BatchSize(ServiceControlDbContext dbContext); + + public abstract Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); + + public Task Prepare(ServiceControlDbContext dbContext, MigrationBatch batch, CancellationToken cancellationToken = default) => + PrepareDocuments(dbContext, [.. batch.Rows.Select(row => (row, DocumentOf(row)))], cancellationToken); + + /// + /// Does what promises, for rows already cast. + /// + /// The context to read the target through. Nothing may be written through it here. + /// Every row in the batch, in source order, each with its document. + /// Cancels any read of the target. + /// One prepared row or one skip for every entry in . + protected abstract Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, TDocument Document)> documents, CancellationToken cancellationToken = default); + + TDocument DocumentOf(MigrationRow row) => + row.Document as TDocument + ?? throw new InvalidCastException($"The {CategoryId} writer takes {typeof(TDocument).FullName} documents, but row {row.SourceId} holds {row.Document?.GetType().FullName ?? "no document"}. The reader and writer for one category must agree on the document type."); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs new file mode 100644 index 0000000000..c4e3faebb6 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/PreparedBatch.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using ServiceControl.Persistence.DataMigration; + +/// +/// One batch turned into rows, ready for the target to insert inside its own transaction. Nothing here has +/// touched the database yet. +/// +/// Runs the insert and returns how many rows it added. The target calls it inside the transaction that also saves the checkpoint. +/// How many rows the writer built, which is more than the insert adds when the table already holds some of their keys. +/// One entry per row the writer will not insert, with the reason and a detail line for the log. +/// One entry per prepared row whose key the target treats as the same key as another row's, either earlier in the batch or already in the table, so the insert keeps the other row and drops this one. Each carries a detail line naming both keys and the one kept, which the target logs as a warning after the commit. The dropped row is counted as already present. +sealed record PreparedBatch( + Func> Insert, + int PreparedRowCount, + IReadOnlyList<(string SourceId, MigrationSkipReason Reason, string Detail)> Skips, + IReadOnlyList<(string SourceId, string Detail)> Merges); diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs new file mode 100644 index 0000000000..24f64d1de2 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/SchemaIsCurrentCheck.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using Implementation; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses a copy into a database whose schema is behind this build. The writers insert into tables by name, so +/// a missing schema migration turns into a failure part way through the first category instead of a refusal. +/// +sealed class SchemaIsCurrentCheck(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IMigrationStartupCheck +{ + public string Name => "the target schema is current"; + + public Task Run(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + string[] pending = [.. await dbContext.Database.GetPendingMigrationsAsync(token)]; + + if (pending.Length > 0) + { + throw new Exception($"The database is missing {pending.Length} schema migration(s): {string.Join(", ", pending)}. Run ServiceControl with --setup before starting a migration."); + } + }, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs new file mode 100644 index 0000000000..b545a8bea6 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/TargetHoldsNoServiceControlDataCheck.cs @@ -0,0 +1,74 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration; + +using DbContexts; +using Implementation; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses a copy into a database that already holds ServiceControl data. A copy into a database that ran on SQL +/// mixes the customer's rows with rows SQL already held, and a retry of a category could then delete rows the copy +/// never wrote. The check stands down once a row that is not known to be optional exists, because that means a copy +/// has started and what the tables hold is what it wrote. +/// +sealed class TargetHoldsNoServiceControlDataCheck(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IMigrationStartupCheck +{ + public string Name => "the migration target holds no ServiceControl data"; + + // One entry per mapped entity but the checkpoint: its CLR type, and a query asking whether its table holds a row. + internal static readonly IReadOnlyList<(Type Entity, Func> HasRows)> Tables = + [ + Entry(dbContext => dbContext.CustomChecks), + Entry(dbContext => dbContext.EndpointSettings), + Entry(dbContext => dbContext.KnownEndpoints), + Entry(dbContext => dbContext.FailedMessages), + Entry(dbContext => dbContext.FailedMessageGroups), + Entry(dbContext => dbContext.GroupComments), + Entry(dbContext => dbContext.MessageRedirects), + Entry(dbContext => dbContext.RetryBatches), + Entry(dbContext => dbContext.RetryBatchNowForwarding), + Entry(dbContext => dbContext.FailedMessageRetries), + Entry(dbContext => dbContext.FailedErrorImports), + Entry(dbContext => dbContext.Settings), + Entry(dbContext => dbContext.Subscriptions), + Entry(dbContext => dbContext.EventLogItems), + Entry(dbContext => dbContext.HistoricRetryOperations), + Entry(dbContext => dbContext.UnacknowledgedRetryOperations), + Entry(dbContext => dbContext.ArchiveOperations), + Entry(dbContext => dbContext.FailedMessageEdits), + Entry(dbContext => dbContext.LicensingEndpoints), + Entry(dbContext => dbContext.LicensingEndpointThroughput), + Entry(dbContext => dbContext.ExternalIntegrationDispatchRequests) + ]; + + // Both halves of an entry come from one DbSet, so the type a test compares and the table the query reads cannot differ. + static (Type Entity, Func> HasRows) Entry(Func> set) where T : class => + (typeof(T), (dbContext, cancellationToken) => set(dbContext).AnyAsync(cancellationToken)); + + public Task Run(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var checkpointed = await dbContext.MigrationCheckpoints.Select(checkpoint => checkpoint.CategoryId).ToListAsync(token); + + if (checkpointed.Any(id => MigrationCategoryRegistry.Find(id)?.Kind != MigrationCategoryKind.Optional)) + { + return; + } + + var holdingRows = new List(); + + foreach (var (entity, hasRows) in Tables) + { + if (await hasRows(dbContext, token)) + { + holdingRows.Add(dbContext.Model.FindEntityType(entity)!.GetTableName()!); + } + } + + if (holdingRows.Count > 0) + { + throw new Exception($"The database already holds ServiceControl data in {string.Join(", ", holdingRows)}, so ServiceControl has already run on it, and a migration copies only into an empty database. Create a new database, run ServiceControl with --setup against it, and do not start ServiceControl on it before migrating; then start again with migration on."); + } + }, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs new file mode 100644 index 0000000000..485518ca67 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/EndpointSettingsWriter.cs @@ -0,0 +1,68 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes every per-endpoint setting as stored. A setting for an endpoint the target does not know is copied too, +/// and the heartbeat settings sync removes it by its own rule once the instance opens. On a target whose name +/// column ignores case, two names that differ only in case are one row, and the writer reports each setting the +/// insert drops as a merge. +/// +sealed class EndpointSettingsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.EndpointSettings; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatement(MigrationInsert.For(dbContext).Columns.Count); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.EndpointSettings.LongCountAsync(cancellationToken); + + protected override async Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, EndpointSettings Document)> documents, CancellationToken cancellationToken = default) + { + List rows = [.. documents.Select(document => new EndpointSettingsEntity { Name = document.Document.Name, TrackInstances = document.Document.TrackInstances })]; + List rowSourceIds = [.. documents.Select(document => document.Row.SourceId)]; + + // The source yields document-id order, and the statement keeps the first of any keys the database treats as one. + return new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, + rows.Count, + [], + await MergesIn(dbContext, rows, rowSourceIds, cancellationToken)); + } + + // Mirrors the insert's de-duplication, so an operator can see which settings a case-insensitive name column dropped. + async Task> MergesIn(ServiceControlDbContext dbContext, List rows, List rowSourceIds, CancellationToken cancellationToken) + { + var sameName = migrationDialect.KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)); + string[] rowNames = [.. rows.Select(row => row.Name)]; + + // The database compares under the column's collation, so this also returns a stored name that differs only in case. + var keptNames = await dbContext.EndpointSettings.AsNoTracking() + .Select(stored => stored.Name) + .Where(name => rowNames.Contains(name)) + .ToListAsync(cancellationToken); + + var merges = new List<(string SourceId, string Detail)>(); + + for (var index = 0; index < rows.Count; index++) + { + var name = rows[index].Name; + var keptName = keptNames.FirstOrDefault(kept => sameName.Equals(kept, name)); + + if (keptName is null) + { + keptNames.Add(name); + } + else if (!string.Equals(keptName, name, StringComparison.Ordinal)) + { + merges.Add((rowSourceIds[index], $"the name column treats '{name}' and '{keptName}' as one endpoint, so it kept '{keptName}' and dropped the settings of '{name}'")); + } + } + + return merges; + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs new file mode 100644 index 0000000000..a0ec9ff796 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/DataMigration/Writers/KnownEndpointsWriter.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.Persistence.EFCore.DataMigration.Writers; + +using DbContexts; +using Entities; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +/// +/// Writes the endpoints ServiceControl has heard from. The row's key is worked out from the endpoint's own +/// details rather than carried over, so the same endpoint lands on the same row whichever side wrote it first. +/// +sealed class KnownEndpointsWriter(IMigrationSqlDialect migrationDialect) : MigrationCategoryWriter +{ + public override string CategoryId => MigrationCategoryIds.KnownEndpoints; + + public override int BatchSize(ServiceControlDbContext dbContext) => migrationDialect.RowsPerStatement(MigrationInsert.For(dbContext).Columns.Count); + + public override Task Count(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default) => + dbContext.KnownEndpoints.LongCountAsync(cancellationToken); + + protected override Task PrepareDocuments(ServiceControlDbContext dbContext, IReadOnlyList<(MigrationRow Row, KnownEndpoint Document)> documents, CancellationToken cancellationToken = default) + { + var rows = new List(documents.Count); + var skips = new List<(string SourceId, MigrationSkipReason Reason, string Detail)>(); + + foreach (var (row, endpoint) in documents) + { + if (endpoint.EndpointDetails?.Name is null || endpoint.EndpointDetails.Host is null) + { + var column = endpoint.EndpointDetails?.Name is null ? nameof(KnownEndpointEntity.Name) : nameof(KnownEndpointEntity.Host); + skips.Add((row.SourceId, MigrationSkipReason.RequiredValueMissing, $"KnownEndpoints.{column} is NOT NULL and the document has no value for it")); + continue; + } + + rows.Add(new KnownEndpointEntity + { + Id = endpoint.EndpointDetails.GetDeterministicId(), + Name = endpoint.EndpointDetails.Name, + HostId = endpoint.EndpointDetails.HostId, + Host = endpoint.EndpointDetails.Host, + Monitored = endpoint.Monitored + }); + } + + return Task.FromResult(new PreparedBatch( + async (context, token) => (await migrationDialect.InsertMissing(context, rows, token)).Count, + rows.Count, + skips, + Merges: [])); + } +} diff --git a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs index 3717c6a7f3..3884ce1e95 100644 --- a/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs +++ b/src/ServiceControl.Persistence.EFCore/DbContexts/ServiceControlDbContext.cs @@ -34,6 +34,7 @@ public abstract class ServiceControlDbContext(DbContextOptions options) : DbCont public DbSet LicensingEndpoints { get; set; } public DbSet LicensingEndpointThroughput { get; set; } public DbSet ExternalIntegrationDispatchRequests { get; set; } + public DbSet MigrationCheckpoints { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) => optionsBuilder.EnableDetailedErrors(); @@ -68,6 +69,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.ApplyConfiguration(new LicensingEndpointConfiguration()); modelBuilder.ApplyConfiguration(new LicensingEndpointThroughputConfiguration()); modelBuilder.ApplyConfiguration(new ExternalIntegrationDispatchRequestConfiguration()); + modelBuilder.ApplyConfiguration(new MigrationCheckpointConfiguration()); } public abstract bool IsDuplicateKeyException(DbUpdateException exception); diff --git a/src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs b/src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs new file mode 100644 index 0000000000..2f43e79e78 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Entities/MigrationCheckpointEntity.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.EFCore.Entities; + +using ServiceControl.Persistence.DataMigration; + +public class MigrationCheckpointEntity +{ + public required string CategoryId { get; set; } + public MigrationCategoryState State { get; set; } + public string? Cursor { get; set; } + public long CopiedCount { get; set; } + public long SkippedCount { get; set; } + public long? SourceTotal { get; set; } + public IReadOnlyDictionary? SkipReasons { get; set; } + public DateTime? StartedAt { get; set; } + public DateTime? LastProgressAt { get; set; } + public DateTime? SettledAt { get; set; } + public string? LastError { get; set; } + public long AlreadyPresentCount { get; set; } + public long Version { get; set; } + public long? StartedWindowSeconds { get; set; } + + internal MigrationCheckpoint ToCheckpoint() => new( + CategoryId, State, Cursor, + CopiedCount, SkippedCount, SourceTotal, SkipReasons, + StartedAt, LastProgressAt, SettledAt, LastError, + AlreadyPresentCount, Version, StartedWindowSeconds); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs new file mode 100644 index 0000000000..9cc35d77f5 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/EntityConfigurations/MigrationCheckpointConfiguration.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.EFCore.EntityConfigurations; + +using System.Text.Json; +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.ChangeTracking; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using ServiceControl.Persistence.DataMigration; + +class MigrationCheckpointConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(e => e.CategoryId); + builder.Property(e => e.CategoryId).HasMaxLength(ColumnLengths.ShortTextLength).ValueGeneratedNever(); + builder.Property(e => e.Version).IsConcurrencyToken(); + + // Null never reaches a converter: EF Core stores NULL for it, so a checkpoint with no skips has no JSON. + builder.Property(e => e.SkipReasons).HasConversion( + reasons => ToJson(reasons!), + json => FromJson(json), + new ValueComparer>( + (left, right) => left!.Count == right!.Count && !left.Except(right).Any(), + reasons => reasons.Aggregate(0, (hash, entry) => HashCode.Combine(hash, entry.Key, entry.Value)), + reasons => reasons.ToDictionary(entry => entry.Key, entry => entry.Value))); + } + + // Keyed by name, so a stored count keeps its meaning whatever order the enum's members are in. + static string ToJson(IReadOnlyDictionary reasons) => + JsonSerializer.Serialize(reasons.ToDictionary(entry => entry.Key.ToString(), entry => entry.Value)); + + // A name this build has no member for buckets into Unknown rather than throwing, because ReadAll is what + // decides whether the host may start; grouped because several unknown names collapse onto the one member. + static IReadOnlyDictionary FromJson(string json) => + JsonSerializer.Deserialize>(json)! + .GroupBy(entry => Enum.TryParse(entry.Key, out var reason) && Enum.IsDefined(reason) ? reason : MigrationSkipReason.Unknown) + .ToDictionary(group => group.Key, group => group.Sum(entry => entry.Value)); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs b/src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs new file mode 100644 index 0000000000..3f343f44e3 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Implementation/EFMigrationCheckpointStore.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.EFCore.Implementation; + +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Persistence.DataMigration; + +// Every method opens its own scope and context through DataStoreBase, so none of them can join a +// caller's transaction. Inside one, call ServiceControlDbContext.UpsertCheckpoint instead. +public class EFMigrationCheckpointStore(IServiceScopeFactory scopeFactory) : DataStoreBase(scopeFactory), IMigrationCheckpointStore +{ + public Task> ReadAll(CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var entities = await dbContext.MigrationCheckpoints.AsNoTracking().ToListAsync(token); + return (IReadOnlyList)[.. entities.Select(entity => entity.ToCheckpoint())]; + }, cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => + ExecuteWithDbContext(async (dbContext, token) => + { + var entity = await dbContext.MigrationCheckpoints.AsNoTracking().SingleOrDefaultAsync(e => e.CategoryId == categoryId, token); + return entity?.ToCheckpoint(); + }, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + ExecuteWithDbContext((dbContext, token) => dbContext.UpsertCheckpoint(checkpoint, token), cancellationToken); +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs new file mode 100644 index 0000000000..864562c443 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IMigrationSqlDialect.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using ServiceControl.Persistence.EFCore.DbContexts; + +/// +/// The migration's provider-specific SQL. Statements run inside the caller's transaction and insert only what is absent. +/// +public interface IMigrationSqlDialect +{ + /// + /// Reads, once, whatever the statements need from the schema. On SQL Server that is the collation of every + /// string column of every primary key, composite keys included. + /// + Task Open(ServiceControlDbContext dbContext, CancellationToken cancellationToken = default); + + /// + /// How the database compares one string key column, so a caller can work out which of its own rows the insert will treat as one. The insert compares in the database itself; this only predicts what it will find. + /// Call first, and name a property that is a string column of the entity's primary key. + /// + IEqualityComparer KeyComparer(Type entityType, string propertyName); + + /// + /// How many rows the provider will take in one statement, for rows carrying this many values each. + /// + int RowsPerStatement(int parametersPerRow); + + /// + /// Inserts the rows whose key the table does not hold, merges any whose keys the database treats as one, and returns the rows it inserted. + /// + Task> InsertMissing(ServiceControlDbContext dbContext, IReadOnlyList rows, CancellationToken cancellationToken = default) where TEntity : class; +} diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs new file mode 100644 index 0000000000..37ac476786 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/MigrationInsert.cs @@ -0,0 +1,141 @@ +namespace ServiceControl.Persistence.EFCore.Infrastructure; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage; +using ServiceControl.Persistence.EFCore.DbContexts; + +/// +/// The table, columns and primary key maps to, and the run of one provider's insert statement over a chunk of those rows. +/// +public sealed class MigrationInsert where TEntity : class +{ + static readonly ExactKey KeyEquality = new(); + + readonly IReadOnlyList properties; + readonly IReadOnlyList keyProperties; + + MigrationInsert(string table, IReadOnlyList properties, IReadOnlyList keyProperties, Func columnName) + { + Table = table; + this.properties = properties; + this.keyProperties = keyProperties; + Columns = [.. properties.Select(columnName)]; + KeyColumns = [.. keyProperties.Select(columnName)]; + } + + /// The table name, quoted for this provider and carrying its schema. + public string Table { get; } + + /// Every column the insert writes, quoted, in the order the parameters of one row follow. + public IReadOnlyList Columns { get; } + + /// The primary key columns, quoted, in the order the statement has to return them. + public IReadOnlyList KeyColumns { get; } + + /// + /// Reads the table, columns and key out of the EF Core model for this entity. + /// + /// The entity is not mapped, has no primary key, or has a column this insert cannot write, such as one the database generates. Rows like that go through EF in the caller's context instead. + public static MigrationInsert For(ServiceControlDbContext dbContext) + { + var entityType = dbContext.Model.FindEntityType(typeof(TEntity)) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} is not an entity in the EF Core model, so the migration cannot tell which table it lands in."); + var tableName = entityType.GetTableName() + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} is not mapped to a table."); + var key = entityType.FindPrimaryKey() + ?? throw new InvalidOperationException($"{typeof(TEntity).Name} has no primary key, so there is nothing for a row to be absent by."); + IProperty[] properties = [.. entityType.GetProperties()]; + + if (properties.Any(property => property.ValueGenerated != ValueGenerated.Never) + || entityType.GetComplexProperties().Any() + || entityType.GetNavigations().Any(navigation => navigation.TargetEntityType.IsOwned())) + { + throw new InvalidOperationException($"{typeof(TEntity).Name} has a store-generated, complex or owned property, whose columns InsertMissing does not write. Add these rows through EF in the caller's context instead."); + } + + var storeObject = StoreObjectIdentifier.Table(tableName, entityType.GetSchema()); + var sql = dbContext.GetService(); + + return new MigrationInsert( + sql.DelimitIdentifier(tableName, entityType.GetSchema()), + properties, + key.Properties, + property => sql.DelimitIdentifier(property.GetColumnName(storeObject) + ?? throw new InvalidOperationException($"{typeof(TEntity).Name}.{property.Name} has no column in {tableName}."))); + } + + /// + /// Runs one insert statement over a chunk of rows and returns the rows of that chunk it inserted. + /// The statement reads parameters named @p0 upward, row after row in order, and must return the of each row it inserted, in that order. + /// It runs on the caller's open transaction, and throws when there is none or when a returned key matches no row of the chunk. + /// + public async Task> Execute(ServiceControlDbContext dbContext, string sql, TEntity[] chunk, CancellationToken cancellationToken = default) + { + await using var command = dbContext.Database.GetDbConnection().CreateCommand(); + command.Transaction = (dbContext.Database.CurrentTransaction + ?? throw new InvalidOperationException("InsertMissing must run inside the caller's transaction, so its rows commit with the checkpoint.")).GetDbTransaction(); + command.CommandText = sql; + // A raw command starts at the provider's own 30 seconds, where SaveChangesAsync took the configured value. + command.CommandTimeout = dbContext.Database.GetCommandTimeout() ?? command.CommandTimeout; + + var index = 0; + foreach (var row in chunk) + { + foreach (var property in properties) + { + // The mapping applies the value converter and the provider type, as EF's own inserts do. + command.Parameters.Add(property.GetRelationalTypeMapping().CreateParameter(command, $"@p{index++}", property.GetGetter().GetClrValue(row), property.IsNullable)); + } + } + + var rowsByKey = new Dictionary(KeyEquality); + foreach (var row in chunk) + { + // A key repeated exactly is one row to every database, so the first row given carries it. + rowsByKey.TryAdd(KeyOf(row), row); + } + + var inserted = new List(chunk.Length); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + object?[] key = [.. keyProperties.Select((property, column) => FromProvider(property, reader.GetValue(column)))]; + + // A key can come back as a different CLR type than it went in as, a date column read back as + // DateTime say, and a plain lookup would report that as a bare KeyNotFoundException. + inserted.Add(rowsByKey.TryGetValue(key, out var row) + ? row + : throw new InvalidOperationException($"The {Table} insert returned the key [{string.Join(", ", key.Select(Describe))}], which matches no row in the batch that produced it. Compare it against the key types the batch carried: [{string.Join(", ", keyProperties.Select(property => property.ClrType.Name))}].")); + } + + return inserted; + } + + object?[] KeyOf(TEntity row) => [.. keyProperties.Select(property => property.GetGetter().GetClrValue(row))]; + + static string Describe(object? value) => value is null ? "null" : $"{value} ({value.GetType().Name})"; + + static object? FromProvider(IProperty property, object value) => + property.GetRelationalTypeMapping().Converter is { } converter ? converter.ConvertFromProvider(value) : value; + + // Exact, because the database decided which keys were one before the statement ran, and OUTPUT and RETURNING echo the value inserted. + sealed class ExactKey : IEqualityComparer + { + public bool Equals(object?[]? x, object?[]? y) => x!.SequenceEqual(y!); + + public int GetHashCode(object?[] key) + { + var hash = new HashCode(); + + foreach (var value in key) + { + hash.Add(value); + } + + return hash.ToHashCode(); + } + } +} diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs index 7a8e403170..7da4cc4bfe 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/SettingKeys.cs @@ -9,4 +9,7 @@ static class SettingKeys public const string ReportMasks = "ReportMasks"; public const string LicensedEndpointDetails = "LicensedEndpointDetails"; public const string NotificationEmails = "NotificationEmails"; + // Written the first time a host starts on a database a migration has already written to. Once it is set, + // going back to RavenDB loses everything ServiceControl has written here since. + public const string MigrationHostOpenedOnTarget = "Migration/HostOpenedOnTarget"; } diff --git a/src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs b/src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs new file mode 100644 index 0000000000..db12c9b895 --- /dev/null +++ b/src/ServiceControl.Persistence.EFCore/MigrationCheckpointExtensions.cs @@ -0,0 +1,96 @@ +namespace ServiceControl.Persistence.EFCore; + +using DbContexts; +using Entities; +using Microsoft.EntityFrameworkCore; +using ServiceControl.Persistence.DataMigration; + +public static class MigrationCheckpointExtensions +{ + /// + /// Saves a migration checkpoint on the caller's own context, so it commits with that context's transaction + /// and flushes whatever else that context is tracking, and returns it carrying the version the save landed on. + /// Throws when the stored row no longer holds + /// . + /// + public static async Task UpsertCheckpoint(this ServiceControlDbContext dbContext, MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + try + { + return await dbContext.UpsertCheckpointCore(checkpoint, cancellationToken); + } + catch (DbUpdateConcurrencyException e) + { + throw new MigrationCheckpointConflictException( + $"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, which the stored row no longer holds. Another writer advanced it, or a retried save is meeting its own committed write after a lost acknowledgement.", e); + } + } + + // Deliberately not UpsertAsync: its duplicate-key fallback reloads and saves again, which inside a + // caller-opened PostgreSQL transaction runs after the failed insert aborted it and fails with 25P02. + static async Task UpsertCheckpointCore(this ServiceControlDbContext dbContext, MigrationCheckpoint checkpoint, CancellationToken cancellationToken) + { + var entity = await dbContext.FindAsync([checkpoint.CategoryId], cancellationToken: cancellationToken); + + if (entity is null) + { + if (checkpoint.Version != 0) + { + throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but no row exists for it."); + } + + var inserted = Insert(checkpoint); + dbContext.Add(inserted); + + try + { + await dbContext.SaveChangesAsync(cancellationToken); + } + catch (DbUpdateException e) when (dbContext.IsDuplicateKeyException(e)) + { + // Another writer created this category's first checkpoint between the read and the insert. Not + // retried: PostgreSQL has already aborted the caller's transaction, so a retry cannot run. + throw new MigrationCheckpointConflictException( + $"Checkpoint {checkpoint.CategoryId} was saved as a first write, but another writer had already created it.", e); + } + + return inserted.ToCheckpoint(); + } + + // The expected version goes in the UPDATE's WHERE clause, so a save from an out-of-date copy matches no row. + dbContext.Entry(entity).Property(e => e.Version).OriginalValue = checkpoint.Version; + entity.Version = checkpoint.Version + 1; + Apply(checkpoint, entity); + + await dbContext.SaveChangesAsync(cancellationToken); + + return entity.ToCheckpoint(); + } + + static MigrationCheckpointEntity Insert(MigrationCheckpoint checkpoint) + { + var entity = new MigrationCheckpointEntity { CategoryId = checkpoint.CategoryId, Version = 1 }; + + Apply(checkpoint, entity); + + return entity; + } + + // One place both branches copy the row through, so a column added to the checkpoint cannot be written on + // an update and forgotten on an insert. + static void Apply(MigrationCheckpoint checkpoint, MigrationCheckpointEntity entity) + { + entity.State = checkpoint.State; + entity.Cursor = checkpoint.Cursor; + entity.CopiedCount = checkpoint.CopiedCount; + entity.SkippedCount = checkpoint.SkippedCount; + entity.SourceTotal = checkpoint.SourceTotal; + entity.SkipReasons = checkpoint.SkipReasons; + entity.StartedAt = checkpoint.StartedAt; + entity.LastProgressAt = checkpoint.LastProgressAt; + entity.SettledAt = checkpoint.SettledAt; + entity.LastError = checkpoint.LastError; + entity.AlreadyPresentCount = checkpoint.AlreadyPresentCount; + entity.StartedWindowSeconds = checkpoint.StartedWindowSeconds; + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs new file mode 100644 index 0000000000..8c05a27f97 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/IMigrationCategoryReader.cs @@ -0,0 +1,25 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// Which documents one category is made of, and how they become rows. Read keeps the contract +/// sets out, for this one category. +/// +interface IMigrationCategoryReader +{ + /// The category this reader handles, named as in . + string CategoryId { get; } + + /// + /// The type of in every row yields. + /// + Type DocumentType { get; } + + IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs new file mode 100644 index 0000000000..590e534e18 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/MigrationCategoryReader.cs @@ -0,0 +1,40 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// A reader whose every row carries a , which is the type its category's writer +/// must take. Reading through ties the documents RavenDB streams to that type. +/// +abstract class MigrationCategoryReader(RavenReadOnlySourceLifecycle lifecycle) : IMigrationCategoryReader where TDocument : class +{ + /// + /// The source connection, for a reader that streams or opens sessions itself instead of using . + /// + protected RavenReadOnlySourceLifecycle Lifecycle { get; } = lifecycle; + + public abstract string CategoryId { get; } + + public Type DocumentType => typeof(TDocument); + + public abstract IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default); + + /// + /// Streams the primary database's documents whose id starts with , each one whole as a row. + /// + protected IAsyncEnumerable WholeDocuments(string prefix, string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + RavenDocumentStream.ByPrefix( + Lifecycle, + CategoryId, + Lifecycle.Settings.DatabaseName, + prefix, + resumeAfter, + batchSize, + RavenDocumentStream.WholeDocument, + cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs new file mode 100644 index 0000000000..892641c1f5 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenDocumentStream.cs @@ -0,0 +1,82 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Runtime.CompilerServices; +using System.Threading; +using Raven.Client.Documents.Commands; +using ServiceControl.Persistence.DataMigration; + +/// +/// The one way every category reads RavenDB: stream the documents whose id starts with a prefix, in id order, +/// and hand them back in batches. Id order is what makes the cursor work, because a resume asks RavenDB to +/// start after an id rather than to skip a count. +/// +static class RavenDocumentStream +{ + /// + /// Streams one collection and yields it in batches of at most . + /// + /// The document id prefix that selects the collection, such as "KnownEndpoints/". + /// The cursor a previous run saved, or null to start at the beginning. + /// Turns one document into a row, or into null for a document this category does not copy. The cursor still moves past a document that becomes null. + /// The source holds no document with the id in , which means the cursor and the database no longer belong together. + public static async IAsyncEnumerable ByPrefix( + RavenReadOnlySourceLifecycle lifecycle, + string categoryId, + string databaseName, + string prefix, + string? resumeAfter, + int batchSize, + Func, MigrationRow?> project, + [EnumeratorCancellation] CancellationToken cancellationToken = default) where TDocument : class + { + using var session = lifecycle.OpenSession(databaseName); + + // RavenDB starts a stream after whatever id it is given, even one that does not exist, so a stale cursor would skip rows nothing has copied. + if (resumeAfter is not null && !await session.Advanced.ExistsAsync(resumeAfter, cancellationToken)) + { + throw new InvalidOperationException( + $"The migration cannot resume the '{categoryId}' category after '{resumeAfter}', because the RavenDB database '{databaseName}' holds no document with that id. The cursor is saved in the target database and names a source document, so restoring RavenDB from a backup or changing which database it reads separates the two. Point the migration back at the RavenDB database this copy started from and restart."); + } + + await using var enumerator = await session.Advanced.StreamAsync( + prefix, startAfter: resumeAfter, token: cancellationToken); + + var rows = new List(batchSize); + var cursor = resumeAfter; + var lastYielded = resumeAfter; + + while (await enumerator.MoveNextAsync()) + { + // The cursor moves on every document the stream hands back, even one that becomes no row, so a resume never walks it again. + cursor = enumerator.Current.Id; + + if (project(enumerator.Current) is { } row) + { + rows.Add(row); + } + + if (rows.Count == batchSize) + { + yield return new MigrationBatch(rows, cursor); + rows = new List(batchSize); + lastYielded = cursor; + } + } + + // A tail whose documents all became no row still goes out, empty, so that the cursor past them is saved. + if (rows.Count > 0 || cursor != lastYielded) + { + yield return new MigrationBatch(rows, cursor!); + } + } + + public static MigrationRow WholeDocument(StreamResult result) where TDocument : class => + new(result.Id, result.Document, NoMetadata); + + static readonly IReadOnlyDictionary NoMetadata = ReadOnlyDictionary.Empty; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs new file mode 100644 index 0000000000..8cb44a7cf7 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -0,0 +1,98 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents.Operations; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +/// +/// Reads a migration out of RavenDB. It holds one reader per category and knows nothing about any of them +/// beyond that, so a category this build cannot read is simply absent from +/// and never reaches the engine. +/// +sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMigrationSource +{ + readonly FrozenDictionary readers = new IMigrationCategoryReader[] + { + new KnownEndpointsReader(lifecycle), + new EndpointSettingsReader(lifecycle) + }.ToFrozenDictionary(reader => reader.CategoryId, StringComparer.Ordinal); + + public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); + + public IReadOnlyList ContributedChecks() => []; + + public async Task Describe(CancellationToken cancellationToken = default) + { + var settings = lifecycle.Settings; + var build = await lifecycle.DocumentStore.Maintenance.Server.SendAsync(new GetBuildNumberOperation(), cancellationToken); + + var server = settings.UseEmbeddedServer + ? new MigrationSourceFact("Server", settings.ServerUrl, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.DatabaseMaintenancePortKey}") + : new MigrationSourceFact("Server", settings.ConnectionString, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.ConnectionStringKey}"); + + return new MigrationSourceDescription(build.ProductVersion, + [ + new MigrationSourceFact("Mode", settings.UseEmbeddedServer ? "embedded" : "external"), + server, + new MigrationSourceFact("Primary database", settings.DatabaseName, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.DatabaseNameKey}"), + new MigrationSourceFact("Throughput database", settings.ThroughputDatabaseName, $"{ThroughputSettings.SettingsNamespace}/{ThroughputSettings.DatabaseNameKey}") + ]); + } + + public async Task> Inventory(CancellationToken cancellationToken = default) + { + var entries = new List(); + + foreach (var databaseName in new[] { lifecycle.Settings.DatabaseName, lifecycle.Settings.ThroughputDatabaseName }) + { + var statistics = await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + entries.AddRange(statistics.Collections.Select(collection => new MigrationSourceInventoryEntry(databaseName, collection.Key, collection.Value))); + } + + return entries; + } + + // Counted through the reader the copy uses, not from RavenDB's collection statistics, because a collection does not map one to one onto a category. + public async Task Count(MigrationCategory category, CancellationToken cancellationToken = default) + { + var total = 0L; + + await foreach (var batch in Read(category, resumeAfter: null, batchSize: CountBatchSize, cancellationToken)) + { + total += batch.Rows.Count; + } + + return total; + } + + public IAsyncEnumerable Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + CancellationToken cancellationToken = default) => + readers.TryGetValue(category.Id, out var reader) + ? reader.Read(resumeAfter, batchSize, cancellationToken) + : throw new NotSupportedException($"The migration source cannot yet read the '{category.Id}' category."); + + public Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) => + throw new NotSupportedException($"The RavenDB migration source cannot read bodies for category {category.Id} yet"); + + public IReadOnlyCollection SupportedCategoryIds => readers.Keys; + + public IReadOnlyDictionary DocumentTypes => readers.ToFrozenDictionary(pair => pair.Key, pair => pair.Value.DocumentType, StringComparer.Ordinal); + + public ValueTask DisposeAsync() => lifecycle.DisposeAsync(); + + // Counting only adds up row counts, so this size changes nothing but how often the stream stops to hand one back. + const int CountBatchSize = 1024; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs new file mode 100644 index 0000000000..901c76f6ac --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs @@ -0,0 +1,240 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Diagnostics; +using System.Net.Http; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Session; +using Raven.Client.Exceptions.Database; +using Raven.Client.Exceptions.Security; +using ServiceControl.Configuration; +using ServiceControl.RavenDB; + +/// +/// The connection to the RavenDB database a migration reads from, and the embedded server it starts when that +/// database is a data directory rather than a URL. Nothing here writes, so the old database stays exactly as it +/// was and the copy can still be thrown away. +/// +sealed class RavenReadOnlySourceLifecycle(RavenPersisterSettings settings, SettingsRootNamespace settingsRoot) : IAsyncDisposable +{ + public RavenPersisterSettings Settings => settings; + + public SettingsRootNamespace SettingsRoot => settingsRoot; + + public IDocumentStore DocumentStore => documentStore ?? throw new InvalidOperationException($"The migration source is not open. Call {nameof(Open)} first."); + + public async Task Open(CancellationToken cancellationToken = default) + { + if (documentStore is not null) + { + throw new InvalidOperationException("The migration source is already open. Opening it twice would abandon the first server without stopping it."); + } + + try + { + var serverUrl = settings.UseEmbeddedServer ? await StartEmbedded(cancellationToken) : settings.ConnectionString; + documentStore = Connect(serverUrl); + + if (!settings.UseEmbeddedServer) + { + // Only an external server can be older than the client; an embedded one ships beside it. + await StartupChecks.EnsureServerVersion(documentStore, cancellationToken); + } + + await EnsureReadable(settings.DatabaseName, $"{settingsRoot}/{RavenBootstrapper.DatabaseNameKey}", cancellationToken); + await EnsureReadable(settings.ThroughputDatabaseName, $"{ThroughputSettings.SettingsNamespace}/{ThroughputSettings.DatabaseNameKey}", cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await DisposeAsync(); + throw; + } + catch (Exception) + { + await DisposeAsync(); + throw; + } + } + + public IAsyncDocumentSession OpenSession(string databaseName) => + DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName, NoTracking = true }); + + async Task EnsureReadable(string databaseName, string settingKey, CancellationToken cancellationToken) + { + var elapsed = Stopwatch.StartNew(); + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + await DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); + return; + } + catch (DatabaseLoadTimeoutException e) when (settings.UseEmbeddedServer && elapsed.Elapsed >= EmbeddedLoadBudget) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but it did not finish loading within {EmbeddedLoadBudget.TotalMinutes:N0} minutes. A data directory held by another process, or one that is corrupt, is the usual cause.", e); + } + catch (DatabaseLoadTimeoutException) when (settings.UseEmbeddedServer) + { + await Task.Delay(EmbeddedLoadRetryDelay, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (DatabaseDoesNotExistException e) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has no database named '{databaseName}'. That name comes from the '{settingKey}' setting. Correct it before migrating: a wrong name reads a database that is not there rather than the one that is.", e); + } + catch (AuthorizationException e) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} refused its client certificate access to the database '{databaseName}'. Grant that certificate Read access to '{databaseName}', or supply one that has it in '{settingsRoot}/{RavenBootstrapper.ClientCertificateBase64Key}' or '{settingsRoot}/{RavenBootstrapper.ClientCertificatePathKey}'. If '{databaseName}' is the wrong name, correct the '{settingKey}' setting instead: RavenDB refuses a certificate that has no access to a database whether or not that database exists.", e); + } + catch (Exception e) when (e is not DatabaseLoadTimeoutException) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but could not load it: {e.Message}", e); + } + } + } + + string Located() => Located(settings, settingsRoot); + + internal static string Located(RavenPersisterSettings sourceSettings, SettingsRootNamespace root) => sourceSettings.UseEmbeddedServer + ? $"{sourceSettings.ServerUrl} (embedded, data directory '{sourceSettings.DatabasePath}', from '{root}/{RavenBootstrapper.DatabasePathKey}')" + : sourceSettings.ConnectionString; + + async Task StartEmbedded(CancellationToken cancellationToken) + { + // A dynamic query is a POST to /queries, which the read-only guard allows, and it builds an auto-index on the + // customer's database. Nothing on the client can tell those queries apart, so the server is told to refuse them. + var configuration = new EmbeddedDatabaseConfiguration(settings.ServerUrl, settings.DatabaseName, settings.DatabasePath, settings.LogPath, settings.LogsMode) { DisableAutoIndexCreation = true }; + + embedded = EmbeddedDatabase.Start(configuration, lifetime); + + try + { + return await embedded.WaitUntilReady(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + throw new InvalidOperationException($"The RavenDB migration source could not start a server for the embedded database at {Located()}: {e.Message} A ServiceControl instance still holding that data directory is one cause, and stopping it lets the report run; a port already in use or a missing RavenDB server are others, which the message above tells apart.", e); + } + } + + IDocumentStore Connect(string serverUrl) + { + var store = new DocumentStore + { + Database = settings.DatabaseName, + Urls = [serverUrl], + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }; + + if (!settings.UseEmbeddedServer) + { + store.Certificate = RavenClientCertificate.FindClientCertificate(settings); + RefuseUnusableCertificate(store.Certificate); + } + + store.OnBeforeRequest += RefuseWrite; + + return store.Initialize(); + } + + // An unusable certificate otherwise surfaces on the first request as a refused connection, which says nothing about the cause. + void RefuseUnusableCertificate(X509Certificate2? certificate) + { + if (certificate is null) + { + if (settings.ConnectionString.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"The RavenDB migration source at '{settings.ConnectionString}' is secured but no client certificate is configured. Set '{settingsRoot}/{RavenBootstrapper.ClientCertificatePathKey}' or '{settingsRoot}/{RavenBootstrapper.ClientCertificateBase64Key}'."); + } + + return; + } + + // X509Certificate2 reports both bounds in local time, so they move to UTC before the comparison. + var notBefore = certificate.NotBefore.ToUniversalTime(); + var notAfter = certificate.NotAfter.ToUniversalTime(); + var now = DateTime.UtcNow; + + if (now < notBefore || now > notAfter) + { + throw new InvalidOperationException($"The RavenDB client certificate '{certificate.Subject}' is valid from {notBefore:u} to {notAfter:u}, which does not include now."); + } + } + + static void RefuseWrite(object? sender, BeforeRequestEventArgs e) + { + if (IsRead(e.Request.Method, new Uri(e.Url).AbsolutePath)) + { + return; + } + + throw new InvalidOperationException($"The RavenDB migration source is open read-only and refused a {e.Request.Method} to '{e.Url}'. Nothing in a migration may write to the database it is reading."); + } + + static bool IsRead(HttpMethod method, string path) + { + if (method == HttpMethod.Get || method == HttpMethod.Head) + { + // HiLo persists the id range it hands out, so it writes despite being a GET. + return !path.Contains(HiLoPathSegment, StringComparison.OrdinalIgnoreCase); + } + + return method == HttpMethod.Post && Array.Exists(ReadOnlyPostPaths, suffix => path.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)); + } + + public async ValueTask DisposeAsync() + { + documentStore?.Dispose(); + documentStore = null; + + if (embedded is not null) + { + // Stop force-kills only once this token cancels; EmbeddedDatabase sets the graceful wait to an hour. + using var shutdown = new CancellationTokenSource(EmbeddedShutdownTimeout); + await embedded.Stop(shutdown.Token); + embedded.Dispose(); + embedded = null; + } + } + + const string HiLoPathSegment = "/hilo/"; + static readonly string[] ReadOnlyPostPaths = ["/queries", "/multi_get", "/streams/queries"]; + static readonly TimeSpan EmbeddedShutdownTimeout = TimeSpan.FromSeconds(30); + static readonly TimeSpan EmbeddedLoadRetryDelay = TimeSpan.FromMilliseconds(500); + // Generous because one DatabaseLoadTimeoutException already means RavenDB waited its own load timeout. + static readonly TimeSpan EmbeddedLoadBudget = TimeSpan.FromMinutes(5); + + IDocumentStore? documentStore; + EmbeddedDatabase? embedded; + readonly SourceLifetime lifetime = new(); + + sealed class SourceLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => CancellationToken.None; + public CancellationToken ApplicationStopped => CancellationToken.None; + + public void StopApplication() + { + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs new file mode 100644 index 0000000000..087194c67c --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/EndpointSettingsReader.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the per-endpoint settings, whole, out of the primary database. The collection holds one document per +/// endpoint plus one with an empty name, which carries the default for every endpoint. +/// +sealed class EndpointSettingsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.EndpointSettings; + + public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + WholeDocuments(EndpointSettingsStore.CollectionName + "/", resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs new file mode 100644 index 0000000000..3372368c5f --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/Readers/KnownEndpointsReader.cs @@ -0,0 +1,18 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration.Readers; + +using System.Collections.Generic; +using System.Threading; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the endpoints ServiceControl has heard from, whole, out of the primary database. +/// +sealed class KnownEndpointsReader(RavenReadOnlySourceLifecycle lifecycle) : MigrationCategoryReader(lifecycle) +{ + public override string CategoryId => MigrationCategoryIds.KnownEndpoints; + + public override IAsyncEnumerable Read(string? resumeAfter, int batchSize, CancellationToken cancellationToken = default) => + WholeDocuments(RavenMonitoringDataStore.KnownEndpointsCollectionName + "/", resumeAfter, batchSize, cancellationToken); +} diff --git a/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs b/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs index 0792bcd8ff..5c88f09c5a 100644 --- a/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/EndpointSettingsStore.cs @@ -47,5 +47,5 @@ public async Task UpdateEndpointSettings(EndpointSettings settings, Cancellation await session.SaveChangesAsync(cancellationToken); } - const string CollectionName = "EndpointSettings"; + internal const string CollectionName = "EndpointSettings"; } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs index 44b20a23a2..c0a2dd3cea 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs @@ -5,11 +5,13 @@ namespace ServiceControl.Persistence.RavenDB using System.Reflection; using Configuration; using CustomChecks; + using DataMigration; using Microsoft.Extensions.Logging; using Particular.LicensingComponent.Contracts; using ServiceControl.Infrastructure; + using ServiceControl.Persistence.DataMigration; - class RavenPersistenceConfiguration : PersistenceConfiguration, IPersistenceConfiguration + class RavenPersistenceConfiguration : PersistenceConfiguration, IPersistenceConfiguration, IMigrationSourceFactory { public const string DataSpaceRemainingThresholdKey = "DataSpaceRemainingThreshold"; const string AuditRetentionPeriodKey = "AuditRetentionPeriod"; @@ -90,5 +92,8 @@ public IPersistence Create(PersistenceSettings settings) var specificSettings = (RavenPersisterSettings)settings; return new RavenPersistence(specificSettings); } + + public IMigrationSource CreateSource(SettingsRootNamespace settingsRoot) => + new RavenMigrationSource(new RavenReadOnlySourceLifecycle((RavenPersisterSettings)CreateSettings(settingsRoot), settingsRoot)); } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs index a23b266f1b..dc2bcb5fc6 100644 --- a/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs @@ -24,6 +24,8 @@ public Task Setup(IHostApplicationBuilder hostBuilder) return Task.CompletedTask; } + public Task InstallSchema(IHost host) => Task.CompletedTask; + public Task PostSetup(IHost host) => Task.CompletedTask; public Task TearDown() => Task.CompletedTask; diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs index 4ce91496d5..b99a642473 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs @@ -51,7 +51,7 @@ public async Task Setup(IHostApplicationBuilder hostBuilder) hostBuilder.Services.AddSingleton(FakeTime); } - public async Task PostSetup(IHost host) + public async Task InstallSchema(IHost host) { this.host = host; @@ -59,6 +59,8 @@ public async Task PostSetup(IHost host) await scope.ServiceProvider.GetRequiredService().ApplyMigrations(); } + public Task PostSetup(IHost host) => Task.CompletedTask; + public async Task TearDown() { DeleteBodyStorage(); diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs new file mode 100644 index 0000000000..66b7214ec9 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/KnownEndpointSourceTests.cs @@ -0,0 +1,66 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +class KnownEndpointSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_known_endpoints_by_the_plural_prefix_in_batches_with_their_monitored_flag() + { + using (var session = await SessionProvider.OpenSession()) + { + foreach (var (name, monitored) in new[] { ("Sales.Orders", true), ("Billing", false), ("Shipping", false) }) + { + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" }, + HostDisplayName = "HOST01", + Monitored = monitored + }; + + await session.StoreAsync(endpoint, $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"); + } + + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.KnownEndpoints); + + var batches = await CollectBatches(source, category, batchSize: 2); + + Assert.That(batches.Select(batch => batch.Rows.Count), Is.EqualTo(new[] { 2, 1 })); + Assert.That(batches.SelectMany(batch => batch.Rows).Count(row => ((KnownEndpoint)row.Document).Monitored), Is.EqualTo(1)); + } + + // KnownEndpointsWriter skips these as RequiredValueMissing, which it can only do if the reader hands them over despite the required members. + [Test] + public async Task Reads_a_known_endpoint_missing_a_required_value_with_that_value_null() + { + using (var session = await SessionProvider.OpenSession()) + { + await session.StoreAsync(new { EndpointDetails = new { HostId = Guid.NewGuid(), Host = "HOST01" }, Monitored = false }, "KnownEndpoints/no-name"); + await session.StoreAsync(new { EndpointDetails = new { Name = "Sales.Orders", HostId = Guid.NewGuid() }, Monitored = false }, "KnownEndpoints/no-host"); + await session.StoreAsync(new { Monitored = false }, "KnownEndpoints/no-details"); + await session.SaveChangesAsync(); + } + + await using var source = await OpenMigrationSource(); + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.KnownEndpoints); + + var endpoints = (await CollectBatches(source, category)).SelectMany(batch => batch.Rows).ToDictionary(row => row.SourceId, row => (KnownEndpoint)row.Document); + + using (Assert.EnterMultipleScope()) + { + Assert.That(endpoints["KnownEndpoints/no-name"].EndpointDetails.Name, Is.Null); + Assert.That(endpoints["KnownEndpoints/no-host"].EndpointDetails.Host, Is.Null); + Assert.That(endpoints["KnownEndpoints/no-details"].EndpointDetails, Is.Null); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs new file mode 100644 index 0000000000..9fb6081bb3 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/MigrationCategoryReadersTests.cs @@ -0,0 +1,37 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB.DataMigration; + +[TestFixture] +class MigrationCategoryReadersTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Every_reader_claims_a_category_the_registry_knows() + { + await using var source = (RavenMigrationSource)await OpenMigrationSource(); + + var unknown = source.SupportedCategoryIds.Where(id => MigrationCategoryRegistry.Find(id) is null).ToArray(); + + Assert.That(unknown, Is.Empty, "a reader for an id no category has is dead code the engine can never reach"); + } + + [Test] + public void Every_reader_derives_from_the_typed_base() + { + var readers = typeof(IMigrationCategoryReader).Assembly.GetTypes() + .Where(type => type is { IsClass: true, IsAbstract: false } && typeof(IMigrationCategoryReader).IsAssignableFrom(type)) + .ToArray(); + + Assert.That(readers, Is.Not.Empty, "the test proves nothing if it finds no reader"); + Assert.That(readers.Where(type => !DerivesFrom(type, typeof(MigrationCategoryReader<>))).Select(type => type.Name), Is.Empty, + "a reader that implements the interface directly can declare a DocumentType it does not yield, which the pairing test cannot see"); + } + + static bool DerivesFrom(Type type, Type genericBase) => + type.BaseType is { } baseType && ((baseType.IsGenericType && baseType.GetGenericTypeDefinition() == genericBase) || DerivesFrom(baseType, genericBase)); +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs new file mode 100644 index 0000000000..358c3f9943 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTestBase.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; + +abstract class RavenMigrationSourceTestBase : RavenPersistenceTestBase +{ + static readonly SettingsRootNamespace SettingsRoot = new("ServiceControl"); + static readonly object SettingsGate = new(); + + protected async Task OpenMigrationSource() + { + var settings = (RavenPersisterSettings)PersistenceSettings; + (string Name, string Value)[] variables = + [ + ("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", settings.ConnectionString), + ("SERVICECONTROL_RAVENDB_DATABASENAME", settings.DatabaseName), + ("SERVICECONTROL_ERRORRETENTIONPERIOD", settings.ErrorRetentionPeriod.ToString()), + ("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", settings.ThroughputDatabaseName) + ]; + + IMigrationSource source; + + // Environment variables are process wide and these tests run in parallel, so without the gate a source + // reads whichever test set them last and opens another test's database. + lock (SettingsGate) + { + try + { + foreach (var (name, value) in variables) + { + Environment.SetEnvironmentVariable(name, value); + } + + source = new RavenPersistenceConfiguration().CreateSource(SettingsRoot); + } + finally + { + foreach (var (name, _) in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + } + } + + await source.Open(); + + return source; + } + + protected static async Task> CollectBatches(IMigrationSource source, MigrationCategory category, string resumeAfter = null, int batchSize = 100) + { + var batches = new List(); + + await foreach (var batch in source.Read(category, resumeAfter, batchSize, TestContext.CurrentContext.CancellationToken)) + { + batches.Add(batch); + } + + return batches; + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs new file mode 100644 index 0000000000..c34641d535 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs @@ -0,0 +1,145 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents.Operations.Indexes; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; + +class RavenMigrationSourceTests : RavenMigrationSourceTestBase +{ + [Test] + public async Task Reads_endpoint_settings_in_document_id_order_in_batches_of_the_requested_size() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + var batches = await CollectBatches(source, EndpointSettingsCategory, batchSize: 2); + + Assert.Multiple(() => + { + Assert.That(batches.Select(batch => batch.Rows.Count), Is.EqualTo(new[] { 2, 2, 1 }), "A batch never exceeds the requested size, and the trailing partial batch is still yielded."); + Assert.That(batches.SelectMany(batch => batch.Rows).Select(row => row.SourceId), Is.Ordered); + }); + } + + [Test] + public async Task Every_row_is_identified_by_the_document_id_the_persister_wrote_it_under() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + var rows = (await CollectBatches(source, EndpointSettingsCategory)).SelectMany(batch => batch.Rows).ToList(); + + Assert.Multiple(() => + { + Assert.That(rows, Has.Count.EqualTo(5)); + Assert.That(rows.Select(row => row.SourceId), Has.All.StartWith("EndpointSettings/"), "The cursor the engine checkpoints is a source document id, so a row identified by anything else cannot be resumed after."); + }); + } + + [Test] + public async Task Resuming_after_a_cursor_yields_only_what_follows_it() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + var firstBatch = (await CollectBatches(source, EndpointSettingsCategory, batchSize: 2))[0]; + var resumed = (await CollectBatches(source, EndpointSettingsCategory, resumeAfter: firstBatch.Cursor, batchSize: 2)).SelectMany(batch => batch.Rows).ToList(); + + Assert.That(resumed, Has.Count.EqualTo(3)); + Assert.That(resumed.Select(row => row.SourceId), Has.No.Member(firstBatch.Rows[0].SourceId)); + Assert.That(resumed.Select(row => row.SourceId), Is.Ordered); + } + + [Test] + public async Task Resuming_after_a_cursor_this_source_never_issued_refuses_instead_of_starting_past_it() + { + // The cursor is saved in the target database, so a restored backup or a corrected database name leaves + // one naming a document this source does not have. + await SeedEndpointSettings(3); + await using var source = await OpenMigrationSource(); + + var refusal = Assert.ThrowsAsync(() => CollectBatches(source, EndpointSettingsCategory, resumeAfter: "EndpointSettings/zzz-from-another-database")); + + Assert.Multiple(() => + { + Assert.That(refusal.Message, Does.Contain(EndpointSettingsCategory.Id)); + Assert.That(refusal.Message, Does.Contain("EndpointSettings/zzz-from-another-database")); + Assert.That(refusal.Message, Does.Contain(DatabaseName), "the operator has to be told which database the cursor was looked for in"); + }); + } + + [Test] + public async Task Counts_every_row_the_category_would_read() + { + await SeedEndpointSettings(5); + await using var source = await OpenMigrationSource(); + + Assert.That(await source.Count(EndpointSettingsCategory), Is.EqualTo(5)); + } + + [Test] + public async Task Counts_a_category_with_nothing_in_it_as_zero() + { + await using var source = await OpenMigrationSource(); + + Assert.That(await source.Count(EndpointSettingsCategory), Is.Zero, "A category with nothing to copy has to be told apart from one this source declines to count."); + } + + [Test] + public async Task Reading_every_category_creates_no_index() + { + await using var source = await OpenMigrationSource(); + + Assert.That(source.SupportedCategoryIds, Is.SubsetOf(Seeds.Keys), "A category read with nothing in it cannot show whether its reader builds an index, so a new reader needs its seed adding here."); + + foreach (var categoryId in source.SupportedCategoryIds) + { + await Seeds[categoryId](this); + } + + var before = await IndexNames(); + + foreach (var categoryId in source.SupportedCategoryIds) + { + await CollectBatches(source, MigrationCategoryRegistry.Find(categoryId)); + } + + var after = await IndexNames(); + + Assert.Multiple(() => + { + Assert.That(after, Is.EquivalentTo(before), "A reader that queries without naming a static index has RavenDB build one and index the whole collection on the customer's live database."); + Assert.That(after, Has.None.StartWith("Auto/")); + }); + } + + // Keyed by category id so a reader added without seed data fails the index test by name instead of passing + // over an empty collection. + static readonly Dictionary> Seeds = new() + { + [MigrationCategoryIds.EndpointSettings] = tests => tests.SeedEndpointSettings(2), + [MigrationCategoryIds.KnownEndpoints] = tests => tests.MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" }) + }; + + string DatabaseName => ((RavenPersisterSettings)PersistenceSettings).DatabaseName; + + async Task IndexNames() => + await DocumentStore.Maintenance.SendAsync(new GetIndexNamesOperation(0, IndexNamePageSize), TestContext.CurrentContext.CancellationToken); + + const int IndexNamePageSize = 1024; + + static readonly MigrationCategory EndpointSettingsCategory = MigrationCategoryRegistry.Find("EndpointSettings"); + + async Task SeedEndpointSettings(int count) + { + for (var index = 0; index < count; index++) + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = $"Endpoint{index}", TrackInstances = true }); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs new file mode 100644 index 0000000000..ba3f892ffe --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs @@ -0,0 +1,380 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Operations.Expiration; +using Raven.Client.Documents.Session; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Configuration; +using ServiceControl.MessageFailures; +using ServiceControl.Operations.BodyStorage.RavenAttachments; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; +using ServiceControl.Persistence.RavenDB.DataMigration; +using ServiceControl.Persistence.Tests; + +[TestFixture] +class ReadOnlySourceLifecycleTests +{ + static readonly SettingsRootNamespace SettingsRoot = new("ServiceControl"); + + string databaseName; + IDocumentStore bootstrapStore; + RavenPersisterSettings sourceSettings; + + [SetUp] + public async Task SetUp() + { + var embeddedServer = await SharedEmbeddedServer.GetInstance(); + databaseName = Guid.NewGuid().ToString("n"); + + bootstrapStore = new DocumentStore + { + Urls = [embeddedServer.ServerUrl], + Database = databaseName, + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }.Initialize(); + + await bootstrapStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(databaseName))); + await bootstrapStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord($"{databaseName}-throughput"))); + + using (var session = bootstrapStore.OpenAsyncSession()) + { + await session.StoreAsync(new FailedMessage { UniqueMessageId = "abc", Status = FailedMessageStatus.Archived }, "FailedMessages/abc"); + await session.SaveChangesAsync(); + } + + sourceSettings = new RavenPersisterSettings + { + DatabaseName = databaseName, + ThroughputDatabaseName = $"{databaseName}-throughput", + ConnectionString = embeddedServer.ServerUrl, + ErrorRetentionPeriod = TimeSpan.FromDays(10) + }; + } + + [TearDown] + public void TearDown() => bootstrapStore?.Dispose(); + + [Test] + public async Task Opening_the_source_creates_no_index() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + var statistics = await bootstrapStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation()); + + Assert.That(statistics.CountOfIndexes, Is.Zero, "Opening a migration source must not create the fifteen ServiceControl indexes on it."); + } + [Test] + public async Task Opening_the_source_creates_no_database() + { + var absentThroughput = $"{databaseName}-absent"; + sourceSettings.ThroughputDatabaseName = absentThroughput; + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + + var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain(absentThroughput).And.Contain("LicensingComponent/RavenDB/ThroughputDatabaseName")); + Assert.That(await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(absentThroughput)), Is.Null, "Opening a migration source must not create a database that was missing."); + } + + [Test] + public async Task Opening_the_source_refuses_a_missing_primary_database() + { + var absentPrimary = $"{databaseName}-absent"; + sourceSettings.DatabaseName = absentPrimary; + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + + var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain(absentPrimary).And.Contain("ServiceControl/RavenDB/DatabaseName")); + Assert.That(await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(absentPrimary)), Is.Null, "Opening a migration source must not create a primary database that was missing."); + } + + [Test] + public async Task Opening_the_source_writes_no_database_settings() + { + var before = (await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName))).Settings; + + await using (var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)) + { + await lifecycle.Open(); + } + + var after = (await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName))).Settings; + + Assert.That(after, Is.EquivalentTo(before), "Opening a migration source must not rewrite the settings of the database it reads."); + } + + [Test] + public async Task Opening_the_source_configures_no_expiration() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + var record = await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName)); + + Assert.That(record.Expiration, Is.Null, "Opening a migration source must not enable RavenDB document expiry against it, or the customer's fallback data is deleted while they are migrating."); + } + + [Test] + public async Task Writing_through_the_source_store_throws() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + await session.StoreAsync(new FailedMessage { UniqueMessageId = "def", Status = FailedMessageStatus.Unresolved }, "FailedMessages/def"); + + var exception = Assert.ThrowsAsync(async () => await session.SaveChangesAsync()); + + Assert.That(exception.Message, Does.Contain("open read-only")); + await AssertAbsent("FailedMessages/def"); + } + + [Test] + public async Task A_source_session_cannot_even_stage_a_write() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + + var exception = Assert.ThrowsAsync(async () => + await session.StoreAsync(new FailedMessage { UniqueMessageId = "def", Status = FailedMessageStatus.Unresolved }, "FailedMessages/def")); + + Assert.That(exception.Message, Does.Contain("tracking is disabled"), "OpenSession is NoTracking, so a write through it fails at Store rather than reaching the RefuseWrite request hook. Both guards have to hold: this one is the only one a copier's own sessions ever meet."); + } + + [Test] + public async Task Failed_message_status_reads_back_as_stored() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + var loaded = await session.LoadAsync("FailedMessages/abc"); + + Assert.That(loaded.Status, Is.EqualTo(FailedMessageStatus.Archived), "Without SaveEnumsAsIntegers every message status is misread, and it looks like data corruption rather than a missing convention."); + } + + [Test] + public async Task A_failed_open_leaves_no_store_behind() + { + sourceSettings.ThroughputDatabaseName = $"{databaseName}-absent"; + + var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + + Assert.ThrowsAsync(async () => await lifecycle.Open()); + + var afterFailure = Assert.Throws(() => _ = lifecycle.DocumentStore); + + Assert.That(afterFailure.Message, Does.Contain("is not open"), "A failed Open must dispose what it created. The caller never received a source, so nothing else can, and on the embedded path what leaks is a live RavenDB server process holding the customer's data directory."); + + await lifecycle.DisposeAsync(); + } + [Test] + public async Task A_source_describes_itself_as_facts_naming_the_setting_each_came_from() + { + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); + await source.Open(); + var description = await source.Describe(); + var facts = description.Facts.ToDictionary(fact => fact.Label); + + Assert.Multiple(() => + { + Assert.That(description.Version, Does.StartWith("6.")); + Assert.That(facts["Mode"].Value, Is.EqualTo("external")); + Assert.That(facts["Primary database"].Value, Is.EqualTo(databaseName)); + Assert.That(facts["Primary database"].SettingKey, Is.EqualTo("ServiceControl/RavenDB/DatabaseName")); + Assert.That(facts["Throughput database"].Value, Is.EqualTo($"{databaseName}-throughput")); + }); + } + + [Test] + public async Task An_unopened_source_refuses_to_describe_itself() + { + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); + + var exception = Assert.ThrowsAsync(async () => await source.Describe()); + + Assert.That(exception.Message, Does.Contain("is not open"), "Separating construction from Open is what lets a host register a source in its container, and it buys a state where nothing is connected yet. That state has to fail loudly rather than return an empty description."); + } + + [Test] + public async Task The_source_inventories_every_collection_by_database() + { + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); + await source.Open(); + var inventory = await source.Inventory(); + + Assert.That(inventory.Single(entry => entry.Scope == databaseName && entry.Name == "FailedMessages").Count, Is.EqualTo(1)); + } + + [Test] + public void An_embedded_source_names_the_data_directory_setting() + { + sourceSettings.ConnectionString = null; + + var location = RavenReadOnlySourceLifecycle.Located(sourceSettings, SettingsRoot); + + Assert.That(location, Does.Contain("'ServiceControl/DbPath'"), "An operator told the embedded source cannot be read needs the setting that points at its data directory."); + } + + [Test] + public async Task Opening_the_source_twice_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain("already open"), "A second Open would abandon the first store, and on the embedded path a running server process with it."); + } + + [Test] + public async Task Generating_an_id_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + + Assert.CatchAsync(async () => + await session.StoreAsync(new FailedMessage { UniqueMessageId = "hilo", Status = FailedMessageStatus.Unresolved })); + + Assert.That(await CountDocuments(), Is.EqualTo(1), "HiLo reserves an id range with a request of its own before SaveChanges, so only the request hook can refuse it."); + } + + [Test] + public async Task A_patch_against_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + await lifecycle.DocumentStore.Operations.ForDatabase(databaseName).SendAsync( + new PatchOperation("FailedMessages/abc", null, new PatchRequest { Script = "this.Status = 1;" }))); + + Assert.That(await LoadStatus("FailedMessages/abc"), Is.EqualTo(FailedMessageStatus.Archived), "A patch is a request of its own with no session, so only the request hook can refuse it; the RavenDB persister writes this way."); + } + + [Test] + public async Task A_bulk_insert_into_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + { + await using var bulk = lifecycle.DocumentStore.BulkInsert(databaseName); + await bulk.StoreAsync(new FailedMessage { UniqueMessageId = "bulk", Status = FailedMessageStatus.Unresolved }, "FailedMessages/bulk"); + }); + + await AssertAbsent("FailedMessages/bulk"); + } + + [Test] + public async Task Reconfiguring_expiry_on_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync( + new ConfigureExpirationOperation(new ExpirationConfiguration { Disabled = false, DeleteFrequencyInSec = 60 }))); + + var record = await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName)); + + Assert.That(record.Expiration, Is.Null, "Enabling expiry on the source would start deleting the customer fallback data."); + } + + [Test] + public async Task Deleting_an_untracked_document_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + session.Delete("FailedMessages/abc"); + + Assert.CatchAsync(async () => await session.SaveChangesAsync()); + + Assert.That(await CountDocuments(), Is.EqualTo(1), "A delete by id of an untracked document only goes out at SaveChanges, as a batch request the hook must refuse."); + } + + [Test] + public async Task Reading_the_source_still_works() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + + var loaded = await session.LoadAsync("FailedMessages/abc"); + var queried = await session.Query().ToListAsync(); + + Assert.Multiple(() => + { + Assert.That(loaded, Is.Not.Null, "A guard that fails closed still has to let every read the migration needs through."); + Assert.That(queried, Has.Count.EqualTo(1)); + }); + } + + [Test] + public async Task A_source_session_reads_a_message_body_attachment() + { + using (var session = bootstrapStore.OpenAsyncSession()) + { + using var stored = new MemoryStream(Encoding.UTF8.GetBytes("1")); + session.Advanced.Attachments.Store("FailedMessages/abc", RavenAttachmentsBodyStorage.AttachmentName, stored, "text/xml"); + await session.SaveChangesAsync(); + } + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var sourceSession = lifecycle.OpenSession(databaseName); + using var attachment = await sourceSession.Advanced.Attachments.GetAsync("FailedMessages/abc", RavenAttachmentsBodyStorage.AttachmentName); + using var read = new MemoryStream(); + await attachment.Stream.CopyToAsync(read); + + Assert.Multiple(() => + { + Assert.That(Encoding.UTF8.GetString(read.ToArray()), Is.EqualTo("1"), "The body copy reads every attachment through this no-tracking session, so a read that needed tracking would need a second kind of source session."); + Assert.That(attachment.Details.ContentType, Is.EqualTo("text/xml")); + }); + } + + async Task AssertAbsent(string documentId) + { + using var session = bootstrapStore.OpenAsyncSession(); + var found = await session.LoadAsync(documentId); + + Assert.That(found, Is.Null, $"The source refused the write, so '{documentId}' must not be in the database. Asserting the exception alone tests the guard, not the guarantee."); + } + + async Task CountDocuments() + { + var statistics = await bootstrapStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation()); + return statistics.CountOfDocuments; + } + + async Task LoadStatus(string documentId) + { + using var session = bootstrapStore.OpenAsyncSession(); + var found = await session.LoadAsync(documentId); + return found.Status; + } +} diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs index 5bcae71f20..aca5f06236 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs @@ -50,6 +50,8 @@ public async Task Setup(IHostApplicationBuilder hostBuilder) persistence.AddInstaller(hostBuilder.Services); } + public Task InstallSchema(IHost host) => Task.CompletedTask; + public async Task PostSetup(IHost host) { DocumentStore = await host.Services.GetRequiredService().GetDocumentStore(); diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs new file mode 100644 index 0000000000..8a90e4965c --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs @@ -0,0 +1,172 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class EndpointSettingsKeyCollationTests : PersistenceTestBase +{ + readonly TargetWarnings targetWarnings = new(); + + public EndpointSettingsKeyCollationTests() => RegisterServices = services => services.AddSingleton(targetWarnings); + + [Test] + public async Task A_case_merge_inside_one_batch_logs_both_names_and_the_one_kept() + { + await OpenACaseInsensitiveTarget("Sales", "sales"); + + await Write(("EndpointSettings/1", "Sales"), ("EndpointSettings/2", "sales")); + + Assert.That(targetWarnings.Messages.Count(message => message.Contains("'sales'") && message.Contains("kept 'Sales'")), Is.EqualTo(1), string.Join(Environment.NewLine, targetWarnings.Messages)); + } + + [Test] + public async Task A_case_merge_with_a_row_an_earlier_batch_wrote_is_logged_and_a_row_written_again_is_not() + { + await OpenACaseInsensitiveTarget("Sales", "sales"); + + await Write(("EndpointSettings/1", "Sales")); + await Write(("EndpointSettings/1", "Sales"), ("EndpointSettings/2", "sales")); + + Assert.That(targetWarnings.Messages, Has.Exactly(1).Contains("kept 'Sales'"), string.Join(Environment.NewLine, targetWarnings.Messages)); + } + + async Task OpenACaseInsensitiveTarget(params string[] knownEndpointNames) + { + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + var entityType = dbContext.Model.FindEntityType(typeof(EndpointSettingsEntity)); + var table = dbContext.GetService().DelimitIdentifier(entityType.GetTableName(), entityType.GetSchema()); + + // Set on the column, because the merge follows the column's collation and the test server's default can be either. + var ignoreCase = $""" + ALTER TABLE {table} DROP CONSTRAINT [PK_EndpointSettings]; + ALTER TABLE {table} ALTER COLUMN [Name] nvarchar(450) COLLATE Latin1_General_CI_AS NOT NULL; + ALTER TABLE {table} ADD CONSTRAINT [PK_EndpointSettings] PRIMARY KEY ([Name]); + """; + + await dbContext.Database.ExecuteSqlRawAsync(ignoreCase); + } + + foreach (var name in knownEndpointNames) + { + await MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" }); + } + + await ServiceProvider.GetRequiredService().Open(); + } + + async Task Write(params (string SourceId, string Name)[] rows) + { + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.EndpointSettings); + var batch = new MigrationBatch( + [.. rows.Select(row => new MigrationRow(row.SourceId, new EndpointSettings { Name = row.Name, TrackInstances = true }, new Dictionary()))], + rows[^1].SourceId); + var checkpoint = await ServiceProvider.GetRequiredService().Read(category.Id) + ?? new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); + + await ServiceProvider.GetRequiredService().Write(category, batch, checkpoint); + } + + [Test] + public async Task The_key_columns_own_collation_decides_a_merge_when_the_database_default_disagrees() + { + bool databaseIgnoresCase; + + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + var database = dbContext.Database; + + databaseIgnoresCase = await database + .SqlQuery($"SELECT CONVERT(int, DATABASEPROPERTYEX(DB_NAME(), 'ComparisonStyle')) & 1 AS [Value]") + .SingleAsync() == 1; + + // Named from the model, because the table sits in the configured schema when the persister has one. + var entityType = dbContext.Model.FindEntityType(typeof(EndpointSettingsEntity)); + var table = dbContext.GetService().DelimitIdentifier(entityType.GetTableName(), entityType.GetSchema()); + + // The column is given the opposite of the database default, because a test where the two agree cannot show which one decided. + var recollate = $""" + DECLARE @columnCollation sysname = IIF(CONVERT(int, DATABASEPROPERTYEX(DB_NAME(), 'ComparisonStyle')) & 1 = 1, N'Latin1_General_CS_AS', N'Latin1_General_CI_AS'); + ALTER TABLE {table} DROP CONSTRAINT [PK_EndpointSettings]; + EXEC (N'ALTER TABLE {table} ALTER COLUMN [Name] nvarchar(450) COLLATE ' + @columnCollation + N' NOT NULL'); + ALTER TABLE {table} ADD CONSTRAINT [PK_EndpointSettings] PRIMARY KEY ([Name]); + """; + + await database.ExecuteSqlRawAsync(recollate); + } + + foreach (var name in new[] { "Sales", "sales" }) + { + await MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = name, HostId = Guid.NewGuid(), Host = "HOST01" }); + } + + var target = ServiceProvider.GetRequiredService(); + await target.Open(); + + var category = MigrationCategoryRegistry.All.Single(entry => entry.Id == MigrationCategoryIds.EndpointSettings); + var batch = new MigrationBatch( + [ + new MigrationRow("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true }, new Dictionary()), + new MigrationRow("EndpointSettings/2", new EndpointSettings { Name = "sales", TrackInstances = false }, new Dictionary()) + ], + "EndpointSettings/2"); + var checkpointToExtend = new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, batch.Cursor, 0, 0, null, null, null, null, null, null); + + var result = await target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(databaseIgnoresCase ? 2 : 1), "the column's own collation decides: two keys where it respects case, one where it ignores case, whatever the database default says"); + Assert.That( + ServiceProvider.GetRequiredService().KeyComparer(typeof(EndpointSettingsEntity), nameof(EndpointSettingsEntity.Name)).Equals("Sales", "sales"), + Is.EqualTo(!databaseIgnoresCase), + "the dry run predicts merges with this comparer, so it follows the same column the statement does"); + } + } + + // RecordingLoggerProvider lives in ServiceControl.Infrastructure.Tests, which this project does not reference. + sealed class TargetWarnings : ILoggerProvider + { + readonly System.Collections.Concurrent.ConcurrentQueue messages = new(); + + public IReadOnlyList Messages => [.. messages]; + + public ILogger CreateLogger(string categoryName) => + categoryName == typeof(EFCoreMigrationTarget).FullName ? new WarningLogger(messages) : Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance; + + public void Dispose() + { + } + + sealed class WarningLogger(System.Collections.Concurrent.ConcurrentQueue messages) : ILogger + { + public IDisposable BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel == LogLevel.Warning; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + if (logLevel == LogLevel.Warning) + { + messages.Enqueue(formatter(state, exception)); + } + } + } + } +} diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs index 9ded3182bc..2dab62abe4 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs @@ -50,7 +50,7 @@ public async Task Setup(IHostApplicationBuilder hostBuilder) hostBuilder.Services.AddSingleton(FakeTime); } - public async Task PostSetup(IHost host) + public async Task InstallSchema(IHost host) { this.host = host; @@ -58,6 +58,8 @@ public async Task PostSetup(IHost host) await scope.ServiceProvider.GetRequiredService().ApplyMigrations(); } + public Task PostSetup(IHost host) => Task.CompletedTask; + public async Task TearDown() { DeleteBodyStorage(); diff --git a/src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs new file mode 100644 index 0000000000..67d7c9c307 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/EFMigrationCheckpointStoreTests.cs @@ -0,0 +1,125 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +class EFMigrationCheckpointStoreTests : PersistenceTestBase +{ + IMigrationCheckpointStore Store => ServiceProvider.GetRequiredService(); + + [Test] + public async Task Read_returns_null_for_a_category_that_has_never_run() + { + Assert.That(await Store.Read("EndpointSettings"), Is.Null); + } + + [Test] + public async Task Upsert_then_Read_round_trips_every_field() + { + var checkpoint = new MigrationCheckpoint( + "EndpointSettings", MigrationCategoryState.CompleteWithErrors, "cursor-1", 5, 2, 40, + new Dictionary { [MigrationSkipReason.BodyUnreadable] = 2 }, Now, Now, Now, "body storage unavailable", AlreadyPresentCount: 33, + StartedWindowSeconds: (long)TimeSpan.FromDays(14).TotalSeconds); + + var saved = await Store.Upsert(checkpoint); + var stored = await Store.Read("EndpointSettings"); + + // Record equality compares SkipReasons by reference, and a round-tripped dictionary is a new instance. + using (Assert.EnterMultipleScope()) + { + Assert.That(stored! with { SkipReasons = null, Version = 0 }, Is.EqualTo(checkpoint with { SkipReasons = null })); + Assert.That(stored.SkipReasons, Is.EquivalentTo(checkpoint.SkipReasons)); + Assert.That(stored.Version, Is.EqualTo(1), "the first save leaves the row at version 1"); + // The engine saves from what Upsert hands back, so a returned version that is not the stored one + // makes the next save conflict against a row nothing else touched. + Assert.That(saved.Version, Is.EqualTo(stored.Version)); + } + } + + [Test] + public async Task A_second_Upsert_for_the_same_category_updates_rather_than_duplicates() + { + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.InProgress, "cursor-1", 5, 0, 40, null, Now, Now, null, null)); + var saved = await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Complete, "cursor-2", 9, 1, 40, null, Now, Now, Now, null, Version: 1)); + + var all = await Store.ReadAll(); + var stored = await Store.Read("EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(all, Has.Count.EqualTo(1)); + Assert.That(saved.Version, Is.EqualTo(stored!.Version), "an update hands back the version it landed on, not the one it was saved from"); + Assert.That(stored.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(stored.Cursor, Is.EqualTo("cursor-2")); + Assert.That(stored.CopiedCount, Is.EqualTo(9)); + Assert.That(stored.SettledAt, Is.EqualTo(Now)); + } + } + + [Test] + public async Task A_checkpoint_saved_as_Abandoned_stores_its_counts_its_reasons_and_when_it_was_abandoned() + { + var reasons = new Dictionary { [MigrationSkipReason.BodyUnreadable] = 400 }; + var halted = Now.AddMinutes(-5); + await Store.Upsert(new MigrationCheckpoint("GroupComments", MigrationCategoryState.Halted, "g-1", 12, 400, 412, reasons, Now, Now, halted, "body storage unavailable")); + await Store.Upsert(new MigrationCheckpoint("GroupComments", MigrationCategoryState.Abandoned, "g-1", 12, 400, 412, reasons, Now, Now, Now, "body storage unavailable", Version: 1)); + + var stored = await Store.Read("GroupComments"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored!.State, Is.EqualTo(MigrationCategoryState.Abandoned)); + // State says which terminal state was reached and SettledAt says when, so abandoning a halted + // category moves the one timestamp on rather than filling a second column beside it. + Assert.That(stored.SettledAt, Is.EqualTo(Now)); + Assert.That(stored.SettledAt, Is.Not.EqualTo(halted)); + Assert.That(stored.SkippedCount, Is.EqualTo(400)); + Assert.That(stored.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 400 })); + Assert.That(stored.LastError, Is.EqualTo("body storage unavailable")); + } + } + + [Test] + public async Task A_restart_after_a_halt_clears_the_fields_the_previous_save_set() + { + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Halted, "cursor-1", 5, 2, 40, + new Dictionary { [MigrationSkipReason.BodyUnreadable] = 2 }, Now, Now, Now, "body storage unavailable")); + + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.InProgress, "cursor-1", 5, 2, 40, + null, Now, Now, null, null, Version: 1)); + + var stored = await Store.Read("EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored!.SettledAt, Is.Null, "a resumed category must not look finished"); + Assert.That(stored.LastError, Is.Null); + Assert.That(stored.SkipReasons, Is.Null); + } + } + + [Test] + public void An_upsert_for_a_category_that_has_never_run_is_refused_when_it_carries_a_version() + { + var conflict = Assert.ThrowsAsync(() => Store.Upsert( + new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.InProgress, "cursor-1", 1, 0, 40, null, Now, Now, null, null, Version: 1))); + + Assert.That(conflict!.Message, Does.Contain("EndpointSettings")); + } + + [Test] + public async Task ReadAll_returns_every_category_that_has_run() + { + await Store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Complete, "e-1", 1, 0, 1, null, Now, Now, Now, null)); + await Store.Upsert(new MigrationCheckpoint("KnownEndpoints", MigrationCategoryState.InProgress, "k-1", 4, 0, 9, null, Now, Now, null, null)); + + var all = await Store.ReadAll(); + + Assert.That(all.Select(c => c.CategoryId), Is.EquivalentTo(new[] { "EndpointSettings", "KnownEndpoints" })); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs new file mode 100644 index 0000000000..2826d26011 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/EndpointSettingsMigrationTargetTests.cs @@ -0,0 +1,143 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class EndpointSettingsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + IMigrationCheckpointStore CheckpointStore => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory EndpointSettingsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.EndpointSettings); + + [Test] + public async Task Every_setting_in_a_batch_arrives_with_its_track_instances_value() + { + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = "Billing", TrackInstances = false }), + ("EndpointSettings/3", new EndpointSettings { Name = "Shipping", TrackInstances = true })), + CheckpointAfter("EndpointSettings/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(3), "Copied is what the insert added, not the size of the batch"); + Assert.That(await TrackInstancesFor("Sales"), Is.True); + Assert.That(await TrackInstancesFor("Billing"), Is.False); + Assert.That(await TrackInstancesFor("Shipping"), Is.True); + } + } + + [Test] + public async Task A_name_already_in_the_target_is_left_alone_and_counted_as_already_present() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf(("EndpointSettings/7", new EndpointSettings { Name = "Sales", TrackInstances = false })), + CheckpointAfter("EndpointSettings/7")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.Zero, "a row the target already holds lost nothing, so it is not a skip"); + Assert.That(await TrackInstancesFor("Sales"), Is.True); + } + } + + [Test] + public async Task A_setting_whose_endpoint_is_not_known_is_copied_and_left_to_the_heartbeat_sync() + { + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = string.Empty, TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = "Retired", TrackInstances = false })), + CheckpointAfter("EndpointSettings/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.Skipped, Is.Zero, "whether a setting outlives its endpoint is the heartbeat sync's rule, not the migration's"); + Assert.That(await TrackInstancesFor(string.Empty), Is.True); + Assert.That(await TrackInstancesFor("Retired"), Is.False); + } + } + + [Test] + public async Task A_batch_that_copies_nothing_still_saves_the_cursor() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Billing", TrackInstances = true }); + + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = false }), + ("EndpointSettings/2", new EndpointSettings { Name = "Billing", TrackInstances = false })), + CheckpointAfter("EndpointSettings/2")); + + var stored = await CheckpointStore.Read(MigrationCategoryIds.EndpointSettings); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(2)); + Assert.That(stored.Cursor, Is.EqualTo("EndpointSettings/2"), "a batch that copied nothing still has to commit the cursor past it, or the restart reads the same rows forever"); + } + } + + [Test] + public async Task Every_mapped_column_is_set_from_a_fully_populated_document() + { + await Target.Write(EndpointSettingsCategory, BatchOf(("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true })), CheckpointAfter("EndpointSettings/1")); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(dbContext.Model, await dbContext.EndpointSettings.AsNoTracking().SingleAsync()); + } + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.EndpointSettings, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); + + async Task TrackInstancesFor(string name) => + (await EndpointSettingsStore.GetAllEndpointSettings().ToListAsync()).Single(settings => settings.Name == name).TrackInstances; + + [Test] + public async Task Two_names_in_one_batch_differing_only_in_case_are_each_copied_or_already_present() + { + var result = await Target.Write( + EndpointSettingsCategory, + BatchOf( + ("EndpointSettings/1", new EndpointSettings { Name = "Sales", TrackInstances = true }), + ("EndpointSettings/2", new EndpointSettings { Name = "sales", TrackInstances = false })), + CheckpointAfter("EndpointSettings/2")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied + result.AlreadyPresent, Is.EqualTo(2), "SQL Server's default collation makes these one key and PostgreSQL makes them two; either way neither row may throw or go uncounted"); + Assert.That(result.Skipped, Is.Zero, "a merge onto a row the batch writes loses nothing the target lacks, and a skip would count toward the halt threshold"); + Assert.That(await Target.Count(EndpointSettingsCategory), Is.EqualTo(result.Copied), "every row reported as copied is a row in the table"); + Assert.That(await TrackInstancesFor("Sales"), Is.True, "the first row in document-id order wins where the key makes the two one"); + } + } + +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs new file mode 100644 index 0000000000..41cf3f5bd5 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/KnownEndpointsMigrationTargetTests.cs @@ -0,0 +1,199 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Operations; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; + +class KnownEndpointsMigrationTargetTests : PersistenceTestBase +{ + [SetUp] + public Task OpenTarget() => Target.Open(); + + IMigrationTarget Target => ServiceProvider.GetRequiredService(); + + static readonly MigrationCategory KnownEndpointsCategory = MigrationCategoryRegistry.All.Single(category => category.Id == MigrationCategoryIds.KnownEndpoints); + + [Test] + public async Task A_known_endpoint_is_written_with_its_monitored_flag() + { + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }, + HostDisplayName = "SALES01", + Monitored = true + }; + var sourceId = $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId)); + + var stored = (await MonitoringDataStore.GetAllKnownEndpoints()).Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(stored.Monitored, Is.True); + Assert.That(stored.EndpointDetails.Name, Is.EqualTo("Sales.Orders")); + } + } + + [Test] + public async Task A_known_endpoint_already_in_the_target_keeps_its_flag_and_is_counted_as_already_present() + { + var details = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }; + await MonitoringDataStore.CreateIfNotExists(details); + + var endpoint = new KnownEndpoint { EndpointDetails = details, HostDisplayName = "SALES01", Monitored = true }; + var sourceId = $"KnownEndpoints/{details.GetDeterministicId()}"; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Single().Monitored, Is.False, "insert-if-absent never updates, so the flag the target already had stays"); + } + } + + [Test] + public async Task A_known_endpoint_with_no_name_or_no_host_is_skipped_as_required_value_missing() + { + var named = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" } }; + var nameless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = null, HostId = Guid.NewGuid(), Host = "SALES02" } }; + var hostless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = null } }; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf(("KnownEndpoints/1", named), ("KnownEndpoints/2", nameless), ("KnownEndpoints/3", hostless)), CheckpointAfter("KnownEndpoints/3")); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EquivalentTo(new[] { "KnownEndpoints/2", "KnownEndpoints/3" })); + Assert.That(result.SkipReasons[MigrationSkipReason.RequiredValueMissing], Is.EqualTo(2), "KnownEndpoints.Name and Host are NOT NULL, and a throw here would halt a required category"); + Assert.That((await MonitoringDataStore.GetAllKnownEndpoints()).Select(endpoint => endpoint.EndpointDetails.Name), Is.EqualTo(new[] { "Sales.Orders" })); + } + } + + [Test] + public async Task A_batch_of_nothing_but_skips_saves_the_cursor_and_writes_no_rows() + { + var nameless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = null, HostId = Guid.NewGuid(), Host = "SALES02" } }; + var hostless = new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = "Billing", HostId = Guid.NewGuid(), Host = null } }; + + var result = await Target.Write(KnownEndpointsCategory, BatchOf(("KnownEndpoints/1", nameless), ("KnownEndpoints/2", hostless)), CheckpointAfter("KnownEndpoints/2")); + + var stored = await ServiceProvider.GetRequiredService().Read(MigrationCategoryIds.KnownEndpoints); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.Zero); + Assert.That(result.AlreadyPresent, Is.Zero, "no key was looked up, so no row may be counted as one the target already held"); + Assert.That(result.Skipped, Is.EqualTo(2)); + Assert.That(await MonitoringDataStore.GetAllKnownEndpoints(), Is.Empty); + Assert.That(stored.Cursor, Is.EqualTo("KnownEndpoints/2"), "a batch that copied nothing still has to commit the cursor past it, or the restart reads the same rows forever"); + } + } + + [Test] + public void A_batch_reporting_more_copied_and_skipped_rows_than_it_held_is_refused() + { + // The guard only counts rows, so what the documents hold cannot change its answer. + var batch = BatchOf(("KnownEndpoints/1", new object()), ("KnownEndpoints/2", new object())); + + var exception = Assert.Throws(() => EFCoreMigrationTarget.AlreadyPresentIn(KnownEndpointsCategory, batch, copied: 2, skipped: 1)); + + Assert.That(exception.Message, Does.Contain("copied 2").And.Contain("skipped 1").And.Contain(MigrationCategoryIds.KnownEndpoints)); + } + + [Test] + public async Task A_batch_whose_checkpoint_save_fails_leaves_no_rows_behind() + { + // The rows save first and the checkpoint second, so this is the only order in which the two can part + // company: a stale version fails the checkpoint after the endpoint row is already in the transaction. + var store = ServiceProvider.GetRequiredService(); + await store.Upsert(CheckpointAfter("KnownEndpoints/0")); + + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }, + Monitored = true + }; + var sourceId = $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"; + + Assert.ThrowsAsync(async () => + await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await MonitoringDataStore.GetAllKnownEndpoints(), Is.Empty, "the rows and the checkpoint commit together, so a failed checkpoint takes the rows with it"); + Assert.That((await store.Read(MigrationCategoryIds.KnownEndpoints)).Cursor, Is.EqualTo("KnownEndpoints/0"), "the stored cursor must still describe the rows the target actually holds"); + } + } + + [Test] + public async Task Every_mapped_column_is_set_from_a_fully_populated_document() + { + var endpoint = new KnownEndpoint + { + EndpointDetails = new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "SALES01" }, + HostDisplayName = "SALES01", + Monitored = true + }; + var sourceId = $"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}"; + + await Target.Write(KnownEndpointsCategory, BatchOf((sourceId, endpoint)), CheckpointAfter(sourceId)); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + MigrationEntityCoverage.AssertEveryMappedPropertyIsSet(dbContext.Model, await dbContext.KnownEndpoints.AsNoTracking().SingleAsync()); + } + + + [Test] + public async Task A_batch_spanning_several_statements_counts_what_each_statement_inserted() + { + var rows = Enumerable.Range(0, 500) + .Select(index => new KnownEndpoint { EndpointDetails = new EndpointDetails { Name = $"Endpoint{index}", HostId = Guid.NewGuid(), Host = "HOST01" }, Monitored = index % 2 == 0 }) + .Select(endpoint => ($"KnownEndpoints/{endpoint.EndpointDetails.GetDeterministicId()}", (object)endpoint)) + .ToArray(); + var batch = BatchOf(rows); + + var first = await Target.Write(KnownEndpointsCategory, batch, CheckpointAfter(batch.Cursor)); + // The engine carries the committed checkpoint into the next write, and the version guard refuses anything else. + var second = await Target.Write(KnownEndpointsCategory, batch, first.Saved with { Cursor = batch.Cursor }); + + using (Assert.EnterMultipleScope()) + { + Assert.That(first.Copied, Is.EqualTo(500)); + Assert.That(second.Copied, Is.Zero, "every key is present on the second write, so no statement may report a row it did not insert"); + Assert.That(second.AlreadyPresent, Is.EqualTo(500)); + } + } + + + [Test] + public void A_batch_whose_writer_neither_prepared_nor_skipped_a_row_is_refused() + { + // The already-present count is a subtraction, so it only catches a writer that over-reports. A row dropped in silence under-reports, and without this guard it is counted as a row the target already held. + var batch = BatchOf(("KnownEndpoints/1", new object()), ("KnownEndpoints/2", new object())); + var prepared = new PreparedBatch((_, _) => Task.FromResult(1), PreparedRowCount: 1, Skips: [], Merges: []); + + var exception = Assert.Throws(() => EFCoreMigrationTarget.AccountForEveryRow(KnownEndpointsCategory, batch, prepared)); + + Assert.That(exception.Message, Does.Contain("prepared 1").And.Contain("skipped 0").And.Contain(MigrationCategoryIds.KnownEndpoints)); + } + + static MigrationBatch BatchOf(params (string Id, object Document)[] rows) => + new([.. rows.Select(row => new MigrationRow(row.Id, row.Document, new Dictionary()))], rows[^1].Id); + + static MigrationCheckpoint CheckpointAfter(string cursor) => + new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, cursor, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs new file mode 100644 index 0000000000..7fa6186c1e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationCategoryWritersTests.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DataMigration; + +class MigrationCategoryWritersTests : PersistenceTestBase +{ + [Test] + public void Every_writer_claims_a_category_the_registry_knows() + { + var unknown = ServiceProvider.GetRequiredService().SupportedCategoryIds + .Where(id => MigrationCategoryRegistry.Find(id) is null) + .ToArray(); + + Assert.That(unknown, Is.Empty, "a writer for an id no category has is dead code the engine can never reach"); + } + + [Test] + public void Every_writer_derives_from_the_typed_base() + { + var writers = typeof(IMigrationCategoryWriter).Assembly.GetTypes() + .Where(type => type is { IsClass: true, IsAbstract: false } && typeof(IMigrationCategoryWriter).IsAssignableFrom(type)) + .ToArray(); + + Assert.That(writers, Is.Not.Empty, "the test proves nothing if it finds no writer"); + Assert.That(writers.Where(type => !DerivesFrom(type, typeof(MigrationCategoryWriter<>))).Select(type => type.Name), Is.Empty, + "a writer that implements the interface directly skips the named cast and can declare a DocumentType it does not take"); + } + + static bool DerivesFrom(Type type, Type genericBase) => + type.BaseType is { } baseType && ((baseType.IsGenericType && baseType.GetGenericTypeDefinition() == genericBase) || DerivesFrom(baseType, genericBase)); + + [Test] + public async Task Every_writer_names_its_category_and_the_row_when_a_document_is_not_its_type() + { + var target = ServiceProvider.GetRequiredService(); + await target.Open(); + + using (Assert.EnterMultipleScope()) + { + foreach (var (categoryId, document) in target.SupportedCategoryIds.SelectMany(id => new (string, object)[] { (id, new object()), (id, null) })) + { + var category = MigrationCategoryRegistry.Find(categoryId)!; + var batch = new MigrationBatch([new MigrationRow("Wrong/1", document, new Dictionary())], "Wrong/1"); + var checkpoint = new MigrationCheckpoint(categoryId, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null); + + var exception = Assert.ThrowsAsync(() => target.Write(category, batch, checkpoint)); + + Assert.That(exception!.Message, Does.Contain(categoryId).And.Contain("Wrong/1").And.Contain(target.DocumentTypes[categoryId].FullName), "an engine halt quotes this message, so it has to say which pair disagrees"); + } + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs new file mode 100644 index 0000000000..d3ede8f8ec --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverage.cs @@ -0,0 +1,34 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections; +using System.Linq; +using Microsoft.EntityFrameworkCore.Metadata; +using NUnit.Framework; + +static class MigrationEntityCoverage +{ + public static void AssertEveryMappedPropertyIsSet(IModel model, TEntity entity, params string[] legitimatelyDefault) + { + var entityType = model.FindEntityType(typeof(TEntity)) + ?? throw new ArgumentException($"{typeof(TEntity).Name} is not an entity in the EF Core model.", nameof(entity)); + + // Reflection rather than a new() constraint, which an entity with required members cannot satisfy. + var defaults = Activator.CreateInstance(typeof(TEntity)); + + // Every property in this model that is not ValueGenerated.Never is an identity column the database assigns. + var unset = entityType.GetProperties() + .Where(property => property.ValueGenerated == ValueGenerated.Never && !legitimatelyDefault.Contains(property.Name)) + .Where(property => SameValue(property.GetGetter().GetClrValue(entity), property.GetGetter().GetClrValue(defaults))) + .Select(property => property.Name) + .ToArray(); + + Assert.That(unset, Is.Empty, $"{typeof(TEntity).Name} has mapped properties equal to a default-constructed {typeof(TEntity).Name}. Set each from the source document, feed the test a document whose values all differ from those defaults, or name the property in legitimatelyDefault if RavenDB genuinely holds that value."); + } + + // Collections compare by item, and an empty one is unset whether or not the entity initialises it. + static bool SameValue(object actual, object defaultValue) => + actual is IEnumerable items and not string + ? !items.Cast().Any() || (defaultValue is IEnumerable defaultItems && items.Cast().SequenceEqual(defaultItems.Cast())) + : Equals(actual, defaultValue); +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs new file mode 100644 index 0000000000..d5d0fedf42 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationEntityCoverageClaimTests.cs @@ -0,0 +1,33 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Linq; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.EFCore.DbContexts; + +class MigrationEntityCoverageClaimTests : PersistenceTestBase +{ + [Test] + public void Every_property_the_database_fills_in_is_a_key_it_assigns_on_insert() + { + using var scope = ServiceProvider.CreateScope(); + var model = scope.ServiceProvider.GetRequiredService().Model; + + var databaseFilled = model.GetEntityTypes() + .SelectMany(entityType => entityType.GetProperties()) + .Where(property => property.ValueGenerated != ValueGenerated.Never) + .ToArray(); + + var notKeysAssignedOnInsert = databaseFilled + .Where(property => !property.IsPrimaryKey() || property.ValueGenerated != ValueGenerated.OnAdd) + .Select(property => $"{property.DeclaringType.ClrType.Name}.{property.Name} is {property.ValueGenerated}") + .ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(databaseFilled, Is.Not.Empty, "the model no longer has a single property the database fills in, so this test is watching nothing. Either the identity keys have gone, or the model was never built."); + Assert.That(notKeysAssignedOnInsert, Is.Empty, "MigrationEntityCoverage checks only the properties that are ValueGenerated.Never, on the claim that every other one is a key the database assigns on insert. These properties break that claim, so the coverage check silently stops watching them and a migration can copy a row with them left unset."); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs new file mode 100644 index 0000000000..da87ec7dcd --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetCoverageTests.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.Tests; + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +class MigrationTargetCoverageTests : PersistenceTestBase +{ + [Test] + public async Task Every_category_the_target_supports_has_a_batch_size_and_a_count() + { + var target = ServiceProvider.GetRequiredService(); + + Assert.That(target.SupportedCategoryIds, Is.SupersetOf(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings }), "the test proves nothing if the target supports no category"); + + foreach (var id in target.SupportedCategoryIds) + { + var category = MigrationCategoryRegistry.Find(id)!; + + Assert.That(await target.BatchSizeFor(category, CancellationToken.None), Is.Positive, $"{id}: every supported category has a batch size"); + Assert.That(await target.Count(category, CancellationToken.None), Is.Zero, id); + } + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs new file mode 100644 index 0000000000..caffb35e0e --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs @@ -0,0 +1,226 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.Abstractions; +using ServiceControl.Persistence.EFCore.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; +using ServiceControl.Persistence.EFCore.Implementation.BodyStorage; +using ServiceControl.Persistence.EFCore.Infrastructure; + +class MigrationTargetReadinessTests : PersistenceTestBase +{ + IMigrationTargetReadiness Readiness => ServiceProvider.GetRequiredService(); + + [Test] + public void The_target_contributes_the_three_checks_only_it_can_make() => + Assert.That( + Readiness.ContributedChecks().Select(check => check.GetType()), + Is.EqualTo(new[] { typeof(SchemaIsCurrentCheck), typeof(TargetHoldsNoServiceControlDataCheck), typeof(BodyStorageIsWritableCheck) })); + + [Test] + public void Every_contributed_check_passes_against_a_migrated_database() + { + using (Assert.EnterMultipleScope()) + { + foreach (var check in Readiness.ContributedChecks()) + { + Assert.DoesNotThrowAsync(() => check.Run(), $"a healthy target must pass every contributed check, and it failed '{check.Name}'"); + } + } + } + + [Test] + public async Task A_target_holding_a_row_is_refused_while_no_checkpoint_exists() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + + var exception = Assert.ThrowsAsync(() => TargetCheck.Run()); + + Assert.That(exception.Message, Does.Contain(TableName()).And.Contain("--setup")); + } + + // A plain start writes this row first, so a key someone later ignores fails here before it reaches a customer. + [Test] + public async Task A_settings_row_counts_as_data() + { + await ServiceProvider.GetRequiredService().StoreTrialEndDate(new DateOnly(2030, 1, 1)); + + var exception = Assert.ThrowsAsync(() => TargetCheck.Run()); + + Assert.That(exception.Message, Does.Contain(TableName())); + } + + [Test] + public async Task A_target_with_a_required_checkpoint_row_is_not_judged() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await SeedCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete); + + Assert.DoesNotThrowAsync(() => TargetCheck.Run()); + } + + [Test] + public async Task A_row_under_an_id_this_build_does_not_know_counts_as_a_started_copy() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await SeedCheckpoint("SomeCategoryFromANewerBuild", MigrationCategoryState.InProgress); + + Assert.DoesNotThrowAsync(() => TargetCheck.Run()); + } + + // --migration-abandon writes this row before any copy has run, so on a database that already served on SQL it must not switch the check off. + [Test] + public async Task An_abandoned_optional_row_alone_does_not_skip_the_check() + { + await EndpointSettingsStore.UpdateEndpointSettings(new EndpointSettings { Name = "Sales", TrackInstances = true }); + await SeedCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.Abandoned); + + var exception = Assert.ThrowsAsync(() => TargetCheck.Run()); + + Assert.That(exception.Message, Does.Contain(TableName())); + } + + [Test] + public void Every_mapped_entity_but_the_checkpoint_is_judged() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var mapped = dbContext.Model.GetEntityTypes().Select(type => type.ClrType).Where(type => type != typeof(MigrationCheckpointEntity)); + + Assert.That(TargetHoldsNoServiceControlDataCheck.Tables.Select(entry => entry.Entity), Is.EquivalentTo(mapped)); + } + + IMigrationStartupCheck TargetCheck => Readiness.ContributedChecks().OfType().Single(); + + Task SeedCheckpoint(string categoryId, MigrationCategoryState state) => + ServiceProvider.GetRequiredService().Upsert(new MigrationCheckpoint(categoryId, state, null, 0, 0, null, null, null, null, null, null)); + + string TableName() + { + using var scope = ServiceProvider.CreateScope(); + + return scope.ServiceProvider.GetRequiredService().Model.FindEntityType(typeof(T))!.GetTableName()!; + } + + [Test] + public async Task The_schema_check_refuses_a_database_whose_migrations_have_not_been_applied() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + // EF's own history repository, because it is the only thing that knows where the history table is once the persister is given a schema. + var history = dbContext.GetService(); + + foreach (var row in await history.GetAppliedMigrationsAsync()) + { + await dbContext.Database.ExecuteSqlRawAsync(history.GetDeleteScript(row.MigrationId)); + } + + var exception = Assert.ThrowsAsync(() => Readiness.ContributedChecks().OfType().Single().Run()); + + Assert.That(exception.Message, Does.Contain(dbContext.Database.GetMigrations().First()).And.Contain("--setup")); + } + + [Test] + public async Task The_body_storage_check_leaves_no_probe_body_behind() + { + var bodyStorage = ServiceProvider.GetRequiredService(); + + await Readiness.ContributedChecks().Single(check => check.Name == "message body storage is writable").Run(); + + Assert.That(await bodyStorage.ReadBody("migration-writable-probe"), Is.Null, "a probe left in the store is a body the source never had, which a count comparison reads as an extra rather than a loss"); + } + + [Test] + public void The_body_storage_check_refuses_a_store_that_cannot_write() + { + var parentThatIsAFile = Path.Combine(Path.GetTempPath(), $"sc-not-a-directory-{Guid.NewGuid():n}"); + File.WriteAllText(parentThatIsAFile, string.Empty); + + try + { + var check = new BodyStorageIsWritableCheck(new FileSystemBodyStoragePersistence( + new FileSystemBodyStorageSettings { StoragePath = Path.Combine(parentThatIsAFile, "bodies") })); + + Assert.CatchAsync(() => check.Run()); + } + finally + { + File.Delete(parentThatIsAFile); + } + } + + // The copy runs before any hosted service starts, so a target that only works once one has started is broken exactly when the migration needs it. + [Test] + public async Task The_target_answers_from_a_container_whose_hosted_services_have_not_started() + { + var (host, context) = await BuildHostWithoutStarting(); + + try + { + await using var scope = host.Services.GetRequiredService().CreateAsyncScope(); + + Assert.That(await scope.ServiceProvider.GetRequiredService().EndpointSettings.LongCountAsync(), Is.Zero); + Assert.DoesNotThrowAsync(() => host.Services.GetRequiredService().ContributedChecks().Single(check => check.Name == "message body storage is writable").Run()); + } + finally + { + await context.TearDown(); + host.Dispose(); + } + } + + // PersistenceTestBase already started a host in SetUp, so this builds a second one over its own test database. + static async Task<(IHost Host, PersistenceTestsContext Context)> BuildHostWithoutStarting() + { + var context = new PersistenceTestsContext(); + var hostBuilder = Host.CreateApplicationBuilder(); + + await context.Setup(hostBuilder); + var host = hostBuilder.Build(); + await context.InstallSchema(host); + + return (host, context); + } + + [Test] + public async Task The_host_opened_record_is_absent_until_written_and_stays_after_a_second_write() + { + var readiness = ServiceProvider.GetRequiredService(); + + Assert.That(await readiness.HasHostOpened(), Is.False, "a target nothing has opened on must not claim otherwise"); + + await readiness.RecordHostOpened(); + var firstOpened = Now; + + AdvanceClock(TimeSpan.FromDays(7)); + await readiness.RecordHostOpened(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await readiness.HasHostOpened(), Is.True); + // The stored instant is what tells an operator the clean abort is over, so a later start must not move it forward. + Assert.That(await ReadHostOpenedAt(), Is.EqualTo(firstOpened)); + } + } + + async Task ReadHostOpenedAt() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + return await dbContext.GetSetting(SettingKeys.MigrationHostOpenedOnTarget); + } +} diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs new file mode 100644 index 0000000000..5261d9fffd --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTableTests.cs @@ -0,0 +1,172 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class MigrationCheckpointTableTests : PersistenceTestBase +{ + [Test] + public async Task The_table_round_trips_every_column_written_directly_through_the_DbContext() + { + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + dbContext.MigrationCheckpoints.Add(new MigrationCheckpointEntity + { + CategoryId = "EndpointSettings", + State = MigrationCategoryState.InProgress, + Cursor = "cursor-1", + CopiedCount = 3, + SkippedCount = 1, + SourceTotal = 40, + SkipReasons = new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 }, + StartedAt = Now, + LastProgressAt = Now, + SettledAt = null, + LastError = null, + AlreadyPresentCount = 36, + Version = 7 + }); + await dbContext.SaveChangesAsync(); + } + + using var readScope = ServiceProvider.CreateScope(); + var readContext = readScope.ServiceProvider.GetRequiredService(); + var stored = await readContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(stored.Cursor, Is.EqualTo("cursor-1")); + Assert.That(stored.CopiedCount, Is.EqualTo(3)); + Assert.That(stored.SkippedCount, Is.EqualTo(1)); + Assert.That(stored.SourceTotal, Is.EqualTo(40)); + Assert.That(stored.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); + Assert.That(stored.AlreadyPresentCount, Is.EqualTo(36)); + Assert.That(stored.Version, Is.EqualTo(7)); + Assert.That(stored.StartedAt, Is.EqualTo(Now)); + Assert.That(stored.LastProgressAt, Is.EqualTo(Now)); + Assert.That(stored.SettledAt, Is.Null); + Assert.That(stored.LastError, Is.Null); + } + } + + [Test] + public async Task Every_timestamp_column_comes_back_as_a_Utc_DateTime() + { + Assert.That(Now.Kind, Is.EqualTo(DateTimeKind.Utc), "the engine stamps timings from TimeProvider.GetUtcNow().UtcDateTime, so the test clock has to match"); + + using (var scope = ServiceProvider.CreateScope()) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + dbContext.MigrationCheckpoints.Add(new MigrationCheckpointEntity + { + CategoryId = "KnownEndpoints", + State = MigrationCategoryState.Complete, + Cursor = null, + CopiedCount = 1, + SkippedCount = 0, + SourceTotal = 1, + SkipReasons = null, + StartedAt = Now, + LastProgressAt = Now, + SettledAt = Now, + LastError = null + }); + await dbContext.SaveChangesAsync(); + } + + using var readScope = ServiceProvider.CreateScope(); + var readContext = readScope.ServiceProvider.GetRequiredService(); + var stored = await readContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "KnownEndpoints"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.StartedAt!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(stored.LastProgressAt!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + Assert.That(stored.SettledAt!.Value.Kind, Is.EqualTo(DateTimeKind.Utc)); + } + } + + [Test] + public void The_skip_reasons_JSON_is_keyed_by_reason_name_not_by_its_number() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var converter = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!.FindProperty("SkipReasons")!.GetValueConverter()!; + + var json = (string)converter.ConvertToProvider(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 2 })!; + + // Enum.Parse accepts "0" as readily as "BodyUnreadable", so a round trip cannot tell the two encodings apart. + Assert.That(json, Does.Contain("BodyUnreadable")); + } + + [Test] + public void A_blank_group_comment_skip_round_trips_by_name() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var converter = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!.FindProperty("SkipReasons")!.GetValueConverter()!; + + var json = (string)converter.ConvertToProvider(new Dictionary { [MigrationSkipReason.BlankGroupComment] = 3 })!; + var reasons = (IReadOnlyDictionary)converter.ConvertFromProvider(json)!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(json, Does.Contain("BlankGroupComment")); + Assert.That(reasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BlankGroupComment] = 3 })); + } + } + + [Test] + public void A_reason_name_this_build_does_not_know_is_read_as_Unknown_rather_than_throwing() + { + // The rows a newer build wrote are what a downgraded instance meets, and the read path that would + // throw is ReadAll, which is what decides whether the host may start. + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var converter = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!.FindProperty("SkipReasons")!.GetValueConverter()!; + + var reasons = (IReadOnlyDictionary)converter.ConvertFromProvider( + """{"BodyUnreadable":2,"SomethingFromTheFuture":3,"AlsoFromTheFuture":1}""")!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(reasons[MigrationSkipReason.BodyUnreadable], Is.EqualTo(2)); + Assert.That(reasons[MigrationSkipReason.Unknown], Is.EqualTo(4), "several unknown names collapse onto the one member, so their counts are summed rather than one of them winning"); + Assert.That(reasons, Has.Count.EqualTo(2)); + } + } + + [Test] + public void The_table_carries_exactly_the_columns_the_checkpoint_needs() + { + // The one thing the one-way door makes permanent. Adding, dropping or renaming one after this + // ships costs a migration against customer data, so it fails here rather than in a customer's + // database. Property names rather than column names, because each provider spells the columns + // its own way and the spelling is a convention the generated migrations already pin. + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var entityType = dbContext.Model.FindEntityType(typeof(MigrationCheckpointEntity))!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(entityType.GetProperties().Select(property => property.Name), Is.EquivalentTo(new[] + { + "CategoryId", "State", "Cursor", "CopiedCount", "SkippedCount", "SourceTotal", + "SkipReasons", "StartedAt", "LastProgressAt", "SettledAt", "LastError", "AlreadyPresentCount", "Version", "StartedWindowSeconds" + })); + + // Underscores stripped so one assertion covers MigrationCheckpoints and migration_checkpoints. + Assert.That(entityType.GetTableName()!.Replace("_", ""), Is.EqualTo("MigrationCheckpoints").IgnoreCase); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs new file mode 100644 index 0000000000..706f41f046 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests/EFCore/MigrationCheckpointTransactionTests.cs @@ -0,0 +1,178 @@ +namespace ServiceControl.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.EFCore; +using ServiceControl.Persistence.EFCore.DbContexts; +using ServiceControl.Persistence.EFCore.Entities; + +class MigrationCheckpointTransactionTests : PersistenceTestBase +{ + static MigrationCheckpoint Checkpoint(DateTime now, string cursor, long copied, long version = 0) => + new("EndpointSettings", MigrationCategoryState.InProgress, cursor, copied, 0, 40, null, now, now, null, null, Version: version); + + async Task SeedCheckpoint(string cursor, long copied) + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + await dbContext.UpsertCheckpoint(Checkpoint(Now, cursor, copied)); + } + + [Test] + public async Task A_failure_before_commit_leaves_neither_the_row_nor_the_checkpoint() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + var thrown = Assert.ThrowsAsync(() => strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-not-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-1", 1)); + + throw new InvalidOperationException("simulated failure just before commit"); + })); + + // Without this the test also passes when UpsertCheckpoint throws before writing anything. + Assert.That(thrown!.Message, Is.EqualTo("simulated failure just before commit")); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.False); + Assert.That(await verifyContext.MigrationCheckpoints.AnyAsync(c => c.CategoryId == "EndpointSettings"), Is.False); + } + } + + [Test] + public async Task A_successful_commit_lands_the_row_and_the_checkpoint_together() + { + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + await strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-1", 1)); + + await transaction.CommitAsync(); + }); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.True); + Assert.That(await verifyContext.MigrationCheckpoints.AnyAsync(c => c.CategoryId == "EndpointSettings"), Is.True); + } + } + + [Test] + public async Task An_existing_checkpoint_is_advanced_by_a_commit_inside_the_callers_transaction() + { + await SeedCheckpoint("cursor-1", 1); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + await strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-2", 2, version: 1)); + + await transaction.CommitAsync(); + }); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService(); + var stored = await verifyContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.True); + Assert.That(stored.Cursor, Is.EqualTo("cursor-2")); + Assert.That(stored.CopiedCount, Is.EqualTo(2)); + } + } + + [Test] + public async Task An_existing_checkpoint_keeps_its_previous_cursor_when_the_transaction_rolls_back() + { + await SeedCheckpoint("cursor-1", 1); + + using var scope = ServiceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var strategy = dbContext.Database.CreateExecutionStrategy(); + + var thrown = Assert.ThrowsAsync(() => strategy.ExecuteAsync(async () => + { + await using var transaction = await dbContext.Database.BeginTransactionAsync(); + + dbContext.Settings.Add(new SettingEntity { Key = "migration-test-row", Value = "should-not-survive" }); + await dbContext.SaveChangesAsync(); + + await dbContext.UpsertCheckpoint(Checkpoint(Now, "cursor-2", 2, version: 1)); + + throw new InvalidOperationException("simulated failure just before commit"); + })); + + // Without this the test also passes when UpsertCheckpoint throws before writing anything. + Assert.That(thrown!.Message, Is.EqualTo("simulated failure just before commit")); + + using var verifyScope = ServiceProvider.CreateScope(); + var verifyContext = verifyScope.ServiceProvider.GetRequiredService(); + var stored = await verifyContext.MigrationCheckpoints.AsNoTracking().SingleAsync(c => c.CategoryId == "EndpointSettings"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await verifyContext.Settings.AnyAsync(s => s.Key == "migration-test-row"), Is.False); + Assert.That(stored.Cursor, Is.EqualTo("cursor-1")); + Assert.That(stored.CopiedCount, Is.EqualTo(1)); + } + } + + [Test] + public async Task A_save_built_from_an_out_of_date_copy_is_refused_rather_than_silently_winning() + { + var store = ServiceProvider.GetRequiredService(); + await store.Upsert(Checkpoint(Now, "cursor-1", 1)); + + // Both read the row at the same version, as a background copy and a required copy on two hosts would. + var first = (await store.Read("EndpointSettings"))!; + var stale = (await store.Read("EndpointSettings"))!; + + await store.Upsert(first with { Cursor = "cursor-first", CopiedCount = 2 }); + + Assert.ThrowsAsync(() => store.Upsert(stale with { Cursor = "cursor-stale", CopiedCount = 99 })); + + var stored = (await store.Read("EndpointSettings"))!; + + using (Assert.EnterMultipleScope()) + { + Assert.That(stored.Cursor, Is.EqualTo("cursor-first")); + Assert.That(stored.CopiedCount, Is.EqualTo(2)); + Assert.That(stored.Version, Is.EqualTo(first.Version + 1)); + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs b/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs index 15d9b76981..5e238f2d22 100644 --- a/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs +++ b/src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs @@ -10,6 +10,13 @@ public interface IPersistenceTestsContext { Task Setup(IHostApplicationBuilder hostBuilder); + /// + /// Puts the schema in place on the built host, before it is started, the way --setup does in + /// production. The EF Core persisters refuse to start against a database whose schema predates the + /// build, and that check runs before any hosted service, so migrating after the start is too late. + /// + Task InstallSchema(IHost host); + Task PostSetup(IHost host); Task TearDown(); diff --git a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs index f065c190f6..dc5ae1178e 100644 --- a/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs +++ b/src/ServiceControl.Persistence.Tests/PersistenceTestBase.cs @@ -53,6 +53,7 @@ public async Task SetUp() host = hostBuilder.Build(); + await PersistenceTestsContext.InstallSchema(host); await host.StartAsync(); await PersistenceTestsContext.PostSetup(host); } diff --git a/src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs b/src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs new file mode 100644 index 0000000000..7bf02d735b --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/CheckpointMigrationState.cs @@ -0,0 +1,42 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +/// +/// Answers from the saved checkpoints, read once before the host opens rather than queried live. +/// +public sealed class CheckpointMigrationState(IMigrationCheckpointStore? checkpointStore = null) : IMigrationState +{ + volatile bool anyCategoryIncomplete; + IReadOnlyCollection selectedCategoryIds = []; + + public bool AnyCategoryIncomplete => anyCategoryIncomplete; + + /// + /// Reads every checkpoint and fixes the answer for the life of the host. + /// + /// The required categories this instance copies, which are the only ones that hold back the services that delete or overwrite copied data. They are passed in because the store holds a row only for a category some run has already been given. + public async Task Seed(IReadOnlyCollection selectedIds, CancellationToken cancellationToken = default) + { + // Only a persister that can be migrated into keeps checkpoints, so no store means no migration has run. + if (checkpointStore is null) + { + return; + } + + selectedCategoryIds = selectedIds; + + Recompute(await checkpointStore.ReadAll(cancellationToken)); + } + + /// + /// Works the answer out again from the checkpoints given, without reading the store. + /// + /// Every checkpoint the store holds. A selected category with no checkpoint among them has not started, which is not finished. + public void Recompute(IReadOnlyList checkpoints) => + anyCategoryIncomplete = selectedCategoryIds.Any(id => + checkpoints.FirstOrDefault(checkpoint => checkpoint.CategoryId == id)?.State.IsFinished() != true); +} diff --git a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs new file mode 100644 index 0000000000..534e120b7d --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// +/// The rule that stops a category early because it has lost too much to keep copying. +/// +public static class HaltThreshold +{ + /// + /// Whether the skipped rows are enough to stop the category. Both limits have to be passed, so the floor + /// ignores a few bad rows in a small category and the percentage ignores a small share of a large one. + /// + /// Rows skipped as faults. A skip the product would have dropped anyway does not belong here. + /// Rows dealt with over the same stretch as , copied, skipped and already present alike. + /// The share of skipped rows, as a percentage, that has to be passed. + /// The number of skipped rows that has to be passed before the percentage counts at all. + /// True when both limits are passed, which means the category stops. + public static bool Exceeded(long skippedCount, long totalCount, int percentThreshold, int minimumFloor) + { + if (skippedCount <= minimumFloor || totalCount == 0) + { + return false; + } + + var percent = skippedCount * 100m / totalCount; + return percent > percentThreshold; + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs new file mode 100644 index 0000000000..0731525dab --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs @@ -0,0 +1,146 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +/// +/// Where one category has got to. says which of these +/// let the host open, and which wait for the operator. A +/// start picks up every category that is neither. +/// +public enum MigrationCategoryState +{ + /// No run has read a row of this category yet. + NotStarted, + + /// + /// A run is copying this category, or a run stopped without settling it. On an optional category, a LastError + /// means an exception stopped it or a start could not open the source, and the next start resumes it. + /// + InProgress, + + /// + /// The category reached the end of the source with no fault skips. Harmless ones stay counted. + /// + Complete, + + /// + /// The category reached the end of the source with fault skips, so it is Failed until the operator retries or abandons it. + /// + CompleteWithErrors, + + /// + /// The category stopped early. It waits for --migration-retry or --migration-abandon, and no start re-reads it. + /// + Halted, + + /// + /// The operator gave up on a Failed or started required category, on any optional one, or on one whose rows + /// depend on such a category. It is final: nothing copies it again, even in a later migration. + /// + Abandoned, + + /// + /// This category follows a category that is not Done or Abandoned, so it did not run. A restart clears it once that one is. + /// + Blocked +} + +/// +/// A category's saved progress: where a restart carries on from, and what the status and verify commands report. +/// Every count is the total across every run, not this run alone. +/// +/// The point the last committed batch reached. A restart reads the source after it. Null means nothing has been read. +/// No longer written: the engine does not count the source, so this is null on every row it saves. The column stays only because dropping it is a schema change. +/// How many rows each reason skipped. The counts here add up to . +/// The moment the category stopped running, whatever state it stopped in. Read it beside , because a halt settles too. +/// Why the category stopped or did not run, in the words the operator is shown: a halt, a category it must follow that is not finished, an exception that left an optional category copying, or a source a start could not open, on an optional category still copying or not started. It stays until a start runs the category again. +/// Rows the target already held, so they were neither copied nor skipped. They still count as accounted for. +/// The optimistic concurrency token, which is the guard against two writers. A store sets it on save and refuses a checkpoint carrying a value the stored row no longer holds. +/// The window, in whole seconds, an optional category started with. It is null until the copier writes it, and always null for a required category. +public sealed record MigrationCheckpoint( + string CategoryId, + MigrationCategoryState State, + string? Cursor, + long CopiedCount, + long SkippedCount, + long? SourceTotal, + IReadOnlyDictionary? SkipReasons, + DateTime? StartedAt, + DateTime? LastProgressAt, + DateTime? SettledAt, + string? LastError, + long AlreadyPresentCount = 0, + long Version = 0, + long? StartedWindowSeconds = null) +{ + /// + /// Adds one batch's outcome to this checkpoint. A target calls it inside the transaction that writes the rows, + /// so the saved counts are the real ones. + /// + /// Rows this batch wrote. + /// Rows this batch could not write. Every one of them needs a reason. + /// Rows this batch found the target already held. + /// How many rows each reason skipped in this batch. + /// A copy with this batch's counts and reasons added to the totals. + /// The reasons do not add up to , which would leave the verify command unable to account for a row. + public MigrationCheckpoint Extend(int copied, int skipped, int alreadyPresent, IReadOnlyDictionary? skipReasons) + { + var explained = skipReasons?.Values.Sum() ?? 0; + if (explained != skipped) + { + throw new InvalidOperationException($"The target reported {skipped} skipped rows in category {CategoryId} but gave reasons for {explained}. Every skipped row needs a reason, or --migration-verify cannot account for it."); + } + + return this with + { + CopiedCount = CopiedCount + copied, + SkippedCount = SkippedCount + skipped, + AlreadyPresentCount = AlreadyPresentCount + alreadyPresent, + SkipReasons = AddSkipReasons(SkipReasons, skipReasons) + }; + } + + internal static IReadOnlyDictionary? AddSkipReasons(IReadOnlyDictionary? totals, IReadOnlyDictionary? additions) + { + if (additions is not { Count: > 0 }) + { + return totals; + } + + Dictionary sum = totals is null ? [] : new(totals); + foreach (var (reason, count) in additions) + { + sum[reason] = sum.GetValueOrDefault(reason) + count; + } + + return sum; + } +} + +/// +/// Where the checkpoints are kept. The target database holds them, so progress and the rows it describes +/// commit together. +/// +public interface IMigrationCheckpointStore +{ + /// + /// Every checkpoint the store holds. A run saves a not-started row for every category it will copy before it + /// copies any of them, so a category is absent when no run has recorded it. + /// + Task> ReadAll(CancellationToken cancellationToken = default); + + /// + /// One category's checkpoint, or null when no run has recorded it. + /// + Task Read(string categoryId, CancellationToken cancellationToken = default); + + /// + /// Saves the checkpoint and returns it as stored, carrying the version the save landed on. + /// + /// The stored row has moved on, which means another writer saved it, so this save is refused. + Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs new file mode 100644 index 0000000000..7e969ffa5f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// +/// Implemented by a persister that can be read as the old database a migration copies from. Nothing here +/// writes to the source, because the old database has to stay usable if the migration is thrown away. +/// +public interface IMigrationSource : IAsyncDisposable +{ + /// + /// Connects to the source read-only. Every other member except throws + /// until this has run. + /// + Task Open(CancellationToken cancellationToken = default); + + /// + /// The checks this source wants run before the copy starts, in the order they must run. Call it after Open. + /// + IReadOnlyList ContributedChecks(); + + /// + /// What the source report and the dry run print about the source. + /// + Task Describe(CancellationToken cancellationToken = default); + + /// + /// A count of everything the source holds, including data no category copies. + /// + Task> Inventory(CancellationToken cancellationToken = default); + + /// + /// How many rows the source holds for one category, which --migration-verify reports. + /// + Task Count(MigrationCategory category, CancellationToken cancellationToken = default); + + /// + /// Reads a category in batches, after the checkpoint cursor, or from the start when it is null. Throws on a cursor it never issued. + /// + /// A ceiling, not a target: returning fewer costs nothing, returning more fails the target's write. + IAsyncEnumerable Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + CancellationToken cancellationToken = default); + + /// + /// Reads one row's message body when Read did not attach it. Returns null when the row has no body. + /// + Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default); + + /// + /// The categories this source can read. A category outside this set is never handed to the engine. + /// Answers before Open and does not change across it. + /// + IReadOnlyCollection SupportedCategoryIds { get; } + + /// + /// The type of in every row this source reads, for each category in + /// . The target's writer for the same category must take that type. + /// Answers before Open and does not change across it. + /// + IReadOnlyDictionary DocumentTypes { get; } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs new file mode 100644 index 0000000000..1f4de16640 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs @@ -0,0 +1,10 @@ +namespace ServiceControl.Persistence.DataMigration; + +using ServiceControl.Configuration; + +/// Implemented by a persister that can be read as the old database a migration copies from. +public interface IMigrationSourceFactory +{ + /// Builds the source from the instance's own settings, without connecting to it. + IMigrationSource CreateSource(SettingsRootNamespace settingsRoot); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs new file mode 100644 index 0000000000..bd08c1df27 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationStartupCheck.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Threading; +using System.Threading.Tasks; + +/// +/// One check the host runs before the copy starts. The host, the source and the target each contribute their own. +/// +public interface IMigrationStartupCheck +{ + /// + /// What the check is called in the refusal the host prints, so it is phrased to finish the sentence + /// "Migration startup check '...' failed". + /// + string Name { get; } + + /// + /// Runs the check. Throws when it fails, which stops the host before anything is copied. + /// + Task Run(CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationState.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationState.cs new file mode 100644 index 0000000000..212e8694a5 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationState.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// +/// Whether this instance is still part way through copying its old database in. +/// +public interface IMigrationState +{ + /// + /// True while any required category is unfinished. Services that delete or overwrite copied data stand down while it + /// is true. Optional categories never make it true, because they copy beside those services. + /// + bool AnyCategoryIncomplete { get; } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs new file mode 100644 index 0000000000..1e457a27a8 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -0,0 +1,64 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// +/// Implemented by a persister that can be the new database a migration copies into. +/// +public interface IMigrationTarget +{ + /// + /// Makes the target ready to write. The host calls it before the copy and after the target's own checks have passed. + /// + Task Open(CancellationToken cancellationToken = default); + + /// + /// The most rows a source may return in one batch for this category. + /// The target picks it because its own database sets the limit, and a bigger batch fails the write. + /// + Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default); + + /// + /// Saves the batch's rows and the checkpoint in one transaction, so progress never gets ahead of the data. + /// Add this batch's own outcome with and save that, so the stored counts are never a guess. + /// + /// Prior totals, the cursor this batch reached, and any rows the engine itself skipped. Nothing the target does is counted yet. + Task Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + CancellationToken cancellationToken = default); + + /// + /// How many rows the target holds for one category, for progress and verify. + /// Counts only that category, even where two categories share a table. + /// + Task Count(MigrationCategory category, CancellationToken cancellationToken = default); + + /// + /// The categories this target can write. A category outside this set is never handed to the engine. + /// Answers before Open and does not change across it. + /// + IReadOnlyCollection SupportedCategoryIds { get; } + + /// + /// The type of this target takes, for each category in + /// . A row holding any other type fails its batch with an + /// . Answers before Open and does not change across it. + /// + IReadOnlyDictionary DocumentTypes { get; } +} + +/// +/// What the target did with one batch, and the checkpoint it committed alongside the rows. Every skipped row must have a reason in SkipReasons. +/// +/// The checkpoint as the target stored it, carrying the version that save landed on. The engine carries on from this one, never from the one it passed in. +/// Rows this batch wrote. The same number must show up as the rise in the saved copied count, because the halt threshold reads one and status and verify read the other. +/// Rows this batch could not write, benign ones included. +/// The source ids of those rows, so the engine can name each one in the log. +/// Rows the target already held. They were neither copied nor skipped, and they still count as accounted for. +/// How many rows each reason skipped. The counts have to add up to Skipped, or the checkpoint refuses the batch. +public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null); diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs new file mode 100644 index 0000000000..8cde500f8f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTargetReadiness.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// +/// The target's own startup checks, plus a marker kept in the target database saying a host has already started on it. +/// +public interface IMigrationTargetReadiness +{ + /// + /// The checks this target wants run before the copy starts, in the order they must run. + /// + IReadOnlyList ContributedChecks(); + + /// + /// Stamps the marker the first time a host opens on the target, and leaves that first stamp alone on every start after it. + /// + Task RecordHostOpened(CancellationToken cancellationToken = default); + + /// + /// Whether a host has already opened on the target, which is what makes discarding a partial copy a loss rather than a clean abort. + /// + Task HasHostOpened(CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs new file mode 100644 index 0000000000..7a36324b40 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs @@ -0,0 +1,41 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; + +/// +/// When a category is copied. Required categories are copied with ServiceControl closed, because the host must +/// not serve a half copied instance. Optional ones are copied in the background once it is open. +/// +public enum MigrationCategoryKind { Required, Optional } + +/// One kind of data, such as endpoint settings, copied as a unit and resumed from its own cursor. +/// The name in , which is also the key of its checkpoint row. +/// Whether its rows have message bodies, which the engine fetches separately when the source does not attach them. +/// The order within one kind, counting from 1. Required and optional both start at 1, so the two lists are never sorted together. +/// The category that has to finish first, or null when nothing has to. A category whose predecessor is unfinished is recorded as instead of running. +public sealed record MigrationCategory( + string Id, + MigrationCategoryKind Kind, + bool CarriesBodies, + int Order, + string? MustFollow = null); + +/// A message body read from the source, as bytes plus its content type. +public sealed record MigrationBody(ReadOnlyMemory Content, string ContentType); + +/// One item read from the source. +/// The row's identifier in the old database. It is what the engine logs when the row is skipped, and what takes. +/// The row itself, as the object the source read. The target's writer for that category is what knows the type. +/// Facts about the row that the document does not carry. Empty when the source has none to add. +/// The message body, or null when the source did not attach one and the engine has not fetched it. +public sealed record MigrationRow( + string SourceId, + object Document, + IReadOnlyDictionary Metadata, + MigrationBody? Body = null); + +/// Rows read from the source together, plus the cursor to resume after them. +/// The rows, which can be empty when every document in this stretch turned into no row. The cursor past them still has to be saved. +/// Where the source got to. A later read handed this back carries on after it. +public sealed record MigrationBatch(IReadOnlyList Rows, string Cursor); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs new file mode 100644 index 0000000000..fadc585355 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// +/// The name of every category a migration can copy. The name is what the checkpoint row is keyed on, so renaming +/// one strands the progress already saved under the old name. says when each one is copied. +/// +public static class MigrationCategoryIds +{ + public const string KnownEndpoints = nameof(KnownEndpoints); + public const string EndpointSettings = nameof(EndpointSettings); + public const string MessageRedirects = nameof(MessageRedirects); + public const string Subscriptions = nameof(Subscriptions); + public const string NotificationSettings = nameof(NotificationSettings); + public const string TrialEndDate = nameof(TrialEndDate); + public const string RetryOperations = nameof(RetryOperations); + public const string LicensingEndpoints = nameof(LicensingEndpoints); + public const string LicensingThroughput = nameof(LicensingThroughput); + public const string LicensingReportMasks = nameof(LicensingReportMasks); + public const string LicensedEndpointDetails = nameof(LicensedEndpointDetails); + public const string UnresolvedAndRetryIssuedFailedMessages = nameof(UnresolvedAndRetryIssuedFailedMessages); + + public const string EventLog = nameof(EventLog); + public const string CustomChecks = nameof(CustomChecks); + public const string FailedErrorImports = nameof(FailedErrorImports); + public const string ArchivedAndResolvedFailedMessages = nameof(ArchivedAndResolvedFailedMessages); + public const string GroupComments = nameof(GroupComments); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs new file mode 100644 index 0000000000..13e8a0fbca --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs @@ -0,0 +1,51 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.MessageFailures; + +/// +/// Every category a migration can copy, and when each one is copied. This list is the whole of what a migration +/// covers: a source or a target that cannot handle a category says so through its own supported ids, and the +/// engine never invents one. +/// +public static class MigrationCategoryRegistry +{ + /// + /// The failed message statuses that are copied before the host opens. A message in one of these is waiting on + /// somebody, so an operator who cannot see it after the cutover has lost work. + /// + public static readonly IReadOnlyList UnresolvedAndRetryIssuedStatuses = [FailedMessageStatus.Unresolved, FailedMessageStatus.RetryIssued]; + + /// + /// The failed message statuses that are copied in the background. These are history, so the instance is usable + /// while they are still arriving. + /// + public static readonly IReadOnlyList ArchivedAndResolvedStatuses = [FailedMessageStatus.Archived, FailedMessageStatus.Resolved]; + + public static readonly IReadOnlyList All = + [ + // Required, copied with ServiceControl closed. + new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryKind.Required, CarriesBodies: false, Order: 1), + new(MigrationCategoryIds.EndpointSettings, MigrationCategoryKind.Required, CarriesBodies: false, Order: 2, MustFollow: MigrationCategoryIds.KnownEndpoints), + new(MigrationCategoryIds.MessageRedirects, MigrationCategoryKind.Required, CarriesBodies: false, Order: 3), + new(MigrationCategoryIds.Subscriptions, MigrationCategoryKind.Required, CarriesBodies: false, Order: 4), + new(MigrationCategoryIds.NotificationSettings, MigrationCategoryKind.Required, CarriesBodies: false, Order: 5), + new(MigrationCategoryIds.TrialEndDate, MigrationCategoryKind.Required, CarriesBodies: false, Order: 6), + new(MigrationCategoryIds.RetryOperations, MigrationCategoryKind.Required, CarriesBodies: false, Order: 7), + new(MigrationCategoryIds.LicensingEndpoints, MigrationCategoryKind.Required, CarriesBodies: false, Order: 8), + new(MigrationCategoryIds.LicensingThroughput, MigrationCategoryKind.Required, CarriesBodies: false, Order: 9, MustFollow: MigrationCategoryIds.LicensingEndpoints), + new(MigrationCategoryIds.LicensingReportMasks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 10), + new(MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryKind.Required, CarriesBodies: false, Order: 11), + new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryKind.Required, CarriesBodies: true, Order: 12), + new(MigrationCategoryIds.CustomChecks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 14), + new(MigrationCategoryIds.FailedErrorImports, MigrationCategoryKind.Required, CarriesBodies: true, Order: 15), + new(MigrationCategoryIds.GroupComments, MigrationCategoryKind.Required, CarriesBodies: false, Order: 16, MustFollow: MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages), + + // Optional, copied in the background once the host is open. + new(MigrationCategoryIds.EventLog, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 1), + new(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 2), + ]; + + public static MigrationCategory? Find(string id) => All.FirstOrDefault(c => c.Id == id); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs new file mode 100644 index 0000000000..0a6ae5bb2c --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryStateExtensions.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Persistence.DataMigration; + +public static class MigrationCategoryStateExtensions +{ + /// + /// Whether the category is done with, so a restart passes over it and it no longer holds the host back: Done, + /// or Abandoned by the operator. A Failed category () is not finished. It waits for + /// --migration-retry or --migration-abandon, and a category that must follow it stays blocked until then. + /// + public static bool IsFinished(this MigrationCategoryState state) => + state is MigrationCategoryState.Complete + or MigrationCategoryState.Abandoned; + + /// + /// Whether the category is Failed: it stopped early, or reached the end having skipped rows to faults. It waits + /// for the operator to run --migration-retry or --migration-abandon, and no start copies it until then. + /// + public static bool IsFailed(this MigrationCategoryState state) => + state is MigrationCategoryState.Halted + or MigrationCategoryState.CompleteWithErrors; +} + +public static class MigrationSkipReasonExtensions +{ + /// + /// Whether the running product would have dropped this row anyway, which makes the skip no loss. The halt + /// threshold and the settle rule never count a harmless skip, so no number of them can stop a category or + /// leave it Failed. The list is fixed. is not on it, because a + /// newer build's reason read back here may be a real loss. + /// + public static bool IsBenign(this MigrationSkipReason reason) => + reason is MigrationSkipReason.PastRetention + or MigrationSkipReason.BlankGroupComment; + + /// + /// Whether no retry can fix a row skipped for this reason, because the row itself can never be stored. + /// + public static bool IsPermanent(this MigrationSkipReason reason) => reason is MigrationSkipReason.RequiredValueMissing; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs new file mode 100644 index 0000000000..cc2822822f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs @@ -0,0 +1,6 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; + +/// Thrown when a checkpoint save carries a version the stored row no longer holds, because another writer moved it on. +public sealed class MigrationCheckpointConflictException(string message, Exception? innerException = null) : Exception(message, innerException); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs new file mode 100644 index 0000000000..bf975e1e21 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -0,0 +1,321 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +/// +/// Copies one category at a time from the source to the target, and records where it got to after every batch. +/// The engine knows nothing about either database: what a row is, how it is written and how big a batch can be +/// all come from the source and the target. A category that fails comes back as a Failed checkpoint rather than +/// an exception, and no later run copies it until the operator retries or abandons it. An optional category an +/// exception stopped comes back still copying instead, with the error on its row. A shutdown and a checkpoint +/// conflict come out as exceptions, because neither is the category's fault, and so does a failure to save the +/// checkpoint a category stopped or settled on, because then there is no row to record it on. +/// +public sealed class MigrationEngine( + IMigrationSource source, + IMigrationTarget target, + IMigrationCheckpointStore checkpointStore, + TimeProvider timeProvider, + MigrationEngineOptions options, + ILogger logger) +{ + /// How many times a message body is read before the row is skipped as unreadable. + public const int MaxBodyReadAttempts = 3; + + /// + /// The categories of one kind to copy, in the order to copy them. Optional categories the operator did not + /// ask for are left out. + /// + public IReadOnlyList SelectCategories(MigrationCategoryKind kind) => + MigrationCategoryRegistry.All + .Where(c => c.Kind == kind) + .Where(c => kind == MigrationCategoryKind.Required || options.SelectedOptionalCategoryIds.Contains(c.Id)) + .OrderBy(c => c.Order) + .ToArray(); + + /// + /// Copies the categories one after another and returns where each one ended, in the same order. A category + /// that halts does not stop the ones after it. Before copying anything it saves a not-started checkpoint for + /// every category that has none, so a copy stopped between two categories still lists the ones it never reached. + /// + /// Another writer saved one of these checkpoints, which means a second instance is copying into the same database. + /// The host is shutting down. + /// Saving the checkpoint a category stopped or settled on failed. Its stored row is the last one saved, and the categories after it were not run. + // Runs in the order given without re-sorting: required and optional orders both start at 1, so + // sorting a mixed list would put an optional category in front of a required one. + public async Task> RunCategories( + IReadOnlyList categories, + CancellationToken cancellationToken = default) + { + // The gates that keep a host off an unfinished copy read only the rows that exist. + foreach (var category in categories) + { + if (await checkpointStore.Read(category.Id, cancellationToken) is null) + { + await checkpointStore.Upsert(NotStarted(category), cancellationToken); + } + } + + var results = new List(categories.Count); + + foreach (var category in categories) + { + results.Add(await RunCategoryAsync(category, cancellationToken)); + } + + return results; + } + + /// + /// Copies one category, carrying on from its saved cursor, and returns the checkpoint it ended on. A category + /// already finished or Failed is returned untouched without reading the source. + /// + /// The stored checkpoint, whose state says how it ended and whose LastError says why it stopped. + /// Another writer saved this category's checkpoint, which means a second instance is copying into the same database. The state is left as that writer set it. + /// The host is shutting down. The last committed batch saved its own counts, so the stored checkpoint is already correct and a restart carries on from it. + /// Saving the checkpoint the category stopped or settled on failed, such as a halt the store could not write. The stored row is the last one saved, so it still reads as copying from its last committed batch. + public async Task RunCategoryAsync(MigrationCategory category, CancellationToken cancellationToken = default) + { + var checkpoint = await checkpointStore.Read(category.Id, cancellationToken) ?? NotStarted(category); + + if (checkpoint.State.IsFinished() || checkpoint.State.IsFailed()) + { + return checkpoint; + } + + if (category.MustFollow is { } mustFollowId) + { + var predecessor = await checkpointStore.Read(mustFollowId, cancellationToken); + if (predecessor?.State.IsFinished() != true) + { + var predecessorState = predecessor?.State.ToString() ?? "not started"; + var blocked = checkpoint with + { + State = MigrationCategoryState.Blocked, + LastError = $"Blocked: {category.Id} must follow {mustFollowId}, which is {predecessorState}" + }; + + logger.LogWarning("Category {CategoryId} did not run: it must follow {PredecessorId}, which is {PredecessorState}", + category.Id, mustFollowId, predecessorState); + + return await checkpointStore.Upsert(blocked, cancellationToken); + } + } + + // A LastError on a row still copying is cleared here, because this start is trying it again. + if (checkpoint.State is MigrationCategoryState.NotStarted or MigrationCategoryState.Blocked || checkpoint.LastError is not null) + { + checkpoint = await checkpointStore.Upsert(checkpoint with + { + State = MigrationCategoryState.InProgress, + StartedAt = checkpoint.StartedAt ?? timeProvider.GetUtcNow().UtcDateTime, + LastProgressAt = timeProvider.GetUtcNow().UtcDateTime, + SettledAt = null, + LastError = null + }, cancellationToken); + } + + var isFirstBatch = true; + // Per run, not the persisted totals: the skips an earlier run made stay on the row, so counting them + // again would stop a run resumed from its cursor on its first batch. + var skippedThisRun = 0L; + var processedThisRun = 0L; + var rowsReadThisRun = 0L; + // Saved after the try rather than inside it, so a failed save is not caught below as the category's own error. + MigrationCheckpoint? halt = null; + + try + { + var batchSize = await target.BatchSizeFor(category, cancellationToken); + + await foreach (var batch in source.Read(category, checkpoint.Cursor, batchSize, cancellationToken).WithCancellation(cancellationToken)) + { + if (!isFirstBatch && category.Kind == MigrationCategoryKind.Optional) + { + await Pause(options.ThrottlePause, cancellationToken); + } + isFirstBatch = false; + rowsReadThisRun += batch.Rows.Count; + + var batchToWrite = batch; + // Kept out of checkpoint until the write commits: the catch below saves checkpoint, and a + // restart re-reads the uncommitted batch and would count these skips twice. + var bodySkips = 0; + if (category.CarriesBodies) + { + var (withBodies, failed) = await FetchBodiesWithRetry(category, batch, cancellationToken); + batchToWrite = withBodies; + bodySkips = failed.Count; + + foreach (var (id, lastAttemptError) in failed) + { + logger.LogWarning(lastAttemptError, "Skipped {SourceId} in category {CategoryId}: body unreadable after {MaxAttempts} attempts", id, category.Id, MaxBodyReadAttempts); + } + } + + // The target adds its own outcome inside the transaction that writes the rows, + // so nothing provisional is ever stored. + var checkpointToExtend = checkpoint with + { + Cursor = batch.Cursor, + LastProgressAt = timeProvider.GetUtcNow().UtcDateTime, + SkippedCount = checkpoint.SkippedCount + bodySkips, + SkipReasons = MigrationCheckpoint.AddSkipReasons(checkpoint.SkipReasons, bodySkips == 0 ? null : new Dictionary { [MigrationSkipReason.BodyUnreadable] = bodySkips }) + }; + + var result = await target.Write(category, batchToWrite, checkpointToExtend, cancellationToken); + + // The target has committed this row, so every halt below settles from it. A halt settling from the + // older version would be refused by the store as a conflict and lose its reason. + checkpoint = result.Saved; + + // The result states the batch's outcome twice, as its own counts and as deltas on the checkpoint + // it committed. The halt threshold reads the first and status and verify read the second. + var committed = (result.Saved.CopiedCount - checkpointToExtend.CopiedCount, result.Saved.SkippedCount - checkpointToExtend.SkippedCount, result.Saved.AlreadyPresentCount - checkpointToExtend.AlreadyPresentCount); + if (committed != (result.Copied, result.Skipped, result.AlreadyPresent)) + { + var mismatch = $"The target reported copying {result.Copied}, skipping {result.Skipped} and finding {result.AlreadyPresent} already present in category {category.Id}, but the checkpoint it committed moved by {committed}."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, mismatch); + halt = checkpoint with { State = MigrationCategoryState.Halted, LastError = mismatch }; + break; + } + + foreach (var id in result.SkippedIds) + { + logger.LogWarning("Skipped {SourceId} in category {CategoryId}", id, category.Id); + } + + processedThisRun += bodySkips + result.Copied + result.Skipped + result.AlreadyPresent; + skippedThisRun += bodySkips + FaultSkips(result.SkipReasons); + + if (HaltThreshold.Exceeded(skippedThisRun, processedThisRun, options.HaltThresholdPercent, options.HaltThresholdMinimum)) + { + var reason = $"Halted: {skippedThisRun} of {processedThisRun} rows skipped in this run exceeds the configured threshold of {options.HaltThresholdPercent}% and {options.HaltThresholdMinimum} rows."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, reason); + halt = checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }; + break; + } + } + } + // A shutdown is not a halt: the last committed batch saved its counts with its own rows, + // so the checkpoint on disk is already correct and resumable. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Another writer holds this row, which no amount of halting resolves. Leave the state alone so their row stands. + catch (MigrationCheckpointConflictException) + { + throw; + } + catch (Exception ex) + { + var position = checkpoint.Cursor is null ? "at the start" : $"at cursor {checkpoint.Cursor}"; + var reason = $"{ex.GetType().Name} {position}: {ex.Message}"; + + // An optional category copies while ServiceControl serves, so an error there waits for the next start rather than for the operator. + if (category.Kind == MigrationCategoryKind.Optional) + { + logger.LogError(ex, "Optional category {CategoryId} stopped at cursor {Cursor} and stays copying until the next start", category.Id, checkpoint.Cursor); + return await checkpointStore.Upsert(checkpoint with { LastError = $"{reason.TrimEnd('.')}. It stays copying and the next start resumes it from the cursor." }, cancellationToken); + } + + logger.LogError(ex, "Category {CategoryId} halted at cursor {Cursor}", category.Id, checkpoint.Cursor); + return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); + } + + if (halt is not null) + { + return await Settle(halt, cancellationToken); + } + + // The one check a category runs on itself, judged over this run because the counts on a resumed row cover earlier runs. + if (rowsReadThisRun != processedThisRun) + { + var reason = $"Halted: {category.Id} read {rowsReadThisRun} rows in this run but the target accounted for {processedThisRun} of them as copied, skipped or already present."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, reason); + return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); + } + + return await Settle(checkpoint with { State = FaultSkips(checkpoint.SkipReasons) > 0 ? MigrationCategoryState.CompleteWithErrors : MigrationCategoryState.Complete }, cancellationToken); + } + + // Harmless skips are rows the product would have removed anyway, so they neither stop a category nor leave it Failed. + static long FaultSkips(IReadOnlyDictionary? skipReasons) => + skipReasons is null ? 0 : skipReasons.Where(reason => !reason.Key.IsBenign()).Sum(reason => reason.Value); + + // Halts log before settling: the store shares the target's database, so a failed save would hide the cause. + Task Settle(MigrationCheckpoint settled, CancellationToken cancellationToken) => + checkpointStore.Upsert(settled with { SettledAt = timeProvider.GetUtcNow().UtcDateTime }, cancellationToken); + + async Task<(MigrationBatch Batch, IReadOnlyList<(string SourceId, Exception LastAttemptError)> Failed)> FetchBodiesWithRetry(MigrationCategory category, MigrationBatch batch, CancellationToken cancellationToken) + { + var survivors = new List(batch.Rows.Count); + var failed = new List<(string SourceId, Exception LastAttemptError)>(); + + foreach (var row in batch.Rows) + { + if (row.Body is not null) + { + survivors.Add(row); + continue; + } + + MigrationBody? body = null; + Exception? lastAttemptError = null; + var succeeded = false; + + for (var attempt = 1; attempt <= MaxBodyReadAttempts && !succeeded; attempt++) + { + try + { + body = await source.ReadBody(category, row.SourceId, cancellationToken); + succeeded = true; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // A shutdown is not a transient body failure. Retrying it to the attempt limit and then + // recording the message as permanently unreadable would lose a row to a restart. + throw; + } + catch (Exception ex) when (!IsDefect(ex)) + { + lastAttemptError = ex; + logger.LogWarning(ex, "Attempt {Attempt} to read the body for {SourceId} failed", attempt, row.SourceId); + if (attempt < MaxBodyReadAttempts) + { + await Pause(options.BodyRetryBackoff, cancellationToken); + } + } + } + + if (succeeded) + { + survivors.Add(row with { Body = body }); + } + else + { + failed.Add((row.SourceId, lastAttemptError!)); + } + } + + return (batch with { Rows = survivors }, failed); + } + + // These fail the same way on every attempt, so retrying would only turn a code defect into skipped messages. + static bool IsDefect(Exception exception) => + exception is NotSupportedException or NotImplementedException or InvalidOperationException or ArgumentException or NullReferenceException or InvalidCastException; + + // Zero means no throttling, and it must return without waiting: against a fake clock + // nobody advances, even a zero-length wait never finishes. + Task Pause(TimeSpan duration, CancellationToken cancellationToken) => + duration <= TimeSpan.Zero ? Task.CompletedTask : Task.Delay(duration, timeProvider, cancellationToken); + + static MigrationCheckpoint NotStarted(MigrationCategory category) => + new(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs new file mode 100644 index 0000000000..8dee699d9f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs @@ -0,0 +1,90 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using ServiceControl.Configuration; + +/// +/// The engine's tuning: the pause between optional batches, when a category halts, and which optional categories +/// to copy and how far back. +/// +/// How long to wait between batches of an optional category, which is what keeps the background copy off the instance's back. Zero waits not at all. +/// The share of skipped rows, as a percentage, that stops a category. +/// The number of skipped rows that has to be passed before the percentage counts, so a handful of bad rows in a small category is not a halt. +/// The optional categories to copy. An optional category outside this list is never copied. +public sealed record MigrationEngineOptions( + TimeSpan ThrottlePause, + int HaltThresholdPercent, + int HaltThresholdMinimum, + IReadOnlyCollection SelectedOptionalCategoryIds) +{ + public static readonly TimeSpan DefaultBodyRetryBackoff = TimeSpan.FromMilliseconds(200); + + /// How long to wait before trying again to read a message body that failed. Not read from settings. + public TimeSpan BodyRetryBackoff { get; init; } = DefaultBodyRetryBackoff; + + /// + /// How far back the event log copy goes, measured against when each event was raised. + /// + public TimeSpan EventLogWindow { get; init; } + + /// + /// How far back the archived and resolved failed messages copy goes, measured against when each message was + /// archived or resolved. + /// + public TimeSpan ArchivedAndResolvedFailedMessagesWindow { get; init; } + + /// + /// Reads the options from the instance's settings. Anything not set takes the default from + /// , except the two windows, which default to the retention periods passed in. + /// An optional category is selected when its window is greater than zero. + /// + /// The settings root the migration keys are read under. + /// The instance's event retention period, used as the event log window when none is set. + /// The instance's error retention period, used as the archived and resolved window when none is set. + /// A window setting is not a time span of zero or more, which the operator has to see before the copy starts. + public static MigrationEngineOptions FromSettings(SettingsRootNamespace settingsRootNamespace, TimeSpan eventRetentionPeriod, TimeSpan errorRetentionPeriod) + { + var throttleMilliseconds = SettingsReader.Read(settingsRootNamespace, MigrationSettings.ThrottlePauseMillisecondsKey, MigrationSettings.DefaultThrottlePauseMilliseconds); + var haltPercent = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdPercentKey, MigrationSettings.DefaultHaltThresholdPercent); + var haltMinimum = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdMinimumKey, MigrationSettings.DefaultHaltThresholdMinimum); + var eventLogWindow = ReadWindow(settingsRootNamespace, MigrationSettings.EventLogWindowKey, eventRetentionPeriod); + var archivedAndResolvedWindow = ReadWindow(settingsRootNamespace, MigrationSettings.ArchivedAndResolvedFailedMessagesWindowKey, errorRetentionPeriod); + + var selectedIds = new List(); + + if (eventLogWindow > TimeSpan.Zero) + { + selectedIds.Add(MigrationCategoryIds.EventLog); + } + + if (archivedAndResolvedWindow > TimeSpan.Zero) + { + selectedIds.Add(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + } + + return new MigrationEngineOptions(TimeSpan.FromMilliseconds(throttleMilliseconds), haltPercent, haltMinimum, selectedIds) + { + EventLogWindow = eventLogWindow, + ArchivedAndResolvedFailedMessagesWindow = archivedAndResolvedWindow + }; + } + + static TimeSpan ReadWindow(SettingsRootNamespace settingsRootNamespace, string key, TimeSpan defaultWindow) + { + var value = SettingsReader.Read(settingsRootNamespace, key); + + if (value is null) + { + return defaultWindow; + } + + if (!TimeSpan.TryParse(value, out var window) || window < TimeSpan.Zero) + { + throw new InvalidOperationException( + $"{key} is '{value}', which is not a time span of zero or more. Set it like a retention period, such as 7.00:00:00 for seven days, or to 0 to leave that category behind."); + } + + return window; + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs new file mode 100644 index 0000000000..6102b22058 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// +/// The migration's setting names, relative to the instance's settings root, and their defaults. +/// explains the tuning ones. +/// +public static class MigrationSettings +{ + public const string ThrottlePauseMillisecondsKey = "Migration/ThrottlePauseMilliseconds"; + public const string HaltThresholdPercentKey = "Migration/HaltThresholdPercent"; + public const string HaltThresholdMinimumKey = "Migration/HaltThresholdMinimum"; + /// + /// How far back the event log copy goes, as a time span. Unset means the event retention period, and zero + /// turns the category off. + /// + public const string EventLogWindowKey = "Migration/EventLogWindow"; + /// + /// How far back the archived and resolved failed messages copy goes, as a time span. Unset means the error + /// retention period, and zero turns the category off. + /// + public const string ArchivedAndResolvedFailedMessagesWindowKey = "Migration/ArchivedAndResolvedFailedMessagesWindow"; + /// + /// Turns the migration on: the required copy runs before the host opens. + /// + public const string EnabledKey = "Migration/Enabled"; + + public const int DefaultThrottlePauseMilliseconds = 100; + public const int DefaultHaltThresholdPercent = 5; + public const int DefaultHaltThresholdMinimum = 100; + public const bool DefaultEnabled = false; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs new file mode 100644 index 0000000000..11d03ffa52 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// +/// Why one row was not copied. Every skipped row carries one, so the verify command can account for the +/// difference between the two databases. says which of +/// these are no loss. +/// +public enum MigrationSkipReason +{ + /// The message body could not be read from the source, after the engine had tried more than once. + BodyUnreadable, + + /// The row is older than the retention period, so the instance would have deleted it soon anyway. + PastRetention, + + /// The source row has no value for something the target column requires. + RequiredValueMissing, + + /// + /// The group comment is null or blank, and the product keeps no row for a blank comment. + /// + BlankGroupComment, + + /// + /// A reason a newer build wrote and this one cannot name, so an older host still starts instead of throwing. + /// Nothing in this build writes it. It counts as a fault, because harmless would let this build settle a + /// category Done over rows the newer build lost; a false Failed from it is cleared by one retry. + /// + Unknown +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs new file mode 100644 index 0000000000..3d5692a56d --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs @@ -0,0 +1,12 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; + +/// What the source report and dry run print about the source: its version and a list of facts. +public sealed record MigrationSourceDescription(string Version, IReadOnlyList Facts); + +/// One line the source report prints, such as which server or database the source read, with the setting to change if it is wrong. +public sealed record MigrationSourceFact(string Label, string Value, string? SettingKey = null); + +/// A count the source report prints, such as the rows in one collection of one database, so an operator sees how much there is to copy. +public sealed record MigrationSourceInventoryEntry(string Scope, string Name, long Count); diff --git a/src/ServiceControl.Persistence/PersistenceSettings.cs b/src/ServiceControl.Persistence/PersistenceSettings.cs index 08a73eb1c3..4d87e98f76 100644 --- a/src/ServiceControl.Persistence/PersistenceSettings.cs +++ b/src/ServiceControl.Persistence/PersistenceSettings.cs @@ -8,25 +8,44 @@ namespace ServiceControl.Persistence /// public abstract class PersistenceSettings { + /// + /// Whether this host starts the database and nothing else, so an operator can repair or inspect it. + /// Only the RavenDB persister supports it. There the embedded server and RavenDB Studio start as + /// usual, and the host registers nothing that ingests or serves data. On any other persister the + /// host refuses to start in maintenance mode. + /// public bool MaintenanceMode { get; set; } - //HINT: This needs to be here so that ServerControl instance can add an instance specific metadata to tweak the DatabasePath value + + /// + /// The directory that holds the database files, or null when this host holds none. The RavenDB + /// persister fills it in from its DbPath setting, and its disk space checks measure the drive that + /// this path names. The SQL persisters leave it null, because their files live on the database server. + /// public string? DatabasePath { get; set; } /// - /// Whether this host owns the background deletion of data past its retention period. Only one - /// host in a deployment should, so error ingestion only hosts turn it off. + /// Whether this host deletes data that is past its retention period. Only one host in a deployment + /// must do this, so a host that only ingests errors turns it off. The SQL persisters start a + /// background sweeper when it is true. RavenDB expires documents on its own and does not read this. /// public bool RunRetentionSweep { get; set; } = true; + /// + /// Whether message bodies are indexed for search. The RavenDB persister decides this as it ingests a + /// message, so a change only reaches messages ingested after it. The SQL persisters always index the + /// body, so the value does not change what they do. + /// public bool EnableFullTextSearchOnBodies { get; set; } = true; /// - /// Wall-clock limit for the message view queries, see . + /// The wall clock limit for a message query, see . It is also how long + /// this instance waits for a remote instance to answer. /// public TimeSpan QueryTimeout { get; set; } = QueryTimeLimit.Default; /// - /// The setting is read from, as named in the timeout error. + /// The name of the setting that comes from. The timeout error names it, so + /// an operator can see which setting to change. /// public const string QueryTimeoutSettingName = "ServiceControl/" + QueryTimeLimit.SettingName; } diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index efdf66b06f..f0b60cc998 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -75,6 +75,11 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi optionalArgs.Add("--RunInMemory=true"); } + if (databaseConfiguration.DisableAutoIndexCreation) + { + optionalArgs.Add("--Indexing.DisableQueryOptimizerGeneratedIndexes=true"); + } + var serverOptions = new ServerOptions { CommandLineArgs = @@ -109,6 +114,10 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi return embeddedDatabase; } + // Start only queues the server up, so connecting by the configured URL instead reaches whatever already holds the port. + public async Task WaitUntilReady(CancellationToken cancellationToken = default) => + (await EmbeddedServer.Instance.GetServerUriAsync(cancellationToken)).ToString(); + void Start(ServerOptions serverOptions) { this.serverOptions = serverOptions; diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs b/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs index 6aec695755..6184a39353 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabaseConfiguration.cs @@ -9,5 +9,8 @@ public class EmbeddedDatabaseConfiguration(string serverUrl, string dbName, stri public string LogsMode { get; } = logsMode; public bool RunInMemory { get; set; } + + /// Makes a dynamic query fail instead of building an auto-index for it. For a server started only to read, where creating an index would be a write. + public bool DisableAutoIndexCreation { get; set; } } } diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index abd4c98313..8a83f80ff0 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -63,6 +63,7 @@ "VirtualDirectory": "", "HeartbeatGracePeriod": "00:00:40", "TransportType": "ServiceControl.Transports.Learning.LearningTransportCustomization, ServiceControl.Transports.Learning", + "MigrationEnabled": false, "ErrorLogQueue": "error.log", "ErrorQueue": "error", "ForwardErrorMessages": false, diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt new file mode 100644 index 0000000000..1e54a214bc --- /dev/null +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt @@ -0,0 +1,17 @@ +Required 1: KnownEndpoints +Required 2: EndpointSettings, after KnownEndpoints +Required 3: MessageRedirects +Required 4: Subscriptions +Required 5: NotificationSettings +Required 6: TrialEndDate +Required 7: RetryOperations +Required 8: LicensingEndpoints +Required 9: LicensingThroughput, after LicensingEndpoints +Required 10: LicensingReportMasks +Required 11: LicensedEndpointDetails +Required 12: UnresolvedAndRetryIssuedFailedMessages, with bodies +Required 14: CustomChecks +Required 15: FailedErrorImports, with bodies +Required 16: GroupComments, after UnresolvedAndRetryIssuedFailedMessages +Optional 1: EventLog +Optional 2: ArchivedAndResolvedFailedMessages, with bodies \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt new file mode 100644 index 0000000000..d0b5240bad --- /dev/null +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt @@ -0,0 +1,4 @@ +Configured: EventLog +Runs as: +Optional 1: EventLog +Never copied: ArchivedAndResolvedFailedMessages \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs new file mode 100644 index 0000000000..d3c8851ed0 --- /dev/null +++ b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.UnitTests.Hosting; + +using NUnit.Framework; +using Particular.ServiceControl.Hosting; +using ServiceControl.Hosting.Commands; + +[TestFixture] +class MigrationSourceReportArgumentTests +{ + [Test] + public void The_flag_selects_the_report_command() => + Assert.That(new HostArguments(["--migration-source-report"]).Command, Is.EqualTo(typeof(MigrationSourceReportCommand))); + + [Test] + public void No_flag_still_selects_the_run_command() => + Assert.That(new HostArguments([]).Command, Is.EqualTo(typeof(RunCommand))); + + [Test] + public void The_setup_flag_is_undisturbed() => + Assert.That(new HostArguments(["--setup"]).Command, Is.EqualTo(typeof(SetupCommand))); +} diff --git a/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs b/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs new file mode 100644 index 0000000000..7a71eb1c2d --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/CheckpointMigrationStateTests.cs @@ -0,0 +1,111 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class CheckpointMigrationStateTests +{ + [Test] + public async Task An_instance_that_has_never_migrated_has_no_store_and_nothing_incomplete() + { + var state = new CheckpointMigrationState(); + + await state.Seed(["EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.False); + } + + [Test] + public async Task Selected_categories_that_all_finished_leave_the_guard_off() + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint("KnownEndpoints", MigrationCategoryState.Complete)); + await store.Upsert(Checkpoint("EndpointSettings", MigrationCategoryState.Complete)); + + var state = new CheckpointMigrationState(store); + await state.Seed(["KnownEndpoints", "EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.False); + } + + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.Blocked)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task One_selected_category_short_of_finished_holds_the_guard_on(MigrationCategoryState unfinished) + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint("KnownEndpoints", MigrationCategoryState.Complete)); + await store.Upsert(Checkpoint("EndpointSettings", unfinished)); + + var state = new CheckpointMigrationState(store); + await state.Seed(["KnownEndpoints", "EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.True); + } + + [Test] + public async Task A_selected_category_with_no_row_at_all_has_not_started() + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(Checkpoint("KnownEndpoints", MigrationCategoryState.Complete)); + + var state = new CheckpointMigrationState(store); + await state.Seed(["KnownEndpoints", "EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.True); + } + + [Test] + public async Task An_unselected_category_that_never_ran_does_not_hold_the_guard_on_forever() + { + var store = new InMemoryMigrationCheckpointStore(); + await store.Upsert(new MigrationCheckpoint("EndpointSettings", MigrationCategoryState.Complete, null, 3, 0, 3, null, null, null, null, null)); + await store.Upsert(new MigrationCheckpoint("EventLog", MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null)); + + var state = new CheckpointMigrationState(store); + await state.Seed(["EndpointSettings"], CancellationToken.None); + + Assert.That(state.AnyCategoryIncomplete, Is.False); + } + + [Test] + public void Exactly_complete_and_abandoned_are_finished() => + Assert.That( + Enum.GetValues().Where(state => state.IsFinished()), + Is.EquivalentTo(new[] { MigrationCategoryState.Complete, MigrationCategoryState.Abandoned })); + + [Test] + public void Exactly_halted_and_complete_with_errors_are_failed() => + Assert.That( + Enum.GetValues().Where(state => state.IsFailed()), + Is.EquivalentTo(new[] { MigrationCategoryState.Halted, MigrationCategoryState.CompleteWithErrors })); + + [Test] + public void Exactly_the_two_harmless_reasons_are_harmless_and_every_other_skip_reason_is_a_fault() => + Assert.That( + Enum.GetValues().Where(reason => reason.IsBenign()), + Is.EquivalentTo(new[] { MigrationSkipReason.PastRetention, MigrationSkipReason.BlankGroupComment }), + "a harmless reason is exempt from the halt threshold and the settle rule, so any number of rows lost to one settles the category Done"); + + [Test] + public void Unknown_stays_the_last_skip_reason() => + Assert.That(Enum.GetValues().Last(), Is.EqualTo(MigrationSkipReason.Unknown)); + + [Test] + public void Exactly_the_reasons_no_retry_can_fix_are_permanent() => + Assert.That( + Enum.GetValues().Where(reason => reason.IsPermanent()), + Is.EquivalentTo(new[] { MigrationSkipReason.RequiredValueMissing })); + + static MigrationCheckpoint Checkpoint(string categoryId, MigrationCategoryState state) => + new(categoryId, state, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs b/src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs new file mode 100644 index 0000000000..e381874eac --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/ClosedWindowProgressTests.cs @@ -0,0 +1,232 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +// No acceptance test can reach the watchdog: the host copies on the real clock and nobody waits half an hour. +[TestFixture] +class ClosedWindowProgressTests +{ + static readonly TimeSpan PollInterval = MigrationStartup.ClosedWindowProgress.PollInterval; + static readonly TimeSpan StallLimit = MigrationStartup.ClosedWindowProgress.StallLimit; + + [Test] + public async Task A_category_that_commits_nothing_for_the_stall_limit_is_stopped() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "a category that committed nothing for longer than the limit was never stopped"); + await progress.DisposeAsync(); + } + + // A call that ignores its token never returns, so the row stays in progress and every later poll sees the same stall. + [Test] + public async Task A_category_already_stopped_is_reported_once_however_long_it_stays_stuck() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + var logger = new CapturingLogger(); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop, logger: logger); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(logger.Entries.Count(entry => entry.Level == LogLevel.Error), Is.EqualTo(1), "the same stall was reported again on the next poll"); + } + + [Test] + public async Task A_resumed_row_carrying_the_previous_runs_stamp_is_not_a_stall() + { + // The row a killed run left behind keeps its last stamp, so an operator restarting hours later + // would otherwise have a healthy copy cancelled on the first tick, every time. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime - TimeSpan.FromHours(2))); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "the window runs from when this category's run began, not from a stamp the previous run left"); + } + + [Test] + public async Task A_row_other_than_the_running_category_is_not_judged() + { + // Both were attempted, but only KnownEndpoints is running, and it has no row yet. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.EndpointSettings, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop, [MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings]); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "the running category was stopped over a row nobody is copying"); + } + + [Test] + public async Task Committing_nothing_for_exactly_the_stall_limit_is_not_a_stall() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "the limit is the point at which a copy has not yet stalled"); + } + + [Test] + public async Task A_category_that_has_committed_nothing_since_its_run_began_is_stopped() + { + // The engine stamps a category when it marks it running, so the window covers the first read, which is + // where a copy that never gets going actually hangs. When the category first started does not matter. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime, startedAt: clock.GetUtcNow().UtcDateTime - TimeSpan.FromDays(3))); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "a category that has committed nothing since its run began was never stopped"); + await progress.DisposeAsync(); + } + + [Test] + public async Task A_row_an_older_build_left_without_a_stamp_is_still_watched() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: null, startedAt: clock.GetUtcNow().UtcDateTime - TimeSpan.FromDays(3))); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "an unstamped row was left unwatched for ever"); + await progress.DisposeAsync(); + } + + [Test] + public async Task A_category_this_run_finished_is_not_stopped_for_having_gone_quiet() + { + // The watch moves to the next category only when that one starts, so a settled row can still be the one watched. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(Settled(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "a category that finished was stopped"); + } + + [Test] + public async Task A_halted_category_this_run_attempted_is_not_stopped_for_having_gone_quiet() + { + // A halt settles the row and leaves its stamp behind. The gate is what refuses the host over it, not the watchdog. + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + await store.Upsert(Settled(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Halted, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + clock.Advance(StallLimit + PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "a halted category was stopped"); + } + + // The watch has no total limit, so a copy that keeps committing outlives any length of run. + [Test] + public async Task A_category_still_committing_batches_is_never_stopped_however_long_it_takes() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + var committed = await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop); + + // Four times the limit, committing a batch every poll, which a total timeout would have killed long ago. + for (var elapsed = TimeSpan.Zero; elapsed < StallLimit * 4; elapsed += PollInterval) + { + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + committed = await store.Upsert(committed with { LastProgressAt = clock.GetUtcNow().UtcDateTime }); + } + + await progress.DisposeAsync(); + + Assert.That(stop.IsCancellationRequested, Is.False, "a copy committing a batch every poll was stopped, so the watch is a deadline rather than a stall detector"); + } + + [Test] + public async Task A_poll_that_fails_leaves_the_watch_running_and_says_so() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + var storeFailure = new InvalidOperationException("the checkpoint store is unreachable"); + store.FailReadAll(1, storeFailure); + await store.Upsert(InProgress(MigrationCategoryIds.KnownEndpoints, lastProgressAt: clock.GetUtcNow().UtcDateTime)); + var logger = new CapturingLogger(); + using var stop = new CancellationTokenSource(); + var progress = await StartWatching(store, clock, MigrationCategoryIds.KnownEndpoints, stop, logger: logger); + + clock.Advance(PollInterval); + await store.WaitForAPollAtTheCurrentTime(); + clock.Advance(StallLimit + PollInterval); + + Assert.That(stop.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10)), Is.True, "a watch that stopped at the first blip never noticed the stall that followed"); + await progress.DisposeAsync(); + Assert.That(logger.Entries.Where(entry => entry.Level == LogLevel.Error).Select(entry => entry.Exception), Has.Member(storeFailure), "a watch that has gone deaf has to say so"); + } + + static async Task StartWatching(PollObservingCheckpointStore store, TimerRecordingTimeProvider clock, string runningCategoryId, CancellationTokenSource stop, string[]? attemptedCategoryIds = null, ILogger? logger = null) + { + var progress = new MigrationStartup.ClosedWindowProgress(store, clock, logger ?? new CapturingLogger(), attemptedCategoryIds ?? [runningCategoryId]); + progress.Watch(runningCategoryId, clock.GetUtcNow().UtcDateTime, stop); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watch never started its timer, so advancing the clock would tick nothing"); + + return progress; + } + + static MigrationCheckpoint InProgress(string categoryId, DateTime? lastProgressAt, DateTime? startedAt = null) => + new(categoryId, MigrationCategoryState.InProgress, null, 0, 0, null, null, startedAt ?? lastProgressAt, lastProgressAt, null, null); + + static MigrationCheckpoint Settled(string categoryId, MigrationCategoryState state, DateTime lastProgressAt) => + new(categoryId, state, null, 0, 0, null, null, lastProgressAt, lastProgressAt, lastProgressAt, null); +} diff --git a/src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs b/src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs new file mode 100644 index 0000000000..c8774a9e9a --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/CopyableCategoryIdsTests.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.UnitTests.Migration; + +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class CopyableCategoryIdsTests +{ + [Test] + public void A_category_only_one_side_supports_counts_as_not_copyable() + { + var source = new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings }; + var target = new[] { MigrationCategoryIds.KnownEndpoints }; + + Assert.That(MigrationStartup.CopyableCategoryIds(source, target), Is.EquivalentTo(new[] { MigrationCategoryIds.KnownEndpoints }), + "a category with a reader and no writer would otherwise be attempted and fail partway through a customer's copy"); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs b/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs new file mode 100644 index 0000000000..24aee91e82 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs @@ -0,0 +1,19 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.DataMigration; + +public sealed class CapturingLogger : ILogger +{ + public List<(LogLevel Level, string Message, Exception? Exception)> Entries { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + Entries.Add((logLevel, formatter(state, exception), exception)); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs new file mode 100644 index 0000000000..78edd726b6 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs @@ -0,0 +1,33 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationCheckpointStore : IMigrationCheckpointStore +{ + readonly ConcurrentDictionary checkpoints = new(); + + public Task> ReadAll(CancellationToken cancellationToken = default) => + Task.FromResult>(checkpoints.Values.ToArray()); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => + Task.FromResult(checkpoints.TryGetValue(categoryId, out var checkpoint) ? checkpoint : null); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + var storedVersion = checkpoints.TryGetValue(checkpoint.CategoryId, out var stored) ? stored.Version : 0; + if (storedVersion != checkpoint.Version) + { + throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but the stored row is at version {storedVersion}."); + } + + var saved = checkpoint with { Version = checkpoint.Version + 1 }; + checkpoints[checkpoint.CategoryId] = saved; + return Task.FromResult(saved); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs new file mode 100644 index 0000000000..ebc2dd8253 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs @@ -0,0 +1,113 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationSource : IMigrationSource +{ + readonly Dictionary> rowsByCategory = []; + readonly Dictionary bodyFailures = []; + readonly Dictionary bodyFailuresInTurn = []; + readonly Dictionary bodyReadAttempts = []; + readonly Dictionary bodies = []; + + public MigrationSourceDescription Description { get; set; } = new("in-memory", [new MigrationSourceFact("Store", "in memory")]); + + public void Seed(string categoryId, params MigrationRow[] rows) => rowsByCategory[categoryId] = [.. rows]; + + public void SetBody(string sourceId, MigrationBody? body) => bodies[sourceId] = body; + + public void FailBodyReads(string sourceId, int times, Exception failure) => bodyFailures[sourceId] = (times, failure); + + /// + /// Fails the body reads for this row with one exception per attempt, in order, so a test can tell one attempt's error from another's. An attempt past the end of the list reads the body normally. + /// + public void FailBodyReadsInTurn(string sourceId, params Exception[] failuresInAttemptOrder) => bodyFailuresInTurn[sourceId] = failuresInAttemptOrder; + + /// + /// Makes the body read for this row behave like a host shutting down: the token is cancelled and the read throws. + /// + public (string SourceId, CancellationTokenSource Source)? StopOnBodyRead { get; set; } + + public int BodyReadAttempts(string sourceId) => bodyReadAttempts.GetValueOrDefault(sourceId); + + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public IReadOnlyList ContributedChecks() => []; + + public Task Describe(CancellationToken cancellationToken = default) => Task.FromResult(Description); + + public Task> Inventory(CancellationToken cancellationToken = default) => + Task.FromResult>( + [.. rowsByCategory.Select(pair => new MigrationSourceInventoryEntry("memory", pair.Key, pair.Value.Count))]); + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + Task.FromResult(rowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0); + + public async IAsyncEnumerable Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var rows = rowsByCategory.GetValueOrDefault(category.Id) ?? []; + var startIndex = 0; + + if (resumeAfter is not null) + { + var cursorIndex = rows.FindIndex(r => r.SourceId == resumeAfter); + if (cursorIndex < 0) + { + throw new InvalidOperationException($"Category {category.Id} was asked to resume after {resumeAfter}, a cursor this source never issued"); + } + startIndex = cursorIndex + 1; + } + + for (var i = startIndex; i < rows.Count; i += batchSize) + { + cancellationToken.ThrowIfCancellationRequested(); + var slice = rows.Skip(i).Take(batchSize).ToArray(); + yield return new MigrationBatch(slice, slice[^1].SourceId); + await Task.Yield(); + } + } + + public async Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) + { + await Task.Yield(); + + var attempt = bodyReadAttempts[sourceId] = BodyReadAttempts(sourceId) + 1; + + if (StopOnBodyRead is { } stop && stop.SourceId == sourceId) + { + await stop.Source.CancelAsync(); + throw new OperationCanceledException(stop.Source.Token); + } + + if (bodyFailures.TryGetValue(sourceId, out var failures) && attempt <= failures.Times) + { + throw failures.Failure; + } + + if (bodyFailuresInTurn.TryGetValue(sourceId, out var failuresInTurn) && attempt <= failuresInTurn.Length) + { + throw failuresInTurn[attempt - 1]; + } + + return bodies.GetValueOrDefault(sourceId); + } + + // Every registered category, not just the seeded ones: Seed writes rowsByCategory after construction, + // and the interface promises an answer that does not change across Open. + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs new file mode 100644 index 0000000000..23492909d4 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs @@ -0,0 +1,82 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class InMemoryMigrationSourceTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Streams_seeded_rows_in_batches_of_the_requested_size() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b"), Row("c"), Row("d"), Row("e")); + + var batches = new List(); + await foreach (var batch in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: null, batchSize: 2)) + { + batches.Add(batch); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches, Has.Count.EqualTo(3)); + Assert.That(batches[0].Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "a", "b" })); + Assert.That(batches[2].Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "e" })); + } + } + + [Test] + public async Task ResumeAfter_skips_everything_up_to_and_including_that_id() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b"), Row("c")); + + var batches = new List(); + await foreach (var batch in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: "a", batchSize: 10)) + { + batches.Add(batch); + } + + Assert.That(batches.Single().Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "b", "c" })); + } + + [Test] + public void A_cursor_the_source_never_issued_throws_rather_than_starting_again() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b")); + + Assert.ThrowsAsync(async () => + { + await foreach (var _ in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: "no-such-row", batchSize: 10)) + { + } + }); + } + + [Test] + public async Task ReadBody_fails_the_configured_number_of_times_then_returns_the_seeded_body() + { + var source = new InMemoryMigrationSource(); + var finalBody = new MigrationBody(new byte[] { 9 }, "text/plain"); + source.SetBody("msg-1", finalBody); + source.FailBodyReads("msg-1", times: 1, new TimeoutException("transient")); + + Assert.ThrowsAsync(() => source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1")); + var secondAttempt = await source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(secondAttempt, Is.EqualTo(finalBody)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(2)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs new file mode 100644 index 0000000000..84c171e912 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -0,0 +1,148 @@ +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget +{ + readonly Dictionary> writtenKeysByCategory = []; + readonly Dictionary> writtenRowsByCategory = []; + readonly Dictionary> rowsHandedToWriteByCategory = []; + readonly HashSet preExistingKeys = []; + readonly Dictionary rejectedKeys = []; + + public int DefaultBatchSize { get; set; } = 3; + public string NoBatchSizeFor { get; set; } + public int? FailOnCallNumber { get; set; } + + /// + /// What FailOnCallNumber throws, when the default simulated failure is the wrong shape for the test. + /// + public Exception FailWith { get; set; } + + /// + /// How many rows each write leaves out of the outcome it reports and commits, without failing, so the two still agree with each other. + /// + public int UnderReportBy { get; set; } + + /// + /// Runs at the start of every write, before any simulated failure, with the checkpoint the engine is extending, so a test can watch a stamp move between batches. + /// + public Action BeforeWrite { get; set; } + + /// + /// Cancels the token on this call and then writes normally, so the stop surfaces from the source's next batch. + /// + public (int CallNumber, CancellationTokenSource Source)? CancelOnCall { get; set; } + /// + /// Cancels the token on this call and throws instead of writing, so the stop surfaces from the write itself. + /// + public (int CallNumber, CancellationTokenSource Source)? StopOnCall { get; set; } + /// + /// Waits on this call until the token is cancelled and then throws, as a write stuck on a database that stopped answering does. + /// + public int? HangOnCall { get; set; } + int callCount; + + public void SeedExistingKey(string sourceId) => preExistingKeys.Add(sourceId); + + public void RejectKey(string sourceId, MigrationSkipReason reason) => rejectedKeys[sourceId] = reason; + + public IReadOnlyList WrittenRows(string categoryId) => + writtenRowsByCategory.TryGetValue(categoryId, out var rows) ? rows : []; + + /// + /// Every row the engine sent to a write that ran, in the order it sent them, including the rows this fake then skipped or found already present. It is the only way to see the same row sent twice, because WrittenRows de-duplicates as the real targets do. Empty for a category never written to. + /// + public IReadOnlyList RowsHandedToWrite(string categoryId) => + rowsHandedToWriteByCategory.TryGetValue(categoryId, out var rows) ? rows : []; + + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => + category.Id == NoBatchSizeFor ? throw new InvalidOperationException($"No batch size is mapped for category {category.Id}") : Task.FromResult(DefaultBatchSize); + + public async Task Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + CancellationToken cancellationToken = default) + { + callCount++; + BeforeWrite?.Invoke(checkpointToExtend); + + if (HangOnCall == callCount) + { + await Task.Delay(Timeout.Infinite, cancellationToken); + } + + if (StopOnCall is { } stop && stop.CallNumber == callCount) + { + await stop.Source.CancelAsync(); + throw new OperationCanceledException(stop.Source.Token); + } + + if (FailOnCallNumber == callCount) + { + throw FailWith ?? new InvalidOperationException($"Simulated failure on write {callCount}"); + } + + if (CancelOnCall is { } cancel && cancel.CallNumber == callCount) + { + await cancel.Source.CancelAsync(); + } + + var keys = writtenKeysByCategory.TryGetValue(category.Id, out var existingKeys) ? existingKeys : writtenKeysByCategory[category.Id] = []; + var rows = writtenRowsByCategory.TryGetValue(category.Id, out var existingRows) ? existingRows : writtenRowsByCategory[category.Id] = []; + + // Recorded here rather than at the top of the method: a write that threw above never committed, so + // the restart that sends its batch again is doing the right thing. + var handedOver = rowsHandedToWriteByCategory.TryGetValue(category.Id, out var existingHandedOver) ? existingHandedOver : rowsHandedToWriteByCategory[category.Id] = []; + handedOver.AddRange(batch.Rows); + + var copied = 0; + var alreadyPresent = 0; + var skippedIds = new List(); + var skipReasons = new Dictionary(); + + foreach (var row in batch.Rows) + { + if (rejectedKeys.TryGetValue(row.SourceId, out var reason)) + { + skippedIds.Add(row.SourceId); + skipReasons[reason] = skipReasons.GetValueOrDefault(reason) + 1; + continue; + } + + if (preExistingKeys.Contains(row.SourceId) || !keys.Add(row.SourceId)) + { + alreadyPresent++; + continue; + } + + rows.Add(row); + copied++; + } + + copied -= UnderReportBy; + + // The real targets extend and save the checkpoint in the transaction that writes the rows, + // so this fake saves it here too. + var saved = await checkpointStore.Upsert( + checkpointToExtend.Extend(copied, skippedIds.Count, alreadyPresent, skipReasons), + cancellationToken); + + return new MigrationWriteResult(saved, copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + Task.FromResult((long)(writtenRowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); + + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs new file mode 100644 index 0000000000..ae52e815dc --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs @@ -0,0 +1,81 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System.Collections.Generic; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class InMemoryMigrationTargetTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationCheckpoint EmptyCheckpoint(string categoryId) => + new(categoryId, MigrationCategoryState.InProgress, Cursor: null, 0, 0, null, null, null, null, null, null); + + [Test] + public async Task Writes_new_rows_and_commits_the_extended_checkpoint_with_them() + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var category = MigrationCategoryRegistry.Find("EndpointSettings")!; + var batch = new MigrationBatch([Row("a"), Row("b")], Cursor: "b"); + // Prior totals and the new cursor. The target adds this batch's own outcome before it saves. + var checkpointToExtend = EmptyCheckpoint(category.Id) with { Cursor = "b" }; + + var result = await target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.Skipped, Is.Zero); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(2)); + Assert.That(await checkpointStore.Read(category.Id), Is.EqualTo(checkpointToExtend with { CopiedCount = 2, Version = 1 })); + Assert.That(result.Saved, Is.EqualTo(checkpointToExtend with { CopiedCount = 2, Version = 1 }), "the result carries the row as stored"); + } + } + + [Test] + public async Task The_committed_checkpoint_carries_the_real_split_rather_than_the_rows_handed_over() + { + // Three rows in, one copied: a checkpoint saying three would survive a crash as three. + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + target.SeedExistingKey("already-present"); + target.RejectKey("rejected", MigrationSkipReason.BodyUnreadable); + var batch = new MigrationBatch([Row("already-present"), Row("rejected"), Row("new-row")], Cursor: "new-row"); + var checkpointToExtend = EmptyCheckpoint(category.Id) with { Cursor = "new-row" }; + + var result = await target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "rejected" })); + Assert.That(result.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(1)); + var stored = (await checkpointStore.Read(category.Id))!; + Assert.That((stored.CopiedCount, stored.SkippedCount, stored.AlreadyPresentCount), Is.EqualTo((1L, 1L, 1L)), "copied, skipped, already present as committed"); + } + } + + [Test] + public async Task Count_answers_for_one_category_rather_than_one_table() + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var required = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var archive = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + await target.Write(required, new MigrationBatch([Row("a"), Row("b")], "b"), EmptyCheckpoint(required.Id)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await target.Count(required), Is.EqualTo(2)); + Assert.That(await target.Count(archive), Is.Zero); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs b/src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs new file mode 100644 index 0000000000..147fac3a8f --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/PollObservingCheckpointStore.cs @@ -0,0 +1,64 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +// Records when each poll read the rows, so a test can wait for a poll that saw the state it is judging. +// Only the stall watchdog calls ReadAll, so every ReadAll is one of its polls. +public sealed class PollObservingCheckpointStore(TimeProvider clock) : IMigrationCheckpointStore +{ + readonly InMemoryMigrationCheckpointStore inner = new(); + readonly SemaphoreSlim polled = new(0); + readonly ConcurrentQueue polledAt = new(); + Exception? readAllFailure; + int readAllFailuresLeft; + + public void FailReadAll(int times, Exception failure) + { + readAllFailuresLeft = times; + readAllFailure = failure; + } + + /// + /// Waits until a poll has read the rows at the clock's current time, and fails the test if none does within ten + /// seconds. Polls at earlier times are passed over. + /// + public async Task WaitForAPollAtTheCurrentTime() + { + var now = clock.GetUtcNow().UtcDateTime; + + while (true) + { + Assert.That(await polled.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, $"no poll read the checkpoints at {now:O}; a watch that stopped polling never reaches one"); + + if (polledAt.TryDequeue(out var at) && at == now) + { + return; + } + } + } + + public Task> ReadAll(CancellationToken cancellationToken = default) + { + polledAt.Enqueue(clock.GetUtcNow().UtcDateTime); + polled.Release(); + + if (readAllFailuresLeft > 0) + { + readAllFailuresLeft--; + return Task.FromException>(readAllFailure!); + } + + return inner.ReadAll(cancellationToken); + } + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => inner.Read(categoryId, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => inner.Upsert(checkpoint, cancellationToken); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs b/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs new file mode 100644 index 0000000000..6178cbde97 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs @@ -0,0 +1,35 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Threading; +using Microsoft.Extensions.Time.Testing; + +// Signals each timer the engine creates, so a test only advances the clock once a pause is waiting on it. +public sealed class TimerRecordingTimeProvider : TimeProvider +{ + readonly FakeTimeProvider clock = new(); + + public SemaphoreSlim TimerCreated { get; } = new(0); + + public List DueTimes { get; } = []; + + public void Advance(TimeSpan delta) => clock.Advance(delta); + + public override DateTimeOffset GetUtcNow() => clock.GetUtcNow(); + + public override long GetTimestamp() => clock.GetTimestamp(); + + public override long TimestampFrequency => clock.TimestampFrequency; + + public override TimeZoneInfo LocalTimeZone => clock.LocalTimeZone; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = clock.CreateTimer(callback, state, dueTime, period); + DueTimes.Add(dueTime); + TimerCreated.Release(); + return timer; + } +} diff --git a/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs new file mode 100644 index 0000000000..ae309d6c8b --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.UnitTests.Migration; + +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class HaltThresholdTests +{ + [Test] + public void Does_not_halt_a_three_row_category_with_one_bad_row() + { + // 1 of 3 is 33%, well past the 5% proportion, but 1 skip never passes the 100-row floor. + var exceeded = HaltThreshold.Exceeded(skippedCount: 1, totalCount: 3, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Halts_a_large_category_with_a_systemic_failure() + { + // 600 of 10,000 is 6%, past both the proportion and the floor. + var exceeded = HaltThreshold.Exceeded(skippedCount: 600, totalCount: 10_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.True); + } + + [Test] + public void Does_not_halt_a_large_category_with_only_a_small_proportion_skipped() + { + // 10,000 of 5,000,000 is 0.2%: past the floor, nowhere near the proportion. + var exceeded = HaltThreshold.Exceeded(skippedCount: 10_000, totalCount: 5_000_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Exactly_at_both_boundaries_does_not_halt_because_both_must_be_exceeded() + { + // Exactly the floor, which settles it before the proportion is ever worked out. "Exceed" means strictly past, not "at". + var exceeded = HaltThreshold.Exceeded(skippedCount: 100, totalCount: 2_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Exactly_on_the_proportion_does_not_halt_once_the_floor_is_behind_it() + { + // 101 of 2,020 is exactly 5% with the floor already passed, so this is the only shape that + // reaches the proportion comparison and depends on it being strictly greater. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 2_020, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void A_hair_past_the_proportion_halts_once_the_floor_is_behind_it() + { + // 101 of 2,000 is 5.05%: the same skip count as above, one row's worth over the line. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 2_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.True); + } + + [Test] + public void A_large_category_losing_a_small_share_does_not_halt_once_past_the_floor() + { + // 101 of 5,000,000 is 0.002%. The floor is already behind it, so only the proportion can stop a halt + // here, and a rule that halted on the floor alone would stop a healthy copy of a huge category. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 5_000_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void A_skip_count_with_nothing_processed_never_halts_and_never_divides_by_zero() + { + // Past the floor with a zero total, which is the only input that reaches the division guard. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 0, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void No_rows_processed_yet_never_halts() + { + Assert.That(HaltThreshold.Exceeded(skippedCount: 0, totalCount: 0, percentThreshold: 5, minimumFloor: 100), Is.False); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs new file mode 100644 index 0000000000..c82e610f09 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs @@ -0,0 +1,110 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationCategoryRegistryTests +{ + [Test] + public void Contains_all_seventeen_categories_with_unique_ids() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(17)); + Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(17)); + } + } + + [Test] + public void Required_categories_are_ordered_exactly_as_the_contract_lists_them() + { + var requiredIds = MigrationCategoryRegistry.All + .Where(c => c.Kind == MigrationCategoryKind.Required) + .OrderBy(c => c.Order) + .Select(c => c.Id) + .ToArray(); + + Assert.That(requiredIds, Is.EqualTo(new[] + { + "KnownEndpoints", "EndpointSettings", "MessageRedirects", "Subscriptions", + "NotificationSettings", "TrialEndDate", "RetryOperations", + "LicensingEndpoints", "LicensingThroughput", "LicensingReportMasks", + "LicensedEndpointDetails", "UnresolvedAndRetryIssuedFailedMessages", + "CustomChecks", "FailedErrorImports", "GroupComments" + })); + } + + [Test] + public void Optional_categories_are_ordered_exactly_as_the_contract_lists_them() + { + var optionalIds = MigrationCategoryRegistry.All + .Where(c => c.Kind == MigrationCategoryKind.Optional) + .OrderBy(c => c.Order) + .Select(c => c.Id) + .ToArray(); + + Assert.That(optionalIds, Is.EqualTo(new[] + { + "EventLog", "ArchivedAndResolvedFailedMessages" + })); + } + + [Test] + public void Three_categories_declare_a_MustFollow() + { + var declared = MigrationCategoryRegistry.All + .Where(c => c.MustFollow is not null) + .ToDictionary(c => c.Id, c => c.MustFollow); + + Assert.That(declared, Is.EqualTo(new Dictionary + { + // A cascading foreign key: throughput rows cannot exist before their endpoint row. + ["LicensingThroughput"] = "LicensingEndpoints", + // Keeps settings back while known endpoints are unfinished. The heartbeat sync is kept off + // the copied settings because both categories finish before the host opens. + ["EndpointSettings"] = "KnownEndpoints", + // A group comment whose failed messages have not arrived reads as an orphan to the + // retention sweeper, which deletes it. + ["GroupComments"] = "UnresolvedAndRetryIssuedFailedMessages" + })); + } + + [Test] + public void The_two_failed_message_categories_and_failed_imports_carry_bodies() + { + var carriesBodies = MigrationCategoryRegistry.All.Where(c => c.CarriesBodies).Select(c => c.Id).ToArray(); + + Assert.That(carriesBodies, Is.EquivalentTo(new[] { "UnresolvedAndRetryIssuedFailedMessages", "ArchivedAndResolvedFailedMessages", "FailedErrorImports" })); + } + + [Test] + public void The_two_failed_message_categories_split_every_status_between_them() + { + var split = MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses.Concat(MigrationCategoryRegistry.ArchivedAndResolvedStatuses).ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(split, Is.Unique); + Assert.That(split, Is.EquivalentTo(Enum.GetValues()), + "a status neither category claims is a failed message no migration copies"); + } + } + + [Test] + public void Find_returns_null_for_an_unknown_id() + { + Assert.That(MigrationCategoryRegistry.Find("NoSuchCategory"), Is.Null); + } + + [Test] + public void Find_returns_the_category_for_a_known_id() + { + Assert.That(MigrationCategoryRegistry.Find("EventLog")?.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs new file mode 100644 index 0000000000..8b981b0665 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs @@ -0,0 +1,83 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationContractShapeTests +{ + [Test] + public void MigrationCategory_carries_no_fact_about_where_a_store_keeps_its_rows() + { + var properties = typeof(MigrationCategory).GetProperties().Select(p => p.Name); + + Assert.That(properties, Is.EquivalentTo(new[] { "Id", "Kind", "CarriesBodies", "Order", "MustFollow" }), + "a source database or target table belongs in that store's own adapter, where a different store pair can map it differently"); + } + + [Test] + public void Category_states_keep_the_integers_stored_checkpoints_already_hold() + { + var stored = System.Enum.GetValues().ToDictionary(state => state.ToString(), state => (int)state); + + Assert.That(stored, Is.EqualTo(new Dictionary + { + ["NotStarted"] = 0, + ["InProgress"] = 1, + ["Complete"] = 2, + ["CompleteWithErrors"] = 3, + ["Halted"] = 4, + ["Abandoned"] = 5, + ["Blocked"] = 6 + }), "checkpoints store State as an integer, so a reordered or inserted member silently changes what every saved checkpoint means"); + } + + [Test] + public void MigrationRow_body_defaults_to_null() + { + var row = new MigrationRow("id-1", new object(), new Dictionary()); + + Assert.That(row.Body, Is.Null); + } + + [Test] + public void MigrationCheckpoint_AlreadyPresentCount_defaults_to_zero() + { + var checkpoint = new MigrationCheckpoint( + CategoryId: "EndpointSettings", + State: MigrationCategoryState.NotStarted, + Cursor: null, + CopiedCount: 0, + SkippedCount: 0, + SourceTotal: null, + SkipReasons: null, + StartedAt: null, + LastProgressAt: null, + SettledAt: null, + LastError: null); + + Assert.That(checkpoint.AlreadyPresentCount, Is.Zero); + } + + [Test] + public void MigrationCheckpoint_StartedWindowSeconds_defaults_to_null() + { + var checkpoint = new MigrationCheckpoint( + CategoryId: "EndpointSettings", + State: MigrationCategoryState.NotStarted, + Cursor: null, + CopiedCount: 0, + SkippedCount: 0, + SourceTotal: null, + SkipReasons: null, + StartedAt: null, + LastProgressAt: null, + SettledAt: null, + LastError: null); + + Assert.That(checkpoint.StartedWindowSeconds, Is.Null); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs new file mode 100644 index 0000000000..4d82cac0f8 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEnabledSettingsTests.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using System.Reflection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationEnabledSettingsTests +{ + static Settings NewSettings() => + new(transportType: "LearningTransport", persisterType: "RavenDB", errorRetentionPeriod: TimeSpan.FromDays(10)); + + [TearDown] + public void ClearEnvironmentVariables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ENABLED", null); + } + + [Test] + public void Migration_defaults_to_off() + { + Assert.That(NewSettings().MigrationEnabled, Is.False); + } + + [Test] + public void MigrationEnabled_is_read_from_the_environment() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ENABLED", "true"); + + Assert.That(NewSettings().MigrationEnabled, Is.True); + } + + [Test] + public void Only_MigrationEngineOptions_parses_the_engine_settings_keys() + { + var parsers = typeof(Settings).Assembly.GetTypes() + .Concat(typeof(MigrationEngineOptions).Assembly.GetTypes()) + .Where(type => type.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static) + .Any(method => method.Name is "FromSettings" or "Read" or "Resolve" + && method.GetParameters() is [{ ParameterType.Name: nameof(SettingsRootNamespace) }, ..])) + .Select(type => type.Name) + .ToArray(); + + Assert.That(parsers, Is.EquivalentTo(new[] { nameof(MigrationEngineOptions) }), + "Migration/ThrottlePauseMilliseconds and its neighbours have exactly one parser. A second one drifts its defaults and its refusal message away from this one, and nothing fails until a customer types a window wrong."); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs new file mode 100644 index 0000000000..dbc94ad95a --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs @@ -0,0 +1,240 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineBodyRetryTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task A_body_that_fails_until_the_last_attempt_then_succeeds_is_written_with_its_body_attached() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var body = new MigrationBody(new byte[] { 1 }, "text/plain"); + source.SetBody("msg-1", body); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts - 1, new TimeoutException("blip")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + // Zeroed because FakeTimeProvider is never advanced here: a 200 ms Task.Delay against a clock + // nobody moves never completes, and the test hangs until the runner kills it. + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(body)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(MigrationEngine.MaxBodyReadAttempts)); + } + } + + // Every type the engine treats as a defect. Each fails the same way on every attempt, so retrying one + // would only turn a code fault into skipped messages. + static readonly Exception[] Defects = + [ + new NotSupportedException("this source cannot read bodies"), + new NotImplementedException("not written yet"), + new InvalidOperationException("the session is closed"), + new ArgumentException("the id is not a document id"), + new NullReferenceException("no attachment"), + new InvalidCastException("not an attachment") + ]; + + [TestCaseSource(nameof(Defects))] + public async Task A_body_read_that_fails_as_a_defect_halts_the_category_on_the_first_attempt_without_skipping_the_message(Exception defect) + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, defect); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain(defect.GetType().Name)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(1)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.SkipReasons, Is.Null); + } + } + + [Test] + public async Task A_shutdown_during_a_body_read_stops_the_run_instead_of_skipping_the_message() + { + // Retrying a shutdown to the attempt limit and then recording the message as permanently + // unreadable is the one path here that silently loses a customer's failed message. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + using var stopping = new CancellationTokenSource(); + source.StopOnBodyRead = ("msg-1", stopping); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category, stopping.Token)); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(1), "a shutdown is not a transient body failure, so it is not retried"); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(persisted.SkippedCount, Is.Zero, "the message is still there to copy on the next run"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task The_configured_backoff_is_waited_between_body_read_attempts() + { + // Without the wait, three attempts against a body store that is briefly down all fail inside a + // millisecond and the message is skipped for an outage it would have survived. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + // Every attempt on msg-1 fails, so the guard against a wait after the final one is the only thing + // keeping the third timer from being created. + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var clock = new TimerRecordingTimeProvider(); + var backoff = TimeSpan.FromMilliseconds(200); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = backoff }; + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var runTask = engine.RunCategoryAsync(category); + + // A wait after the first two failures only. A third would never complete, because the clock is + // not advanced again and the run below would time out waiting for it. + for (var waitNumber = 1; waitNumber <= MigrationEngine.MaxBodyReadAttempts - 1; waitNumber++) + { + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, $"backoff {waitNumber} never started"); + Assert.That(runTask.IsCompleted, Is.False, $"backoff {waitNumber} should still be pending"); + clock.Advance(backoff); + } + + var checkpoint = await runTask.WaitAsync(TimeSpan.FromSeconds(5)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1), "msg-1 was given up on, so all three attempts ran"); + Assert.That(clock.DueTimes, Is.EqualTo(new[] { backoff, backoff }), "no wait after the final attempt, which has nothing left to retry"); + } + } + + [Test] + public async Task The_skip_warning_for_an_unreadable_body_carries_the_last_attempts_exception() + { + // The first failure of a retry run is usually a transient connect error. The last one says what the + // body store was doing when the message was given up on, and it is all the operator gets. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var failures = Enumerable.Range(1, MigrationEngine.MaxBodyReadAttempts) + .Select(attempt => new TimeoutException($"body store unreachable on attempt {attempt}")) + .ToArray(); + source.FailBodyReadsInTurn("msg-1", failures); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + await engine.RunCategoryAsync(category); + + Assert.That(logger.Entries.Single(e => e.Message.StartsWith("Skipped msg-1")).Exception, Is.SameAs(failures[^1])); + } + + [Test] + public async Task A_body_the_source_already_attached_is_written_without_reading_it_again() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + var attached = new MigrationBody(new byte[] { 7 }, "text/plain"); + source.Seed(category.Id, Row("msg-1") with { Body = attached }); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("a body read nobody needed")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(attached)); + } + } + + [Test] + public async Task Exhausted_attempts_skip_the_whole_message_and_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("down")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + // msg-1 is excluded entirely: not written with a missing body, not written at all. + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2" })); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + } + } + + [Test] + public async Task A_body_store_outage_across_many_messages_halts_the_category() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + var ids = Enumerable.Range(1, 1_000).Select(i => $"msg-{i}").ToArray(); + source.Seed(category.Id, [.. ids.Select(Row)]); + // Every body read fails every attempt: a down body store, not a per-message fluke. + foreach (var id in ids) + { + source.FailBodyReads(id, MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + } + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs new file mode 100644 index 0000000000..15cf13aaad --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs @@ -0,0 +1,90 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Particular.Approvals; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCategorySelectionTests +{ + static MigrationEngine BuildEngine(IReadOnlyCollection selectedOptionalIds) + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, selectedOptionalIds); + return new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + } + + static string Describe(MigrationCategory category) => + $"{category.Kind} {category.Order}: {category.Id}" + + (category.CarriesBodies ? ", with bodies" : string.Empty) + + (category.MustFollow is null ? string.Empty : $", after {category.MustFollow}"); + + [Test] + public void Every_category_runs_in_a_fixed_order() + { + var everyOptionalId = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Optional) + .Select(category => category.Id) + .ToArray(); + var engine = BuildEngine(everyOptionalId); + + var runOrder = engine.SelectCategories(MigrationCategoryKind.Required) + .Concat(engine.SelectCategories(MigrationCategoryKind.Optional)) + .Select(Describe); + + Approver.Verify(string.Join(Environment.NewLine, runOrder)); + } + + [Test] + public void Only_configured_optional_categories_are_selected() + { + string[] configured = [MigrationCategoryIds.EventLog]; + var engine = BuildEngine(configured); + + var selected = engine.SelectCategories(MigrationCategoryKind.Optional); + var left = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Optional && !selected.Contains(category)) + .Select(category => category.Id); + + Approver.Verify(string.Join(Environment.NewLine, + [ + $"Configured: {string.Join(", ", configured)}", + "Runs as:", + .. selected.Select(Describe), + $"Never copied: {string.Join(", ", left)}" + ])); + } + + [Test] + public async Task A_category_removed_from_configuration_leaves_its_checkpoint_row_untouched() + { + // Dropping a category from the configuration must not restart, reset or delete what it already + // copied: those rows are in the target, and a row wound back to the start copies every one again. + var stillConfigured = MigrationCategoryIds.EventLog; + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var source = new InMemoryMigrationSource(); + source.Seed(stillConfigured, new MigrationRow("check-1", new object(), new Dictionary())); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, [stillConfigured]); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var previousRun = new MigrationCheckpoint(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryState.CompleteWithErrors, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + await checkpointStore.Upsert(previousRun); + + var results = await engine.RunCategories(engine.SelectCategories(MigrationCategoryKind.Optional)); + + var deselected = await checkpointStore.Read(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(c => c.CategoryId), Is.EqualTo(new[] { stillConfigured }), "the run touched only the configured category"); + Assert.That(deselected, Is.EqualTo(previousRun with { Version = 1 }), "the row is still at the version the seeding save left it"); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs new file mode 100644 index 0000000000..59b53c6a07 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs @@ -0,0 +1,45 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCollisionTests +{ + static MigrationRow Row(string id, string name) => new(id, new { Name = name }, new Dictionary()); + + [Test] + public async Task Rows_already_present_in_the_target_are_counted_as_already_present_across_batches_not_skipped_or_overwritten() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1", "from-source"), Row("msg-2", "from-source"), Row("msg-3", "from-source"), Row("msg-4", "from-source")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + // msg-1 and msg-4 already exist in the target: they failed again after cutover, and real ingestion wrote them. + target.SeedExistingKey("msg-1"); + target.SeedExistingKey("msg-4"); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(2)); + // Asserted through the fake's recorded rows, never by the engine inspecting a document. + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2", "msg-3" })); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs new file mode 100644 index 0000000000..6135b96aa5 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs @@ -0,0 +1,251 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCopyTests +{ + static MigrationRow Row(string id) => new(id, new { Name = id }, new Dictionary()); + + [Test] + public async Task Copies_every_row_and_finishes_Complete_when_nothing_was_skipped() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.Cursor, Is.EqualTo("c")); + Assert.That(checkpoint.StartedAt, Is.Not.Null); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(3)); + } + } + + [Test] + public async Task A_first_run_leaves_the_source_total_null_and_every_batch_moves_LastProgressAt() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var startedAt = new DateTimeOffset(2026, 3, 4, 5, 6, 7, TimeSpan.Zero); + var betweenBatches = TimeSpan.FromMinutes(1); + var clock = new FakeTimeProvider(startedAt); + var stamps = new List(); + // The engine stamps the checkpoint before the target sees it, so moving the clock in here is what + // makes a stamp that never moves show up as two identical entries. + target.BeforeWrite = extending => + { + stamps.Add(extending.LastProgressAt); + clock.Advance(betweenBatches); + }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.SourceTotal, Is.Null, "counting the source is a full pass over it, which on a big category could trip the stall watchdog before a row is copied"); + Assert.That(stamps, Is.EqualTo(new DateTime?[] { startedAt.UtcDateTime, (startedAt + betweenBatches).UtcDateTime }), "the stall watchdog stops a copy whose LastProgressAt stops moving, so every batch has to move it"); + Assert.That(checkpoint.LastProgressAt, Is.EqualTo((startedAt + betweenBatches).UtcDateTime), "the saved row carries the last batch's stamp"); + } + } + + [Test] + public async Task A_category_that_read_fewer_rows_than_the_source_holds_still_settles_complete() + { + // The row says the source held 10 and the source yields 2. The engine judges only its own accounting, + // so a read that ends early is not something it can see, and only --migration-verify shows it. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("d"), Row("e")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, null, 0, 0, 10, null, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + } + } + + [Test] + public async Task An_optional_category_whose_outcomes_do_not_add_up_to_the_rows_read_settles_halted() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var checkpoint = await RunWithOneRowUnaccountedFor(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("read 4 rows").And.Contain("accounted for 3")); + Assert.That(checkpoint.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon")); + } + } + + [Test] + public async Task A_required_category_whose_outcomes_do_not_add_up_to_the_rows_read_settles_halted() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var checkpoint = await RunWithOneRowUnaccountedFor(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Required)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("read 4 rows").And.Contain("accounted for 3")); + Assert.That(checkpoint.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon")); + } + } + + // The target commits a checkpoint and a result that agree with each other, so only the rows read can show the one it lost. + static Task RunWithOneRowUnaccountedFor(MigrationCategory category) + { + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 4, UnderReportBy = 1 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + return engine.RunCategoryAsync(category); + } + + [Test] + public async Task A_category_with_no_rows_finishes_Complete_without_a_cursor() + { + // The ordinary state of several required categories on a small instance: nothing to copy is a + // finished category, not a category that never ran. + var category = MigrationCategoryRegistry.Find("MessageRedirects")!; + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((0L, 0L))); + Assert.That(checkpoint.Cursor, Is.Null); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + } + } + + [Test] + public async Task The_moment_a_category_settles_comes_from_the_injected_clock() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var settledAt = new DateTimeOffset(2026, 3, 4, 5, 6, 7, TimeSpan.Zero); + var clock = new FakeTimeProvider(settledAt); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.SettledAt, Is.EqualTo(settledAt.UtcDateTime), "a wall-clock read here would drift from every other time the migration reports"); + } + + [Test] + public async Task A_category_already_CompleteWithErrors_is_left_alone_on_a_second_run() + { + // Failed waits for the operator, so a start returns the row as it is rather than reading the category again. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var finishedWithSkips = new MigrationCheckpoint(category.Id, MigrationCategoryState.CompleteWithErrors, "b", 1, 1, 2, + new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 }, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + await checkpointStore.Upsert(finishedWithSkips); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(finishedWithSkips with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_category_already_Halted_is_left_alone_on_a_second_run() + { + // Failed waits for the operator, so a start returns the row as it is rather than reading the category again. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var halted = new MigrationCheckpoint(category.Id, MigrationCategoryState.Halted, "a", 1, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the target was unreachable"); + await checkpointStore.Upsert(halted); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(halted with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_category_already_Complete_is_left_alone_on_a_second_run() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var alreadyDone = new MigrationCheckpoint(category.Id, MigrationCategoryState.Complete, "a", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + await checkpointStore.Upsert(alreadyDone); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(alreadyDone with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs new file mode 100644 index 0000000000..9d02e7bb4d --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs @@ -0,0 +1,454 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineFailurePathTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationEngine BuildEngine(InMemoryMigrationSource source, InMemoryMigrationCheckpointStore checkpointStore, InMemoryMigrationTarget target) => + new(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + [Test] + public async Task A_failing_write_halts_the_category_and_records_the_error_instead_of_throwing() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Required)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("Simulated failure")); + // The first batch committed, so the cursor is real and a later run resumes from it. + Assert.That(checkpoint.Cursor, Is.EqualTo("b")); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + } + } + + [Test] + public async Task The_halt_is_durable_so_the_health_check_and_the_guard_can_read_it() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 1 }; + var engine = BuildEngine(source, checkpointStore, target); + + await engine.RunCategoryAsync(category); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(persisted.LastError, Is.Not.Null); + } + } + + [Test] + public void A_cancelled_run_is_a_shutdown_rather_than_a_failure_and_is_not_swallowed() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (1, stopping) }; + var engine = BuildEngine(source, checkpointStore, target); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category, stopping.Token)); + } + + [Test] + public async Task A_halted_category_is_left_alone_until_the_operator_puts_it_back_in_progress() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var halted = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); + + // The cause is fixed, but nothing re-runs a Failed category until the operator says so. + target.FailOnCallNumber = null; + var untouched = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + // The reset --migration-retry writes: a re-read from the start that keeps only when the category first started. + await checkpointStore.Upsert(untouched with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + SkipReasons = null, + SettledAt = null, + LastError = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0 + }); + var finished = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(untouched, Is.EqualTo(halted), "a start re-ran a Failed category the operator had not put back"); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.LastError, Is.Null, "a retried category does not keep the error it failed with"); + Assert.That(finished.SourceTotal, Is.Null); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EquivalentTo(new[] { "a", "b", "c", "d" })); + // Rows a and b were committed before the failure, so the re-read finds them already present. + Assert.That((finished.CopiedCount, finished.AlreadyPresentCount), Is.EqualTo((2L, 2L))); + } + } + + [Test] + public async Task Body_skips_in_a_batch_whose_write_fails_are_counted_once_across_the_restart() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + // msg-1's body is unreadable on both runs: every attempt fails on each. + source.FailBodyReads("msg-1", 2 * MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 1 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var firstRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var halted = await firstRun.RunCategoryAsync(category); + + // The row a crash just before the failing batch commits leaves, which a start resumes from its cursor. + await checkpointStore.Upsert(halted with { State = MigrationCategoryState.InProgress, LastError = null, SettledAt = null }); + target.FailOnCallNumber = null; + var secondRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var finished = await secondRun.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); + // The failed batch never committed, so its skip is recorded only when the batch is read again. + Assert.That(halted.SkippedCount, Is.Zero); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(finished.SkippedCount, Is.EqualTo(1)); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2" })); + } + } + + [Test] + public async Task An_abandoned_category_is_left_exactly_as_it_is() + { + // Abandoned is the one end a person chooses: this category will never be copied, and the + // guard and the health check both treat it as settled. A later run must not quietly restart it. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var abandoned = new MigrationCheckpoint(category.Id, MigrationCategoryState.Abandoned, "a", 1, 3, 4, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the body store was unreachable"); + await checkpointStore.Upsert(abandoned); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(abandoned with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_target_with_no_batch_size_for_a_category_halts_it_and_the_next_category_still_runs() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var unmapped = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var next = MigrationCategoryRegistry.Find("MessageRedirects")!; + source.Seed(unmapped.Id, Row("k-1")); + source.Seed(next.Id, Row("r-1")); + var target = new InMemoryMigrationTarget(checkpointStore) { NoBatchSizeFor = unmapped.Id }; + var engine = BuildEngine(source, checkpointStore, target); + + var results = await engine.RunCategories([unmapped, next]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[0].LastError, Does.Contain("No batch size")); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public void A_write_failure_halt_logs_its_exception_before_it_settles() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { FailOnCallNumber = 1 }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), logger); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Exception?.Message), Does.Contain("Simulated failure on write 1")); + } + + [Test] + public void A_threshold_halt_logs_its_reason_before_it_settles() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("a", MigrationSkipReason.BodyUnreadable); + var logger = new CapturingLogger(); + // A floor of zero lets the one rejected row halt the category. + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 0, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Message), Has.Some.Contains("Halted: 1 of 1 rows skipped")); + } + + [Test] + public void An_optional_threshold_halt_whose_save_fails_throws_rather_than_staying_copying() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("a", MigrationSkipReason.BodyUnreadable); + // A floor of zero lets the one rejected row halt the category. + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 0, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + var stored = checkpointStore.Read(category.Id).GetAwaiter().GetResult(); + using (Assert.EnterMultipleScope()) + { + Assert.That(stored!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(stored.LastError, Is.Null, "the failed save of a halt was recorded as the category's own error, which leaves a Failed category reading as still copying"); + } + } + + [Test] + public async Task An_optional_category_that_throws_stays_copying_with_the_error_and_its_cursor() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + + var checkpoint = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + var stored = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + foreach (var row in new[] { checkpoint, stored! }) + { + Assert.That(row.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(row.Cursor, Is.EqualTo("b")); + Assert.That(row.CopiedCount, Is.EqualTo(2)); + Assert.That(row.SettledAt, Is.Null); + Assert.That(row.LastError, Does.Contain("Simulated failure")); + Assert.That(row.LastError, Does.Not.Contain("--migration-abandon").And.Not.Contain("--migration-retry")); + } + } + } + + [Test] + public async Task An_optional_category_an_exception_left_copying_resumes_on_the_next_start_and_clears_its_error() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + target.FailOnCallNumber = null; + var finished = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.LastError, Is.Null, "the error stays on the row only until a start tries again"); + Assert.That(finished.CopiedCount, Is.EqualTo(4)); + Assert.That(target.RowsHandedToWrite(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "a", "b", "c", "d" }), "the resume did not carry on from the cursor"); + } + } + + [Test] + public async Task A_checkpoint_conflict_leaves_the_other_writer_alone_instead_of_halting_over_it() + { + // Two hosts pointed at one target is what the version token exists for. Settling this as halted + // would write over the progress of whichever host is still copying. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + // Save 1 moves the row to in progress; save 2 is the first batch, by which point the other host has moved it on. + var checkpointStore = new ConflictOnNthSaveCheckpointStore { ConflictOnSave = 2 }; + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + var persisted = await checkpointStore.Read(category.Id); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "no halted row was written over the conflict"); + } + + [Test] + public async Task A_cancellation_that_is_not_a_shutdown_halts_the_category_like_any_other_failure() + { + // An inner timeout surfaces as the same exception type as a host stopping, and only the token + // says which. Treating a timeout as a shutdown would end the run with no reason on the row. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) + { + FailOnCallNumber = 1, + FailWith = new OperationCanceledException("the query timed out") + }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("OperationCanceledException").And.Contain("the query timed out")); + } + } + + [Test] + public async Task The_halt_reason_names_the_cursor_the_copy_had_reached() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.LastError, Does.Contain("at cursor b"), "the cursor is the only pointer an operator has to where it stopped"); + } + + [Test] + public async Task The_halt_reason_says_at_the_start_when_the_first_batch_never_committed() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { FailOnCallNumber = 1 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.LastError, Does.Contain("at the start")); + } + + [Test] + public async Task Every_row_the_target_skips_is_named_in_the_log() + { + // The counts say how much was left behind. Only the log says which rows, and it is the way back + // to them while the RavenDB database still exists. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("b", MigrationSkipReason.PastRetention); + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), logger); + + await engine.RunCategoryAsync(category); + + Assert.That(logger.Entries.Select(entry => entry.Message), Has.Some.EqualTo("Skipped b in category KnownEndpoints")); + } + + [Test] + public async Task A_stop_between_batches_leaves_the_row_in_progress_at_the_batch_that_committed() + { + // The stop lands in the source rather than in a write, so nothing is mid-transaction: the row + // still has to describe the batches that did commit, and stay resumable. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, CancelOnCall = (1, stopping) }; + var engine = BuildEngine(source, checkpointStore, target); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category, stopping.Token)); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown is not a halt"); + Assert.That(persisted.Cursor, Is.EqualTo("b")); + Assert.That(persisted.CopiedCount, Is.EqualTo(2)); + Assert.That(persisted.SettledAt, Is.Null); + } + } + + // Another host moved the row on between this host reading it and saving it. + sealed class ConflictOnNthSaveCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore saved = new(); + int saves; + + public int ConflictOnSave { get; init; } + + public Task> ReadAll(CancellationToken cancellationToken = default) => saved.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => saved.Read(categoryId, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + ++saves == ConflictOnSave + ? throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but the stored row has moved on.") + : saved.Upsert(checkpoint, cancellationToken); + } + + // The store shares the target's database, which has become unreachable by the time the halt is saved. + sealed class HaltSaveFailsCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore saved = new(); + + public Task> ReadAll(CancellationToken cancellationToken = default) => saved.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => saved.Read(categoryId, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + checkpoint.State == MigrationCategoryState.Halted ? throw new TimeoutException("checkpoint store unreachable") : saved.Upsert(checkpoint, cancellationToken); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs new file mode 100644 index 0000000000..0ece00a601 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -0,0 +1,326 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineHaltTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task A_systemic_failure_halts_the_category_partway_through() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + // Every 5th of 1,000 rows is rejected: a steady 20% spread evenly rather than clustered at the + // start, past both the 5% proportion and the 100-row floor. + var rows = Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}")).ToArray(); + source.Seed(category.Id, rows); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("Halted")); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + // Stopped partway: the 1,000th row was never reached. + Assert.That(target.WrittenRows(category.Id).Count, Is.LessThan(800)); + } + } + + [Test] + public async Task Rows_the_target_would_have_deleted_anyway_never_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + // The same 20% that halts the category above, except these rows are past the target's retention + // cutoff, so leaving them behind is the copy working rather than failing. + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((800L, 200L))); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(800), "the last row was reached, so nothing halted partway"); + } + } + + [TestCase(25, MigrationCategoryState.CompleteWithErrors, TestName = "A_mix_of_benign_and_fault_skips_runs_on_while_the_faults_stay_under_the_threshold")] + [TestCase(10, MigrationCategoryState.Halted, TestName = "A_mix_of_benign_and_fault_skips_halts_once_the_faults_alone_pass_the_threshold")] + public async Task A_batch_mixing_benign_and_fault_skips_is_judged_on_the_faults_alone(int everyNthIsAFault, MigrationCategoryState expected) + { + // A real archive copy loses rows both ways at once: retention takes some, unreadable bodies take + // others. This is the only shape where the sum has to leave some reasons out. + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 5_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + // A fifth of the category is past retention either way, which on its own is four times the threshold. + foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention); + } + // The offset keeps the faults clear of the benign rows: 4% of the category in one case, 10% in the other. + foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % everyNthIsAFault == 3)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(expected)); + } + + [Test] + public async Task Rows_already_present_keep_a_category_under_the_halt_threshold() + { + // Already-present rows are in the denominator because the run did handle them. Drop them from it + // and this category's 4% fault rate reads as 12%, halting a copy that is merely being re-run. + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 3_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 2_000)) + { + target.SeedExistingKey($"row-{i}"); + } + // 120 faults spread through the whole category: past the 100-row floor, and 4% of 3,000. + foreach (var i in Enumerable.Range(1, 3_000).Where(i => i % 25 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(120)); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(1_920), "the 80 already-present rows that are also faults are refused before the collision check"); + } + } + + [Test] + public async Task Rows_already_present_in_the_target_never_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.SeedExistingKey($"row-{i}"); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(800)); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(200)); + } + } + + [Test] + public async Task A_resumed_run_counts_only_its_own_skips_and_keeps_the_earlier_ones() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + // What a shutdown after the sixth batch leaves: 120 fault skips in 600 rows, past both the floor and 5% if counted again. + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.InProgress, "row-600", 480, 120, null, + new Dictionary { [MigrationSkipReason.BodyUnreadable] = 120 }, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + + var finished = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the skips still on the row must not halt a run that skips nothing"); + Assert.That((finished.CopiedCount, finished.SkippedCount), Is.EqualTo((880L, 120L)), "copied, skipped at the end"); + } + } + + [Test] + public async Task A_category_smaller_than_the_floor_that_loses_every_row_ends_Failed_rather_than_Done() + { + // Ninety rows is under the hundred-row floor, so the threshold the engine checks after every batch + // can never fire, however many rows are lost. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 90).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 30 }; + foreach (var i in Enumerable.Range(1, 90)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.RequiredValueMissing); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.State.IsFinished(), Is.False, "a required category that copied nothing must not let the host open"); + Assert.That(checkpoint.CopiedCount, Is.Zero); + } + } + + [Test] + public async Task A_halted_category_is_returned_untouched_and_its_source_is_not_read() + { + // Failed waits for the operator, so a start that read the category again would copy rows nobody asked for. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 90).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var halted = new MigrationCheckpoint(category.Id, MigrationCategoryState.Halted, null, 0, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the target was unreachable"); + await checkpointStore.Upsert(halted); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 30 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(halted with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.RowsHandedToWrite(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_small_category_losing_rows_the_product_would_drop_anyway_still_completes() + { + // Harmless skips are rows the target would have deleted anyway, so no number of them may stop a + // category or leave it Failed, however small the category is. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 90).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 30 }; + foreach (var i in Enumerable.Range(1, 90)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + } + + [Test] + public async Task A_run_that_ends_with_one_fault_skip_settles_CompleteWithErrors_however_small_the_share() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + target.RejectKey("row-500", MigrationSkipReason.RequiredValueMissing); + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "one row in a thousand is far under the threshold, and it is still a row the product wanted"); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((999L, 1L))); + } + } + + [Test] + public async Task A_run_whose_only_skips_are_harmless_settles_Complete_and_keeps_its_reasons() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + target.RejectKey("a", MigrationSkipReason.PastRetention); + target.RejectKey("b", MigrationSkipReason.PastRetention); + target.RejectKey("c", MigrationSkipReason.BlankGroupComment); + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((1L, 3L)), "a harmless skip is still a skip, and status and verify count it"); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary + { + [MigrationSkipReason.PastRetention] = 2, + [MigrationSkipReason.BlankGroupComment] = 1 + })); + } + } + + [Test] + public async Task The_threshold_halt_names_its_counts_and_no_command() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.ArchivedAndResolvedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + + var checkpoint = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + // 120 skips in 600 rows is the first point past both the floor and 5%. + Assert.That(checkpoint.LastError, Does.Contain("120 of 600").And.Contain("5%").And.Contain("100 rows")); + // The engine cannot know which commands a category may take, so the readers of the row add them. + Assert.That(checkpoint.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon").And.Not.Contain("restart to resume")); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs new file mode 100644 index 0000000000..71e62dac7b --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs @@ -0,0 +1,88 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using NUnit.Framework; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationEngineOptionsTests +{ + static readonly SettingsRootNamespace Namespace = new("ServiceControl"); + static readonly TimeSpan EventRetention = TimeSpan.FromDays(3); + static readonly TimeSpan ErrorRetention = TimeSpan.FromDays(11); + + [TearDown] + public void ClearEnvironmentVariables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", null); + } + + static MigrationEngineOptions Read() => MigrationEngineOptions.FromSettings(Namespace, EventRetention, ErrorRetention); + + [Test] + public void Defaults_match_the_contract_when_nothing_is_configured() + { + var options = Read(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(100))); + Assert.That(options.HaltThresholdPercent, Is.EqualTo(5)); + Assert.That(options.HaltThresholdMinimum, Is.EqualTo(100)); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages }), "both optional categories are copied unless turned off"); + Assert.That(options.EventLogWindow, Is.EqualTo(EventRetention), "the event log window defaults to the event retention period"); + Assert.That(options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(ErrorRetention), "the archived and resolved window defaults to the error retention period"); + Assert.That(options.BodyRetryBackoff, Is.EqualTo(TimeSpan.FromMilliseconds(200))); + } + } + + [Test] + public void Reads_configured_values_from_environment_variables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", "2500"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", "10"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", "50"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", "2.00:00:00"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "5.12:00:00"); + + var options = Read(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(2500))); + Assert.That(options.HaltThresholdPercent, Is.EqualTo(10)); + Assert.That(options.HaltThresholdMinimum, Is.EqualTo(50)); + Assert.That(options.EventLogWindow, Is.EqualTo(TimeSpan.FromDays(2))); + Assert.That(options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(TimeSpan.FromDays(5.5))); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages })); + } + } + + [TestCase("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", MigrationCategoryIds.ArchivedAndResolvedFailedMessages)] + [TestCase("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", MigrationCategoryIds.EventLog)] + public void A_zero_window_turns_its_category_off(string variable, string stillSelected) + { + Environment.SetEnvironmentVariable(variable, "0"); + + var options = Read(); + + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { stillSelected })); + } + + [TestCase("a week")] + [TestCase("-1.00:00:00")] + public void Refuses_a_window_that_is_not_a_time_span_of_zero_or_more(string value) + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", value); + + var ex = Assert.Throws(() => Read()); + + Assert.That(ex.Message, Does.Contain(value).And.Contain(MigrationSettings.EventLogWindowKey), "the refusal has to name both the value and the setting holding it for the customer to fix it"); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs new file mode 100644 index 0000000000..056805e938 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs @@ -0,0 +1,172 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineOrderingTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationEngine BuildEngine(InMemoryMigrationSource source, InMemoryMigrationCheckpointStore checkpointStore, InMemoryMigrationTarget target) => + new(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + [Test] + public async Task LicensingThroughput_does_not_start_before_LicensingEndpoints_completes_and_says_so_on_its_checkpoint_row() + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + // LicensingEndpoints has never run: no checkpoint row for it at all. + var checkpoint = await engine.RunCategoryAsync(throughputCategory); + + var persisted = await checkpointStore.Read(throughputCategory.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(target.WrittenRows(throughputCategory.Id), Is.Empty); + // A row exists, so status can print it. Without one, an operator cannot tell a category + // waiting on another from a category nobody asked for. + Assert.That(persisted, Is.Not.Null); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(persisted.LastError, Does.Contain("LicensingEndpoints")); + } + } + + [Test] + public async Task EndpointSettings_does_not_start_before_KnownEndpoints_completes() + { + // Not a foreign key: settings wait for known endpoints, and a halted required category keeps + // the host, and with it the heartbeat sync, closed. + var settingsCategory = MigrationCategoryRegistry.Find("EndpointSettings")!; + var source = new InMemoryMigrationSource(); + source.Seed(settingsCategory.Id, Row("EndpointSettings/1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(settingsCategory); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(target.WrittenRows(settingsCategory.Id), Is.Empty); + } + } + + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + public async Task GroupComments_does_not_start_before_the_unresolved_failed_messages_complete(MigrationCategoryState predecessorState) + { + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("UnresolvedAndRetryIssuedFailedMessages", predecessorState, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow UnresolvedAndRetryIssuedFailedMessages, which is {predecessorState}")); + Assert.That(target.WrittenRows(comments.Id), Is.Empty); + } + } + + [Test] + public async Task A_blocked_category_runs_once_the_category_it_follows_settles() + { + // A block nothing can clear would strand group comments and keep the host closed for good. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var unresolved = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var predecessorRunning = new MigrationCheckpoint(unresolved.Id, MigrationCategoryState.InProgress, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null); + var saved = await checkpointStore.Upsert(predecessorRunning); + var target = new InMemoryMigrationTarget(checkpointStore); + + var blocked = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + await checkpointStore.Upsert(saved with { State = MigrationCategoryState.Complete, SettledAt = DateTime.UtcNow }); + var unblocked = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(blocked.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(unblocked.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(unblocked.LastError, Is.Null, "the block reason does not outlive the block"); + Assert.That(target.WrittenRows(comments.Id), Has.Count.EqualTo(1)); + } + } + + [Test] + public async Task A_blocked_category_has_not_settled_because_it_has_not_finished() + { + // Anything reading SettledAt beside State would otherwise take a blocked category for a finished one. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + + await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + var persisted = await checkpointStore.Read(comments.Id); + Assert.That(persisted!.SettledAt, Is.Null); + } + + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.Blocked)] + public async Task A_predecessor_that_has_a_row_but_has_not_run_is_named_by_the_state_on_that_row(MigrationCategoryState predecessorState) + { + // A missing row reads as "not started"; a row that exists says what it actually holds, which is + // how an operator tells a category waiting its turn from one waiting on a chain. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("UnresolvedAndRetryIssuedFailedMessages", predecessorState, null, 0, 0, null, null, null, null, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + + var checkpoint = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow UnresolvedAndRetryIssuedFailedMessages, which is {predecessorState}")); + } + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Abandoned)] + public async Task LicensingThroughput_proceeds_once_LicensingEndpoints_is_finished_or_abandoned(MigrationCategoryState endpointsState) + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", endpointsState, "e-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(throughputCategory); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs new file mode 100644 index 0000000000..3a2dca44d7 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs @@ -0,0 +1,172 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineResumeTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task A_restart_keeps_the_moment_the_category_first_started() + { + // StartedAt is what an operator reads to see how long a background copy has been going. Stamping + // it again on every restart would report minutes for a copy that has been running for days. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstStart = new DateTimeOffset(2026, 3, 1, 9, 0, 0, TimeSpan.Zero); + var halted = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(firstStart), options, NullLogger.Instance) + .RunCategoryAsync(category); + + // A day later, the cause is fixed and the operator puts the category back as --migration-retry does. + target.FailOnCallNumber = null; + await checkpointStore.Upsert(halted with + { + State = MigrationCategoryState.NotStarted, + Cursor = null, + SkipReasons = null, + SettledAt = null, + LastError = null, + CopiedCount = 0, + SkippedCount = 0, + AlreadyPresentCount = 0 + }); + var restart = firstStart.AddDays(1); + var finished = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(restart), options, NullLogger.Instance) + .RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(halted.StartedAt, Is.EqualTo(firstStart.UtcDateTime)); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.StartedAt, Is.EqualTo(firstStart.UtcDateTime), "a retried category keeps the moment it first started"); + Assert.That(finished.SettledAt, Is.EqualTo(restart.UtcDateTime)); + } + } + + [Test] + public async Task Restarting_after_a_mid_category_stop_produces_no_duplicates_and_no_gaps() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + var allIds = Enumerable.Range(1, 6).Select(i => $"row-{i}").ToArray(); + source.Seed(category.Id, [.. allIds.Select(Row)]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (3, stopping) }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + // Batches are 2 rows each; the host stops on the 3rd call to Write, so exactly 2 batches (4 rows) land. + Assert.ThrowsAsync(() => firstEngine.RunCategoryAsync(category, stopping.Token)); + + var afterStop = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + // Still InProgress, not Halted: a cancelled run is a shutdown, not a failure. + Assert.That(afterStop!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(afterStop.CopiedCount, Is.EqualTo(4)); + Assert.That(afterStop.Cursor, Is.EqualTo("row-4")); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(4)); + } + + // Restart: same source, same target, same checkpoint store, nothing stopping it this time. + target.StopOnCall = null; + var secondEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var finalCheckpoint = await secondEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finalCheckpoint.CopiedCount, Is.EqualTo(6)); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EquivalentTo(allIds), "no gaps"); + // The target de-duplicates, as the real ones do, so what it kept can never show a row sent twice. + Assert.That(target.RowsHandedToWrite(category.Id).Select(r => r.SourceId), Is.Unique, "no duplicates: the restart resumes past the rows the first run committed"); + } + } + + [Test] + public async Task A_graceful_stop_saves_the_real_split_of_the_last_committed_batch() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 6).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (3, stopping) }; + // Both in the second batch, the last one to commit before the stop. + target.SeedExistingKey("row-3"); + target.RejectKey("row-4", MigrationSkipReason.BodyUnreadable); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + Assert.ThrowsAsync(() => firstEngine.RunCategoryAsync(category, stopping.Token)); + var afterStop = await checkpointStore.Read(category.Id); + + target.StopOnCall = null; + var secondEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var finalCheckpoint = await secondEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(afterStop!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That((afterStop.CopiedCount, afterStop.SkippedCount, afterStop.AlreadyPresentCount), Is.EqualTo((2L, 1L, 1L)), "copied, skipped, already present after the stop"); + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((4L, 1L, 1L)), "copied, skipped, already present at the end"); + Assert.That(finalCheckpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); + } + } + + [Test] + public async Task A_hard_crash_after_a_committed_write_loses_nothing_because_the_target_saved_the_real_split() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 4).Select(i => Row($"row-{i}"))]); + var committed = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(committed) { DefaultBatchSize = 2 }; + target.SeedExistingKey("row-3"); + target.RejectKey("row-4", MigrationSkipReason.BodyUnreadable); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var crashingEngine = new MigrationEngine(source, target, new CrashAfterCommitCheckpointStore(committed, crashAfterCursor: "row-4"), new FakeTimeProvider(), options, NullLogger.Instance); + await crashingEngine.RunCategoryAsync(category); + + var restartedEngine = new MigrationEngine(source, target, committed, new FakeTimeProvider(), options, NullLogger.Instance); + var finalCheckpoint = await restartedEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + // The engine's own settle was lost, but every count came from the target's own transaction. + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((2L, 1L, 1L)), "copied, skipped, already present at the end"); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "row-1", "row-2" })); + } + } + + // Once the target has committed crashAfterCursor, the engine's own saves are lost, as if the process died there. + sealed class CrashAfterCommitCheckpointStore(IMigrationCheckpointStore committed, string crashAfterCursor) : IMigrationCheckpointStore + { + public Task> ReadAll(CancellationToken cancellationToken = default) => committed.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => committed.Read(categoryId, cancellationToken); + + public async Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + (await committed.Read(checkpoint.CategoryId, cancellationToken))?.Cursor == crashAfterCursor + ? checkpoint + : await committed.Upsert(checkpoint, cancellationToken); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs new file mode 100644 index 0000000000..d08b17f548 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs @@ -0,0 +1,158 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineRunCategoriesTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Runs_every_category_it_is_given_in_the_order_it_is_given_them() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + // The reverse of registry order, so an engine that re-sorted by Order would run KnownEndpoints first. + MigrationCategory[] given = [MigrationCategoryRegistry.Find("MessageRedirects")!, MigrationCategoryRegistry.Find("KnownEndpoints")!]; + foreach (var category in given) + { + source.Seed(category.Id, Row($"{category.Id}-1")); + } + + var results = await engine.RunCategories(given); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(c => c.CategoryId), Is.EqualTo(new[] { "MessageRedirects", "KnownEndpoints" })); + Assert.That(results.Select(c => c.State), Is.All.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public async Task Every_category_it_is_given_has_a_row_before_the_first_one_copies_anything() + { + // A copy stopped between two categories must not leave a table that reads as finished, because the + // gates that keep a host off an unfinished copy look only at the rows that exist. + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + IReadOnlyList? rowsAtFirstWrite = null; + var target = new InMemoryMigrationTarget(checkpointStore) { BeforeWrite = _ => rowsAtFirstWrite ??= checkpointStore.ReadAll().GetAwaiter().GetResult() }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + // Three, so recording only the next category would still leave the last one missing. + MigrationCategory[] given = [MigrationCategoryRegistry.Find("KnownEndpoints")!, MigrationCategoryRegistry.Find("EndpointSettings")!, MigrationCategoryRegistry.Find("MessageRedirects")!]; + foreach (var category in given) + { + source.Seed(category.Id, Row($"{category.Id}-1")); + } + + await engine.RunCategories(given); + + Assert.That(rowsAtFirstWrite, Is.Not.Null, "the copy never wrote a batch"); + using (Assert.EnterMultipleScope()) + { + Assert.That(rowsAtFirstWrite!.Select(c => c.CategoryId), Is.EquivalentTo(new[] { "KnownEndpoints", "EndpointSettings", "MessageRedirects" })); + Assert.That(rowsAtFirstWrite!.Where(c => c.CategoryId != "KnownEndpoints").Select(c => c.State), Is.All.EqualTo(MigrationCategoryState.NotStarted)); + } + } + + [Test] + public async Task A_halted_category_is_left_untouched_and_the_next_category_still_runs() + { + var firstStarted = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var halted = await checkpointStore.Upsert(new MigrationCheckpoint("KnownEndpoints", MigrationCategoryState.Halted, "KnownEndpoints-1", 1, 0, null, null, firstStarted, firstStarted, firstStarted, "Halted: earlier run")); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + source.Seed("KnownEndpoints", Row("KnownEndpoints-1"), Row("KnownEndpoints-2")); + // MessageRedirects does not follow KnownEndpoints, so a Failed KnownEndpoints cannot hold it back. + source.Seed("MessageRedirects", Row("MessageRedirects-1")); + + var results = await engine.RunCategories([MigrationCategoryRegistry.Find("KnownEndpoints")!, MigrationCategoryRegistry.Find("MessageRedirects")!]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(target.RowsHandedToWrite("KnownEndpoints"), Is.Empty, "a Failed category waits for the operator"); + Assert.That(results[0], Is.EqualTo(halted), "the halted category's row was changed"); + Assert.That(await checkpointStore.Read("KnownEndpoints"), Is.EqualTo(halted)); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(target.WrittenRows("MessageRedirects").Select(row => row.SourceId), Is.EqualTo(new[] { "MessageRedirects-1" })); + } + } + + [Test] + public async Task A_finished_category_is_not_copied_again() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("KnownEndpoints", MigrationCategoryState.Complete, "KnownEndpoints-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + source.Seed("KnownEndpoints", Row("KnownEndpoints-1")); + + var results = await engine.RunCategories([MigrationCategoryRegistry.Find("KnownEndpoints")!, MigrationCategoryRegistry.Find("EndpointSettings")!]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(target.RowsHandedToWrite("KnownEndpoints"), Is.Empty); + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public async Task Running_no_categories_copies_nothing_and_reports_nothing() + { + // What an instance that selected no optional categories asks for on every background pass. + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + var results = await engine.RunCategories([]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results, Is.Empty); + Assert.That(await checkpointStore.ReadAll(), Is.Empty, "a category nobody ran gets no row"); + } + } + + [Test] + public async Task One_category_halting_does_not_stop_the_ones_after_it() + { + // Everything that can still be copied is copied, and the guard decides what an incomplete migration + // may do. Stopping at the first halt would leave later categories untouched with no reason recorded. + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1, FailOnCallNumber = 1 }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + var first = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var second = MigrationCategoryRegistry.Find("MessageRedirects")!; + source.Seed(first.Id, Row("k-1")); + source.Seed(second.Id, Row("r-1")); + + var results = await engine.RunCategories([first, second]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs new file mode 100644 index 0000000000..374a06de68 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -0,0 +1,204 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineSkipReasonTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Reasons_the_target_reports_add_up_across_batches_on_the_checkpoint() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + // Two reasons over two batches: a and b land in the first, d in the second, so the saved map has to merge both. + target.RejectKey("a", MigrationSkipReason.RequiredValueMissing); + target.RejectKey("b", MigrationSkipReason.RequiredValueMissing); + target.RejectKey("d", MigrationSkipReason.PastRetention); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.SkippedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2, [MigrationSkipReason.PastRetention] = 1 })); + Assert.That((await checkpointStore.Read(category.Id))!.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2, [MigrationSkipReason.PastRetention] = 1 })); + } + } + + [Test] + public async Task A_message_whose_body_is_never_read_is_counted_under_BodyUnreadable() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("down")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); + } + + [Test] + public async Task A_batch_that_loses_rows_two_different_ways_records_both_reasons() + { + // The breakdown is what tells a customer what they lost and why. A merge that overwrote instead of + // summing would leave the total right and the reasons wrong, and verification would still balance. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2"), Row("msg-3")); + // One the engine skips itself because the body will not read, one the target refuses. + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + target.RejectKey("msg-2", MigrationSkipReason.PastRetention); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary + { + [MigrationSkipReason.BodyUnreadable] = 1, + [MigrationSkipReason.PastRetention] = 1 + })); + Assert.That(target.WrittenRows(category.Id).Select(row => row.SourceId), Is.EqualTo(new[] { "msg-3" })); + } + } + + [Test] + public async Task A_target_whose_skip_reasons_do_not_add_up_to_its_skips_halts_the_category() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new UnexplainedSkipTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("reported 1 skipped").And.Contain("reasons for 0")); + Assert.That(checkpoint.Cursor, Is.Null, "the target refused ahead of its commit, so nothing was written and a restart reads the batch again"); + } + } + + [Test] + public async Task A_target_whose_reported_counts_disagree_with_the_checkpoint_it_committed_never_finishes_the_category() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 300).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new MiscountedCopyTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var settled = await engine.RunCategoryAsync(category); + + var stored = await checkpointStore.Read(category.Id); + + using (Assert.EnterMultipleScope()) + { + Assert.That(settled.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(stored!.LastError, Does.Contain("but the checkpoint it committed moved by"), "the halt has to name which two accounts disagreed, or the operator goes looking for the wrong problem"); + Assert.That(stored.State.IsFinished(), Is.False, "the halt threshold never saw the 150 rows the target dropped, so the category settled finished and the host opened on half a category"); + } + } + + [Test] + public async Task An_optional_category_whose_reported_counts_disagree_with_the_checkpoint_it_committed_settles_halted() + { + // An optional category stays copying after an exception, and this check must not become one of those. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 300).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new MiscountedCopyTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var settled = await engine.RunCategoryAsync(category); + + var stored = await checkpointStore.Read(category.Id); + + using (Assert.EnterMultipleScope()) + { + Assert.That(category.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + Assert.That(settled.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(stored!.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(stored.LastError, Does.Contain("reported copying 10").And.Contain("but the checkpoint it committed moved by")); + Assert.That(stored.LastError, Does.Not.Contain("--migration-retry").And.Not.Contain("--migration-abandon")); + } + } + + // Commits half of every batch as skipped and reports the whole batch copied. The saved counts still add up + // to the source total, so nothing later in the run can notice, and the halt threshold sees a clean copy. + sealed class MiscountedCopyTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(10); + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var skipped = batch.Rows.Count / 2; + var reasons = new Dictionary { [MigrationSkipReason.RequiredValueMissing] = skipped }; + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(batch.Rows.Count - skipped, skipped, 0, reasons), cancellationToken); + return new MigrationWriteResult(saved, batch.Rows.Count, 0, []); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); + } + + sealed class UnexplainedSkipTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task BatchSizeFor(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(10); + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(0, batch.Rows.Count, 0, null), cancellationToken); + return new MigrationWriteResult(saved, 0, batch.Rows.Count, []); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + + public IReadOnlyCollection SupportedCategoryIds => [.. MigrationCategoryRegistry.All.Select(category => category.Id)]; + + public IReadOnlyDictionary DocumentTypes => MigrationCategoryRegistry.All.ToDictionary(category => category.Id, _ => typeof(object)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs new file mode 100644 index 0000000000..e568edb6a1 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs @@ -0,0 +1,97 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineThrottleTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Optional_categories_pause_between_batches_for_the_configured_duration() + { + var category = MigrationCategoryRegistry.Find("EventLog")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + var clock = new TimerRecordingTimeProvider(); + var pause = TimeSpan.FromSeconds(1); + var options = new MigrationEngineOptions(pause, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var runTask = engine.RunCategoryAsync(category); + + // Three batches, so a pause before the second and another before the third. + for (var pauseNumber = 1; pauseNumber <= 2; pauseNumber++) + { + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, $"pause {pauseNumber} never started"); + Assert.That(runTask.IsCompleted, Is.False, $"pause {pauseNumber} should still be pending"); + clock.Advance(pause); + } + + var checkpoint = await runTask.WaitAsync(TimeSpan.FromSeconds(5)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(clock.DueTimes, Is.EqualTo(new[] { pause, pause })); + } + } + + [Test] + public async Task A_stop_during_a_pause_ends_the_run_as_a_shutdown() + { + // Most of a throttled background copy's life is spent in this pause, so it is where a host being + // stopped most often lands. + var category = MigrationCategoryRegistry.Find("EventLog")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + var clock = new TimerRecordingTimeProvider(); + using var stopping = new CancellationTokenSource(); + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var runTask = engine.RunCategoryAsync(category, stopping.Token); + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, "the first pause never started"); + await stopping.CancelAsync(); + + Assert.ThrowsAsync(() => runTask); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown mid-pause is not a halt"); + Assert.That(persisted.Cursor, Is.EqualTo("a"), "the batch before the pause committed"); + } + } + + [Test] + public async Task Required_categories_never_pause() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + // A FakeTimeProvider that is never advanced: if the engine tried to pause, this would hang + // until the test runner's own timeout, which WaitAsync turns into a clear failure instead. + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category).WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs new file mode 100644 index 0000000000..638efaaf1b --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationIsReleasedCheckTests.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationIsReleasedCheckTests +{ + [Test] + public void A_build_without_the_whole_migration_refuses_MigrationEnabled() + { + var exception = Assert.ThrowsAsync(() => new MigrationIsReleasedCheck().Run()); + + Assert.That(exception.Message, Does.Contain("does not yet carry the whole migration").And.Contain(MigrationSettings.EnabledKey), + "the customer has to learn that nothing will be copied, and which setting to turn back off"); + } + + // A RavenDB target fails the pair check, so this only passes while the release check runs before it. + [Test] + public void An_unreleased_build_refuses_before_any_other_check() + { + var settings = new Settings(transportType: "LearningTransport", persisterType: "RavenDB", errorRetentionPeriod: TimeSpan.FromDays(10)); + using var services = new ServiceCollection().BuildServiceProvider(); + + var exception = Assert.ThrowsAsync(() => MigrationStartup.RunRequiredCopy(services, settings)); + + Assert.That(exception.Message, Does.Contain("this build carries the whole migration").And.Not.Contain("supported pair")); + } + + [Test] + public void The_test_marker_lets_a_copy_run_on_an_unreleased_build() + { + Assert.DoesNotThrowAsync(() => new MigrationIsReleasedCheck(new AllowUnreleasedMigration()).Run()); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs new file mode 100644 index 0000000000..c25bf44d0c --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationPairIsSupportedCheckTests.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Linq; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence; + +[TestFixture] +class MigrationPairIsSupportedCheckTests +{ + [OneTimeSetUp] + public static void TheTreeIsBuilt() => + Assert.That( + PersistenceFactory.SqlPersistenceNames.Where(name => PersistenceManifestLibrary.Find(name) is null), + Is.Empty, + "These persistence manifests did not resolve, so this fixture cannot tell a supported pair from an unsupported one. Build the tree first with: dotnet build src --configuration Release -graph"); + + [TestCase("SQLServer")] + [TestCase("PostgreSQL")] + public void RavenDB_to_a_SQL_persister_is_supported(string target) => + Assert.DoesNotThrowAsync(() => Check(target).Run()); + + [Test] + public void A_RavenDB_target_is_refused_naming_the_target_setting() + { + var exception = Assert.ThrowsAsync(() => Check("RavenDB").Run()); + + Assert.That(exception.Message, Does.Contain("ServiceControl/PersistenceType")); + } + + static MigrationPairIsSupportedCheck Check(string target) => + new(new Settings(transportType: "LearningTransport", persisterType: target, errorRetentionPeriod: TimeSpan.FromDays(10))); +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs new file mode 100644 index 0000000000..2767013bb3 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationStartupCheckRunnerTests.cs @@ -0,0 +1,75 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationStartupCheckRunnerTests +{ + [Test] + public async Task Checks_run_in_the_order_they_were_given() + { + var ran = new List(); + + await MigrationStartupCheckRunner.Run( + [ + Check("the first", _ => ran.Add("first")), + Check("the second", _ => ran.Add("second")), + Check("the third", _ => ran.Add("third")) + ]); + + Assert.That(ran, Is.EqualTo(new[] { "first", "second", "third" }), "the databases are opened by checks that sit behind the ones refusing the configuration outright"); + } + + [Test] + public void The_first_check_to_refuse_stops_the_ones_behind_it() + { + var ran = new List(); + var failure = new InvalidOperationException("ServiceControl/RetryHistoryDepth is 0"); + + var exception = Assert.ThrowsAsync(() => MigrationStartupCheckRunner.Run( + [ + Check("the first", _ => ran.Add("first")), + Check("the migration target is ready", _ => throw failure), + Check("the last", _ => ran.Add("last")) + ])); + + using (Assert.EnterMultipleScope()) + { + Assert.That(ran, Is.EqualTo(new[] { "first" }), "a check behind a refusal would run against a configuration already known to be wrong, and the two that open a database are at the end of the list"); + Assert.That(exception!.Message, Does.Contain("the migration target is ready").And.Contain(failure.Message), "the operator needs to know which check refused and why"); + Assert.That(exception.InnerException, Is.SameAs(failure), "the original carries the stack and the detail the summary leaves out"); + } + } + + [Test] + public void A_host_being_stopped_is_not_a_check_refusing() + { + using var stopping = new CancellationTokenSource(); + stopping.Cancel(); + + var exception = Assert.ThrowsAsync(() => MigrationStartupCheckRunner.Run( + [Check("the first", token => token.ThrowIfCancellationRequested())], stopping.Token)); + + Assert.That(exception!.Message, Does.Not.Contain("Migration startup check"), "a stop dressed up as a refusal sends the operator after a setting that was never wrong"); + } + + static IMigrationStartupCheck Check(string name, Action run) => new FakeCheck(name, run); + + sealed class FakeCheck(string name, Action run) : IMigrationStartupCheck + { + public string Name => name; + + public Task Run(CancellationToken cancellationToken = default) + { + run(cancellationToken); + return Task.CompletedTask; + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs b/src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs new file mode 100644 index 0000000000..6f4393ab40 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/OptionalCategoryWindowsAreValidCheckTests.cs @@ -0,0 +1,51 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class OptionalCategoryWindowsAreValidCheckTests +{ + static readonly TimeSpan EventRetention = TimeSpan.FromDays(3); + static readonly TimeSpan ErrorRetention = TimeSpan.FromDays(11); + + [TearDown] + public void ClearWindows() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_EVENTLOGWINDOW", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", null); + } + + [Test] + public async Task Unset_windows_pass_and_default_to_the_retention_periods_it_was_given() + { + var check = new OptionalCategoryWindowsAreValidCheck(EventRetention, ErrorRetention); + + await check.Run(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(check.Options.EventLogWindow, Is.EqualTo(EventRetention)); + Assert.That(check.Options.ArchivedAndResolvedFailedMessagesWindow, Is.EqualTo(ErrorRetention)); + } + } + + [Test] + public void A_window_that_is_not_a_time_span_is_refused_and_named() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_ARCHIVEDANDRESOLVEDFAILEDMESSAGESWINDOW", "a week"); + + var exception = Assert.ThrowsAsync(async () => + await new OptionalCategoryWindowsAreValidCheck(EventRetention, ErrorRetention).Run()); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain("a week")); + Assert.That(exception.Message, Does.Contain(MigrationSettings.ArchivedAndResolvedFailedMessagesWindowKey)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs b/src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs new file mode 100644 index 0000000000..cc598257ab --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/RecordSourceOutageTests.cs @@ -0,0 +1,80 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class RecordSourceOutageTests +{ + static readonly MigrationCategory EventLog = MigrationCategoryRegistry.Find(MigrationCategoryIds.EventLog)!; + static readonly MigrationCategory ArchivedAndResolvedFailedMessages = MigrationCategoryRegistry.Find(MigrationCategoryIds.ArchivedAndResolvedFailedMessages)!; + + readonly InMemoryMigrationCheckpointStore checkpointStore = new(); + + [Test] + public async Task A_category_still_copying_keeps_its_state_and_gains_the_error() + { + await checkpointStore.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EventLog, MigrationCategoryState.InProgress, "EventLogItem/40", 40, 0, null, null, null, null, null, null)); + + await RecordOutage(); + + var row = await checkpointStore.Read(MigrationCategoryIds.EventLog); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(row.Cursor, Is.EqualTo("EventLogItem/40")); + Assert.That(row.CopiedCount, Is.EqualTo(40)); + Assert.That(row.SkippedCount, Is.Zero); + Assert.That(row.LastError, Does.Contain("RavenDB is unreachable").And.Contain("next start")); + } + } + + [Test] + public async Task A_category_with_no_row_gets_a_not_started_row_carrying_the_error() + { + await RecordOutage(); + + var row = await checkpointStore.Read(MigrationCategoryIds.ArchivedAndResolvedFailedMessages); + + Assert.That(row, Is.Not.Null, "status and verify read only the rows that exist, so a category with no row would show no error"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(row!.State, Is.EqualTo(MigrationCategoryState.NotStarted)); + Assert.That(row.CopiedCount, Is.Zero); + Assert.That(row.SkippedCount, Is.Zero); + Assert.That(row.AlreadyPresentCount, Is.Zero); + Assert.That(row.StartedAt, Is.Null, "a category that never started must not look as if it had"); + Assert.That(row.LastError, Does.Contain("RavenDB is unreachable").And.Contain("next start")); + } + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + [TestCase(MigrationCategoryState.Abandoned)] + public async Task A_settled_category_is_left_alone(MigrationCategoryState state) + { + var before = await checkpointStore.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EventLog, state, null, 0, 0, null, null, null, null, null, "earlier")); + + await RecordOutage(); + + var after = await checkpointStore.Read(MigrationCategoryIds.EventLog); + + using (Assert.EnterMultipleScope()) + { + Assert.That(after!.Version, Is.EqualTo(before.Version), "a settled row is never saved again"); + Assert.That(after.State, Is.EqualTo(state)); + Assert.That(after.LastError, Is.EqualTo("earlier")); + } + } + + Task RecordOutage() => + MigrationStartup.RecordSourceOutage(checkpointStore, [EventLog, ArchivedAndResolvedFailedMessages], new InvalidOperationException("RavenDB is unreachable")); +} diff --git a/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs new file mode 100644 index 0000000000..2077493bff --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/RequiredCopyGateTests.cs @@ -0,0 +1,395 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class RequiredCopyGateTests +{ + [TestCase(MigrationCategoryState.Complete)] + public void A_finished_required_category_lets_the_host_open(MigrationCategoryState state) => + Assert.DoesNotThrow(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(state)], NewSettings())); + + [TestCase(MigrationCategoryState.Halted)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.NotStarted)] + // Blocked happens because EndpointSettings must follow KnownEndpoints. + [TestCase(MigrationCategoryState.Blocked)] + public void A_required_category_that_is_not_finished_keeps_the_host_closed(MigrationCategoryState state) + { + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(state)], NewSettings())); + + Assert.That(exception.Message, Does.Contain(MigrationCategoryIds.EndpointSettings).And.Contain(state.ToString())); + } + + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Halted)] + public void An_unfinished_row_under_an_id_this_build_does_not_know_keeps_the_host_closed(MigrationCategoryState state) + { + var unknown = Checkpoint(state, UnknownCategoryId); + + var outside = MigrationStartup.UnfinishedRowsOutsideTheCopy([unknown, Checkpoint(MigrationCategoryState.Complete)], [EndpointSettings]); + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Complete), .. outside], NewSettings())); + + Assert.That(exception.Message, Does.Contain($"{UnknownCategoryId} is ").And.Contain(state.ToString()), + "an id this build does not know is required until it is known to be optional, as the ingestion gate already decides"); + } + + [Test] + public void An_unfinished_unknown_row_makes_the_copy_not_settled_and_the_refusal_names_it() + { + var unknown = Checkpoint(MigrationCategoryState.InProgress, UnknownCategoryId); + var checkpoints = new[] { Checkpoint(MigrationCategoryState.Complete), unknown }; + var outside = MigrationStartup.UnfinishedRowsOutsideTheCopy(checkpoints, [EndpointSettings]); + + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Complete)], outside, NewSettings())); + + Assert.Multiple(() => + { + Assert.That(MigrationStartup.RequiredCopyIsSettled(checkpoints, [EndpointSettings]), Is.False); + Assert.That(exception.Message, Does.Contain($"{UnknownCategoryId} is InProgress")); + }); + } + + [Test] + public void A_finished_unknown_row_leaves_the_copy_settled_and_the_refusal_silent() + { + var checkpoints = new[] { Checkpoint(MigrationCategoryState.Complete), Checkpoint(MigrationCategoryState.Abandoned, UnknownCategoryId) }; + var outside = MigrationStartup.UnfinishedRowsOutsideTheCopy(checkpoints, [EndpointSettings]); + + Assert.Multiple(() => + { + Assert.That(MigrationStartup.RequiredCopyIsSettled(checkpoints, [EndpointSettings]), Is.True); + Assert.DoesNotThrow(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Complete)], outside, NewSettings())); + }); + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.Abandoned)] + public void A_finished_row_under_an_id_this_build_does_not_know_holds_nothing(MigrationCategoryState state) => + Assert.That(MigrationStartup.UnfinishedRowsOutsideTheCopy([Checkpoint(state, UnknownCategoryId)], [EndpointSettings]), Is.Empty); + + [Test] + public void An_unfinished_row_known_to_be_optional_holds_nothing() => + Assert.That(MigrationStartup.UnfinishedRowsOutsideTheCopy([Checkpoint(MigrationCategoryState.InProgress, MigrationCategoryIds.EventLog)], [EndpointSettings]), Is.Empty); + + [Test] + public void A_category_that_reported_no_checkpoint_at_all_keeps_the_host_closed() + { + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [], NewSettings())); + + Assert.That(exception.Message, Does.Contain(MigrationCategoryIds.EndpointSettings), + "an empty result reads as success unless the gate checks what it asked for against what came back"); + } + + [Test] + public void Of_two_attempted_categories_the_one_that_reported_nothing_is_the_one_named() + { + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete( + [KnownEndpoints, EndpointSettings], + [Checkpoint(MigrationCategoryState.Complete, MigrationCategoryIds.KnownEndpoints)], + NewSettings())); + + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.EndpointSettings} reported no checkpoint at all").And.Not.Contain(MigrationCategoryIds.KnownEndpoints), + "comparing counts instead of ids would let a two-category run through with one category's fate unknown"); + } + + [Test] + public void The_refusal_reads_to_its_end_as_a_sentence_and_says_how_to_get_back() + { + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [Checkpoint(MigrationCategoryState.Halted)], NewSettings())); + + Assert.Multiple(() => + { + Assert.That(exception.Message, Does.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}").And.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}"), + "a Failed category waits for the operator, so a refusal naming neither command leaves them restarting for ever"); + Assert.That(exception.Message, Does.Not.Contain("the copy resumes from its last committed batch"), "a restart no longer copies a Failed category"); + Assert.That(exception.Message, Does.Contain("both with ServiceControl stopped. Nothing has opened on SQLServer yet") + .And.EndWith("returns the instance to RavenDB with no loss."), + "the operator reads the whole message, and the rollback instructions are at the end of it"); + }); + } + + [Test] + public void Every_failed_required_category_is_named_with_its_skips_and_both_commands() + { + var withErrors = Checkpoint(MigrationCategoryState.CompleteWithErrors, MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages) with + { + CopiedCount = 40, + SkippedCount = 5, + SkipReasons = new Dictionary + { + [MigrationSkipReason.RequiredValueMissing] = 2, + [MigrationSkipReason.BodyUnreadable] = 3 + } + }; + var halted = Checkpoint(MigrationCategoryState.Halted, MigrationCategoryIds.MessageRedirects) with { LastError = "TimeoutException at cursor r-7: the target did not answer" }; + + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete( + [MessageRedirects, UnresolvedAndRetryIssuedFailedMessages], [halted, withErrors], NewSettings())); + + using (Assert.EnterMultipleScope()) + { + foreach (var categoryId in new[] { MigrationCategoryIds.MessageRedirects, MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages }) + { + Assert.That(exception.Message, Does.Contain($"--migration-retry {categoryId} once the cause is fixed").And.Contain($"--migration-abandon {categoryId} to keep what was copied"), + $"one start gives every required category its go, so the refusal has to name every Failed one, {categoryId} included"); + } + + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.MessageRedirects} is Failed (Halted)").And.Contain("the target did not answer")); + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages} is Failed (CompleteWithErrors) after copying 40 and skipping 5")); + Assert.That(exception.Message, Does.Contain("RequiredValueMissing 2, no retry can fix"), "a row no retry can store is a reason to abandon, and the operator has to be told which"); + Assert.That(exception.Message, Does.Contain("BodyUnreadable 3").And.Not.Contain("BodyUnreadable 3, no retry can fix"), "an unreadable body may read on the next try"); + } + } + + [Test] + public void A_category_waiting_behind_a_failed_one_is_named_as_waiting() + { + var halted = Checkpoint(MigrationCategoryState.Halted, MigrationCategoryIds.KnownEndpoints); + var blocked = Checkpoint(MigrationCategoryState.Blocked) with { LastError = "Blocked: EndpointSettings must follow KnownEndpoints, which is Halted" }; + + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([KnownEndpoints, EndpointSettings], [halted, blocked], NewSettings())); + + using (Assert.EnterMultipleScope()) + { + Assert.That(exception.Message, Does.Contain($"{MigrationCategoryIds.EndpointSettings} is Blocked").And.Contain("must follow KnownEndpoints")); + Assert.That(exception.Message, Does.Contain($"--migration-retry {MigrationCategoryIds.KnownEndpoints}")); + Assert.That(exception.Message, Does.Not.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings}").And.Not.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}"), + "a category that never got its go has nothing to retry; it runs once the one ahead of it is dealt with"); + } + } + + [Test] + public void A_stalled_category_is_refused_with_the_limit_its_row_records() + { + var stalled = Checkpoint(MigrationCategoryState.Halted) with { LastError = MigrationStartup.StallExplanation(MigrationCategoryIds.EndpointSettings) }; + + var exception = Assert.Throws(() => MigrationStartup.RefuseIfAnyCategoryDidNotComplete([EndpointSettings], [stalled], NewSettings())); + + Assert.That(exception.Message, Does.Contain($"committed nothing for {MigrationStartup.ClosedWindowProgress.StallLimit.TotalMinutes:0.#} minutes") + .And.Contain("looking for a setting to change; run --migration-retry EndpointSettings"), + "the stall is on the row, and the refusal is where the operator reads it"); + } + + // The last thing said before the cutover is one way, and the only place a skipped total appears at all. + [Test] + public void A_failed_category_that_skipped_rows_is_told_a_retry_re_reads_them() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 6, new Dictionary + { + [MigrationSkipReason.PastRetention] = 3, + [MigrationSkipReason.RequiredValueMissing] = 2, + [MigrationSkipReason.BodyUnreadable] = 1 + })], + logger, + RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Level, Is.EqualTo(LogLevel.Warning), "rows left behind are not an informational matter"); + Assert.That(entry.Message, Does.Contain("6 skipped"), "the total is the number the operator decides on"); + Assert.That(entry.Message, Does.Contain("PastRetention 3").And.Contain("RequiredValueMissing 2").And.Contain("BodyUnreadable 1"), "a total with no reasons cannot be acted on"); + Assert.That(entry.Message, Does.Contain("stay only in the source database").And.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings} re-reads them"), + "without this the operator either waits for a copy that is already over or never learns the rows can still come across"); + Assert.That(entry.Message, Does.Contain("No retry can fix the RequiredValueMissing ones"), "the retry brings the unreadable bodies across but leaves these behind again"); + }); + } + + [Test] + public void A_failed_category_whose_faults_no_retry_can_fix_is_pointed_at_abandon() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.CompleteWithErrors, copied: 7, skipped: 5, new Dictionary + { + [MigrationSkipReason.PastRetention] = 3, + [MigrationSkipReason.RequiredValueMissing] = 2 + })], + logger, + RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("no retry can fix them").And.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}"), + "a retry re-reads the whole category and ends Failed on the same rows"); + Assert.That(entry.Message, Does.Not.Contain("--migration-retry"), "harmless skips are not a reason to retry either"); + }); + } + + [Test] + public void A_category_that_failed_with_only_harmless_skips_is_still_offered_a_retry() + { + var logger = new CapturingLogger(); + + var stoppedByAnException = Settled(MigrationCategoryState.Halted, copied: 7, skipped: 3, new Dictionary { [MigrationSkipReason.PastRetention] = 3 }) + with + { LastError = "TimeoutException at cursor r-7: the target did not answer" }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([stoppedByAnException], logger, RunStartedAt); + + Assert.That(logger.Entries.Single().Message, Does.Contain($"--migration-retry {MigrationCategoryIds.EndpointSettings} re-reads them").And.Not.Contain("--migration-abandon"), + "no fault skip stands in the way of a retry, so the exception's cause is what the operator fixes"); + } + + [Test] + public void A_done_category_with_harmless_skips_is_not_offered_a_retry() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind( + [Settled(MigrationCategoryState.Complete, copied: 7, skipped: 3, new Dictionary { [MigrationSkipReason.PastRetention] = 3 })], + logger, + RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("3 skipped").And.Contain("PastRetention 3")); + Assert.That(entry.Message, Does.Contain("harmless").And.Contain("stay only in the source database")); + Assert.That(entry.Message, Does.Not.Contain("--migration-retry"), "a Done category cannot be retried, so offering it sends the operator to a command that refuses"); + }); + } + + [Test] + public void A_blocked_category_is_left_to_the_refusal_rather_than_reported_as_a_clean_copy() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Blocked, copied: 0, skipped: 0, null)], logger, RunStartedAt); + + Assert.That(logger.Entries, Is.Empty); + } + + // Every refused start reports it again, directly above the refusal that calls it Failed. + [Test] + public void A_failed_category_that_skipped_nothing_is_not_reported_as_a_clean_copy() + { + var logger = new CapturingLogger(); + + var stalled = Settled(MigrationCategoryState.Halted, copied: 10, skipped: 0, null) with { LastError = MigrationStartup.StallExplanation(MigrationCategoryIds.EndpointSettings) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([stalled], logger, RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Level, Is.EqualTo(LogLevel.Warning)); + Assert.That(entry.Message, Does.Contain($"{MigrationCategoryIds.EndpointSettings} is Failed (Halted) after 10 copied").And.Not.Contain("nothing skipped")); + }); + } + + [Test] + public void A_category_that_skipped_nothing_says_so_without_raising_a_warning() + { + var logger = new CapturingLogger(); + + MigrationStartup.ReportWhatTheCopyLeftBehind([Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null)], logger, RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Level, Is.EqualTo(LogLevel.Information), "a clean copy warning about nothing trains the operator to ignore the warning that matters"); + Assert.That(entry.Message, Does.Contain("nothing skipped")); + }); + } + + // Restarting a migrated instance used to reprint the whole copy summary in the present tense, so an operator + // restarting to change a setting read "7 copied" and had no way to tell the copy had not run again. + [Test] + public void A_category_an_earlier_run_finished_is_not_reported_as_copied_again() + { + var logger = new CapturingLogger(); + + var alreadyDone = Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null) + with + { SettledAt = RunStartedAt.AddMinutes(-5) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([alreadyDone], logger, RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Contain("already finished before this start"), "without this the line is indistinguishable from a copy that just ran"); + Assert.That(entry.Message, Does.Contain("This start copied nothing"), "the operator needs to know nothing was written to a target that is already serving"); + Assert.That(entry.Message, Does.Contain("7"), "the historical total is still worth stating, just not as this run's work"); + Assert.That(entry.Level, Is.EqualTo(LogLevel.Information)); + }); + } + + // A category this run actually settled must keep the ordinary wording, or the fix above would silence every report. + [Test] + public void A_category_this_run_finished_is_still_reported_as_copied() + { + var logger = new CapturingLogger(); + + var justDone = Settled(MigrationCategoryState.Complete, copied: 7, skipped: 0, null) + with + { SettledAt = RunStartedAt.AddSeconds(2) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([justDone], logger, RunStartedAt); + + Assert.That(logger.Entries.Single().Message, Does.Contain("7 copied").And.Not.Contain("already finished")); + } + + // A Failed row comes back untouched on every start, so every refused restart reports it again. + [Test] + public void A_category_an_earlier_run_left_failed_is_not_reported_as_finished() + { + var logger = new CapturingLogger(); + + var failedEarlier = Settled(MigrationCategoryState.Halted, copied: 7, skipped: 2, new Dictionary { [MigrationSkipReason.RequiredValueMissing] = 2 }) + with + { SettledAt = RunStartedAt.AddMinutes(-5) }; + + MigrationStartup.ReportWhatTheCopyLeftBehind([failedEarlier], logger, RunStartedAt); + + var entry = logger.Entries.Single(); + + Assert.Multiple(() => + { + Assert.That(entry.Message, Does.Not.Contain("already finished"), "the refusal right after this line calls the same category Failed"); + Assert.That(entry.Message, Does.Contain($"--migration-abandon {MigrationCategoryIds.EndpointSettings}").And.Not.Contain("--migration-retry"), + "the advice has to show on every refused start, not only the first, and no retry can store a row missing a required value"); + }); + } + + static readonly DateTime RunStartedAt = new(2026, 9, 20, 12, 0, 0, DateTimeKind.Utc); + + static MigrationCheckpoint Settled(MigrationCategoryState state, long copied, long skipped, IReadOnlyDictionary? skipReasons) => + new(MigrationCategoryIds.EndpointSettings, state, null, copied, skipped, null, skipReasons, null, null, null, null); + + static readonly MigrationCategory EndpointSettings = MigrationCategoryRegistry.Find(MigrationCategoryIds.EndpointSettings)!; + static readonly MigrationCategory KnownEndpoints = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + static readonly MigrationCategory MessageRedirects = MigrationCategoryRegistry.Find(MigrationCategoryIds.MessageRedirects)!; + static readonly MigrationCategory UnresolvedAndRetryIssuedFailedMessages = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + + const string UnknownCategoryId = "SomeCategoryFromANewerBuild"; + + static MigrationCheckpoint Checkpoint(MigrationCategoryState state, string categoryId = MigrationCategoryIds.EndpointSettings) => + new(categoryId, state, null, 0, 0, null, null, null, null, null, null); + + static Settings NewSettings() => + new(transportType: "LearningTransport", persisterType: "SQLServer", errorRetentionPeriod: TimeSpan.FromDays(10)); +} diff --git a/src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs b/src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs new file mode 100644 index 0000000000..f3aa068120 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/RetryHistoryDepthIsSafeCheckTests.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using NUnit.Framework; +using ServiceControl.Migration.Checks; + +[TestFixture] +class RetryHistoryDepthIsSafeCheckTests +{ + [Test] + public void Passes_at_the_default_depth() => + Assert.DoesNotThrowAsync(() => new RetryHistoryDepthIsSafeCheck(10).Run()); + + [TestCase(0)] + [TestCase(-1)] + public void Refuses_a_depth_that_empties_the_table(int depth) + { + var exception = Assert.ThrowsAsync(() => new RetryHistoryDepthIsSafeCheck(depth).Run()); + + Assert.That(exception.Message, Does.Contain("RetryHistoryDepth").And.Contain("HistoricRetryOperations")); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs new file mode 100644 index 0000000000..2f3b033642 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/StoppedCopyExplanationTests.cs @@ -0,0 +1,194 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +// The copy can stop three ways that are not the engine's to report: the stall watchdog stopping one category, +// the host shutting down, and a second instance writing to the same database. +[TestFixture] +class StoppedCopyExplanationTests +{ + static readonly TimeSpan PollInterval = MigrationStartup.ClosedWindowProgress.PollInterval; + static readonly TimeSpan StallLimit = MigrationStartup.ClosedWindowProgress.StallLimit; + + [Test] + public async Task A_stall_settles_the_stalled_category_failed_and_the_next_category_still_runs() + { + var clock = new TimerRecordingTimeProvider(); + var store = new CancellationHonouringCheckpointStore(); + var writing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var target = new InMemoryMigrationTarget(store) { HangOnCall = 1, BeforeWrite = _ => writing.TrySetResult() }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1")); + // MessageRedirects does not follow KnownEndpoints, so a Failed KnownEndpoints cannot hold it back. + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + + var run = Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger())); + + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(10)), Is.True, "the watchdog never started its timer, so advancing the clock would tick nothing"); + await writing.Task.WaitAsync(TimeSpan.FromSeconds(10)); + clock.Advance(StallLimit + PollInterval); + var results = await run.WaitAsync(TimeSpan.FromSeconds(10)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(checkpoint => checkpoint.CategoryId), Is.EqualTo(new[] { MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects })); + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted), "a stalled category is Failed, so it waits for the operator rather than for the next start"); + Assert.That(results[0].SettledAt, Is.EqualTo(clock.GetUtcNow().UtcDateTime)); + Assert.That(results[0].LastError, Does.Contain($"committed nothing for {StallLimit.TotalMinutes:0.#} minutes").And.Not.Contain("--migration-"), + "the row says what happened, and the refusal is what names the commands"); + Assert.That(await store.Read(MigrationCategoryIds.KnownEndpoints), Is.EqualTo(results[0]), "the stall was reported but never saved, so the next start would copy it again"); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete), "one stalled category ended the whole start"); + Assert.That(target.WrittenRows(MigrationCategoryIds.MessageRedirects).Select(row => row.SourceId), Is.EqualTo(new[] { "MessageRedirects-1" })); + } + } + + [Test] + public async Task A_host_stopping_mid_category_leaves_it_copying_and_the_stop_propagates() + { + var store = new InMemoryMigrationCheckpointStore(); + using var host = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(store) { DefaultBatchSize = 2, StopOnCall = (2, host) }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, Row("KnownEndpoints-1"), Row("KnownEndpoints-2"), Row("KnownEndpoints-3"), Row("KnownEndpoints-4")); + source.Seed(MigrationCategoryIds.MessageRedirects, Row("MessageRedirects-1")); + var clock = new FakeTimeProvider(); + + var run = MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.MessageRedirects), store, clock, new CapturingLogger(), host.Token); + + using (Assert.EnterMultipleScope()) + { + Assert.That(async () => await run, Throws.InstanceOf(), "a shutdown is not the category's fault, so it must not come back as a settled row"); + + var stopped = await store.Read(MigrationCategoryIds.KnownEndpoints); + + Assert.That(stopped!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown settled Halted makes every restart refuse until the operator runs --migration-retry"); + Assert.That(stopped.Cursor, Is.EqualTo("KnownEndpoints-2"), "the next start resumes from the first batch's cursor"); + Assert.That(stopped.CopiedCount, Is.EqualTo(2)); + Assert.That(stopped.SettledAt, Is.Null); + Assert.That(stopped.LastError, Is.Null); + Assert.That((await store.Read(MigrationCategoryIds.MessageRedirects))!.State, Is.EqualTo(MigrationCategoryState.NotStarted), "a stopping host started the next category"); + Assert.That(target.RowsHandedToWrite(MigrationCategoryIds.MessageRedirects), Is.Empty); + } + } + + [Test] + public async Task A_stale_in_progress_row_the_start_has_not_reached_is_not_judged_a_stall() + { + var clock = new TimerRecordingTimeProvider(); + var store = new PollObservingCheckpointStore(clock); + var twoHoursAgo = clock.GetUtcNow().UtcDateTime - TimeSpan.FromHours(2); + await store.Upsert(new MigrationCheckpoint(MigrationCategoryIds.EndpointSettings, MigrationCategoryState.InProgress, null, 0, 0, null, null, twoHoursAgo, twoHoursAgo, null, null)); + var target = new InMemoryMigrationTarget(store) + { + DefaultBatchSize = 1, + // Each KnownEndpoints batch takes five minutes, and the next one waits until the watchdog has looked at the time. + BeforeWrite = checkpoint => + { + if (checkpoint.CategoryId == MigrationCategoryIds.KnownEndpoints) + { + clock.Advance(TimeSpan.FromMinutes(5)); + store.WaitForAPollAtTheCurrentTime().GetAwaiter().GetResult(); + } + } + }; + var source = new InMemoryMigrationSource(); + source.Seed(MigrationCategoryIds.KnownEndpoints, [.. Enumerable.Range(1, 8).Select(index => Row($"KnownEndpoints-{index}"))]); + source.Seed(MigrationCategoryIds.EndpointSettings, Row("EndpointSettings-1")); + + var results = await Task.Run(() => MigrationStartup.RunRequiredCategories( + Engine(source, target, store, clock), Categories(MigrationCategoryIds.KnownEndpoints, MigrationCategoryIds.EndpointSettings), store, clock, new CapturingLogger())) + .WaitAsync(TimeSpan.FromSeconds(30)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(checkpoint => checkpoint.State), Is.All.EqualTo(MigrationCategoryState.Complete), + "forty minutes copying KnownEndpoints were judged against a row the start had not reached yet"); + Assert.That(target.WrittenRows(MigrationCategoryIds.KnownEndpoints), Has.Count.EqualTo(8)); + Assert.That(target.WrittenRows(MigrationCategoryIds.EndpointSettings).Select(row => row.SourceId), Is.EqualTo(new[] { "EndpointSettings-1" })); + } + } + + [Test] + public void A_cancelled_copy_stays_a_cancellation() => + Assert.That(async () => await MigrationStartup.CopyOrExplainWhyItStopped(Task.FromCanceled>(new CancellationToken(canceled: true)), NewSettings()), + Throws.InstanceOf(), + "a shutdown is not a failure, so it must not come out as a refusal telling the operator to go looking at the source and the target"); + + [Test] + public void A_checkpoint_saved_by_another_instance_says_which_instance_to_stop() + { + var conflict = new MigrationCheckpointConflictException("Checkpoint KnownEndpoints was saved from version 3, but the stored row is at version 4."); + + var exception = Assert.ThrowsAsync(async () => await MigrationStartup.CopyOrExplainWhyItStopped( + Task.FromException>(conflict), + NewSettings())); + + Assert.Multiple(() => + { + Assert.That(exception!.Message, Does.Contain("a second ServiceControl pointed at the same SQLServer database"), + "the store's own message names a version, which tells nobody what is actually wrong"); + Assert.That(exception.Message, Does.Contain($"Stop the other instance, then restart with {MigrationSettings.EnabledKey} still on"), + "this is the only refusal in the subsystem where doing nothing makes it worse"); + Assert.That(exception.Message, Does.Contain(conflict.Message).And.Contain("Nothing has opened on SQLServer yet")); + Assert.That(exception.InnerException, Is.SameAs(conflict)); + }); + } + + [Test] + public async Task A_copy_that_finished_is_returned_as_it_came_back() + { + IReadOnlyList copied = [new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.Complete, null, 3, 0, null, null, null, null, null, null)]; + + var finished = await MigrationStartup.CopyOrExplainWhyItStopped(Task.FromResult(copied), NewSettings()); + + Assert.That(finished, Is.SameAs(copied)); + } + + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationCategory[] Categories(params string[] categoryIds) => [.. categoryIds.Select(categoryId => MigrationCategoryRegistry.Find(categoryId)!)]; + + static MigrationEngine Engine(InMemoryMigrationSource source, InMemoryMigrationTarget target, IMigrationCheckpointStore store, TimeProvider clock) => + new(source, target, store, clock, new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + static Settings NewSettings() => + new(transportType: "LearningTransport", persisterType: "SQLServer", errorRetentionPeriod: TimeSpan.FromDays(10)); + + // Refuses a cancelled token as the EF Core store does, so a settle made on the stalled category's token fails here too. + sealed class CancellationHonouringCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore inner = new(); + + public Task> ReadAll(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return inner.ReadAll(cancellationToken); + } + + public Task Read(string categoryId, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return inner.Read(categoryId, cancellationToken); + } + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return inner.Upsert(checkpoint, cancellationToken); + } + } +} diff --git a/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs b/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs index 744fa7bf19..627fee7dee 100644 --- a/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs +++ b/src/ServiceControl.UnitTests/ScatterGather/IncompleteResultsTests.cs @@ -190,12 +190,10 @@ public async Task A_remote_timeout_does_not_cut_the_slower_remotes_short() factory.Register(settings.RemoteInstances[3], Delayed(delay, Healthy("msg-4"))); var api = new TestApi(settings, factory, Local("local-msg")); - var started = DateTime.UtcNow; var result = await api.Execute(Context(), "/api/messages"); - Assert.That(DateTime.UtcNow - started, Is.GreaterThanOrEqualTo(delay), "the composite must wait for the remotes that are still answering"); - Assert.That(result.Results.Select(m => m.MessageId), Is.EquivalentTo(["local-msg", "msg-2", "msg-3", "msg-4"])); + Assert.That(result.Results.Select(m => m.MessageId), Is.EquivalentTo(["local-msg", "msg-2", "msg-3", "msg-4"]), "the delay lives in the fake, so these three cannot arrive unless the composite waited for the remotes that are still answering"); Assert.That(result.IncompleteInstances, Is.EqualTo([new IncompleteInstance(settings.RemoteInstances[0].InstanceId, QueryFailure.TimedOut)])); Assert.That(result.QueryStats.TotalCount, Is.EqualTo(4)); } diff --git a/src/ServiceControl.slnx b/src/ServiceControl.slnx index 5abc0cecec..7ea43dbfe6 100644 --- a/src/ServiceControl.slnx +++ b/src/ServiceControl.slnx @@ -48,6 +48,7 @@ + diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index cc43874a16..03d0d421c8 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -17,6 +17,7 @@ using global::ServiceControl.Operations.Metrics; using global::ServiceControl.Recoverability.Retrying.Metrics; using global::ServiceControl.Persistence; + using global::ServiceControl.Persistence.DataMigration; using global::ServiceControl.Transports; using Licensing; using Microsoft.AspNetCore.HttpLogging; @@ -103,6 +104,11 @@ public static void AddServiceControl(this IHostApplicationBuilder hostBuilder, S services.AddSingleton(provider => new Lazy(provider.GetRequiredService)); services.AddPersistence(settings); + + // The checkpoint store is optional: EF Core registers one, RavenDB does not. + services.TryAddSingleton(provider => + new CheckpointMigrationState(provider.GetService())); + services.AddMetrics(settings.PrintMetrics); hostBuilder.AddTelemetry(settings); services.AddServiceControlHealthChecks(); diff --git a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs index 00d38bb0b3..1890ba5242 100644 --- a/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ErrorIngestionOnlyCommand.cs @@ -5,6 +5,7 @@ namespace ServiceControl.Hosting.Commands using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; + using Microsoft.Extensions.DependencyInjection; using NServiceBus; using Particular.ServiceControl; using Particular.ServiceControl.Hosting; @@ -13,8 +14,10 @@ namespace ServiceControl.Hosting.Commands using ServiceControl.ExternalIntegrations; using ServiceControl.Hosting.Https; using ServiceControl.Infrastructure.Health; + using ServiceControl.Migration; using ServiceControl.Monitoring; using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Recoverability; /// @@ -25,15 +28,13 @@ namespace ServiceControl.Hosting.Commands /// class ErrorIngestionOnlyCommand : AbstractCommand { - static readonly string[] SupportedStorageNames = ["SQLServer", "PostgreSQL"]; - public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) { EnsureStorageCanScaleOut(settings); var app = BuildHost(settings); - await app.RunAsync(settings.RootUrl); + await app.RunAsync(); } internal static WebApplication BuildHost(Settings settings, Action customize = null) @@ -47,12 +48,27 @@ internal static WebApplication BuildHost(Settings settings, Action + new FinishedCopyBeforeAnIngestionNodeOpens( + provider.GetRequiredService(), + "this error ingestion only host")); + + if (settings.MigrationEnabled) + { + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); + } + customize?.Invoke(hostBuilder); var app = hostBuilder.Build(); app.MapServiceControlHealthChecks(); + // Set here rather than passed to RunAsync, so a caller that starts the host itself gets the + // configured address instead of Kestrel's default port. + app.Urls.Clear(); + app.Urls.Add(settings.RootUrl); + return app; } @@ -60,7 +76,7 @@ static void EnsureStorageCanScaleOut(Settings settings) { var manifest = PersistenceManifestLibrary.Find(settings.PersistenceType); - if (manifest == null || !SupportedStorageNames.Contains(manifest.Name, StringComparer.OrdinalIgnoreCase)) + if (manifest == null || !PersistenceFactory.SqlPersistenceNames.Contains(manifest.Name, StringComparer.OrdinalIgnoreCase)) { throw new Exception( $"--error-ingestion-only requires SQL Server or PostgreSQL storage, but this instance is configured to use '{settings.PersistenceType}'. Scaling out error ingestion is not supported for this storage type."); diff --git a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs index af8a53840a..8e4937e504 100644 --- a/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs +++ b/src/ServiceControl/Hosting/Commands/ImportFailedErrorsCommand.cs @@ -14,6 +14,9 @@ using Recoverability; using ServiceBus.Management.Infrastructure.Settings; using ServiceControl.Infrastructure; + using ServiceControl.Migration; + using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; class ImportFailedErrorsCommand : AbstractCommand { @@ -54,6 +57,15 @@ internal static IHost BuildHost(Settings settings) var hostBuilder = Host.CreateApplicationBuilder(); hostBuilder.AddServiceControl(settings, endpointConfiguration, new RecoverabilityComponent()); + // Importing writes failed messages into the database, so on SQL it waits for an unfinished copy as an ingestion-only worker does. + if (PersistenceFactory.SqlPersistenceNames.Contains(PersistenceManifestLibrary.Find(settings.PersistenceType)?.Name, StringComparer.OrdinalIgnoreCase)) + { + hostBuilder.Services.AddHostedService(provider => + new FinishedCopyBeforeAnIngestionNodeOpens( + provider.GetRequiredService(), + "--import-failed-errors")); + } + return hostBuilder.Build(); } } diff --git a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs new file mode 100644 index 0000000000..40d832e124 --- /dev/null +++ b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs @@ -0,0 +1,59 @@ +namespace ServiceControl.Hosting.Commands +{ + using System; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Particular.ServiceControl.Hosting; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence; + + class MigrationSourceReportCommand : AbstractCommand + { + public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) + { + try + { + await Report(settings, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception e) + { + await Console.Error.WriteLineAsync(e.Message); + Environment.ExitCode = 1; + } + } + + static async Task Report(Settings settings, CancellationToken cancellationToken) + { + await using var source = await PersistenceFactory.OpenMigrationSource(settings, cancellationToken); + + var description = await source.Describe(cancellationToken); + + Console.Out.WriteLine("ServiceControl migration source report"); + Console.Out.WriteLine(); + Console.Out.WriteLine($"{"Source persistence",-20}: {PersistenceFactory.MigrationSourcePersistenceType}"); + Console.Out.WriteLine($"{"Version",-20}: {description.Version}"); + + foreach (var fact in description.Facts) + { + var origin = fact.SettingKey is null ? string.Empty : $" (from {fact.SettingKey})"; + Console.Out.WriteLine($"{fact.Label,-20}: {fact.Value}{origin}"); + } + + foreach (var scope in (await source.Inventory(cancellationToken)).GroupBy(entry => entry.Scope)) + { + Console.Out.WriteLine(); + Console.Out.WriteLine($"{scope.Key}:"); + + foreach (var entry in scope.OrderBy(entry => entry.Name, StringComparer.Ordinal)) + { + Console.Out.WriteLine($" {entry.Name,-42}{entry.Count,12:N0}"); + } + } + } + } +} diff --git a/src/ServiceControl/Hosting/Commands/RunCommand.cs b/src/ServiceControl/Hosting/Commands/RunCommand.cs index 33a6361511..436431f6f8 100644 --- a/src/ServiceControl/Hosting/Commands/RunCommand.cs +++ b/src/ServiceControl/Hosting/Commands/RunCommand.cs @@ -1,9 +1,12 @@ namespace ServiceControl.Hosting.Commands { + using System; using System.Threading; using System.Threading.Tasks; using Infrastructure.WebApi; using Microsoft.AspNetCore.Builder; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; using NServiceBus; using Particular.ServiceControl; using Particular.ServiceControl.Hosting; @@ -11,11 +14,19 @@ using ServiceControl; using ServiceControl.Hosting.Auth; using ServiceControl.Hosting.Https; + using ServiceControl.Migration; using ServicePulse; class RunCommand : AbstractCommand { - public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) + public override Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) => + Run(settings, customize: null, cancellationToken); + + /// + /// Builds and runs the full instance. When the migration is turned on, the required copy is added as the + /// first thing the host starts, so the instance opens only once the copy has finished. + /// + internal static async Task Run(Settings settings, Action customize, CancellationToken cancellationToken = default) { var endpointConfiguration = new EndpointConfiguration(settings.InstanceName); var assemblyScanner = endpointConfiguration.AssemblyScanner(); @@ -31,7 +42,17 @@ public override async Task Execute(HostArguments args, Settings settings, Cancel hostBuilder.AddServiceControl(settings, endpointConfiguration); hostBuilder.AddServiceControlApi(settings.CorsSettings); - var app = hostBuilder.Build(); + customize?.Invoke(hostBuilder); + + if (settings.MigrationEnabled) + { + hostBuilder.Services.AddHostedService(provider => new RequiredCopyBeforeTheHostOpens(provider, settings)); + hostBuilder.Services.AddHostedService(provider => new RecordHostOpenedOnTarget(provider)); + } + + // A start that refuses would otherwise leave everything the host built undisposed. + await using var app = hostBuilder.Build(); + app.UseServiceControl(settings.ForwardedHeadersSettings, settings.HttpsSettings); if (settings.EnableIntegratedServicePulse) { @@ -39,7 +60,27 @@ public override async Task Execute(HostArguments args, Settings settings, Cancel } app.UseServiceControlAuthentication(settings.OpenIdConnectSettings.Enabled); - await app.RunAsync(settings.RootUrl); + // WebApplication's RunAsync(url) takes no cancellation token, so set the url here and call IHost's RunAsync below. + app.Urls.Clear(); + app.Urls.Add(settings.RootUrl); + + // Read before RunAsync, which disposes the host before the filter below runs, so reading it there would + // throw and the catch would never fire. + var lifetime = app.Lifetime; + + // Starting the host is what marks the target as opened, through the RecordHostOpenedOnTarget hosted + // service registered above, so nothing here does it. + try + { + await app.RunAsync(cancellationToken); + } + // Stopping the service during the required copy cancels it inside host start, and that is a stop + // rather than a failure: every committed batch is durable and the next start resumes from the cursor. +#pragma warning disable PS0020 // The host cancels on its own lifetime token, not the caller's, so that is the one to filter on + catch (OperationCanceledException) when (settings.MigrationEnabled && lifetime.ApplicationStopping.IsCancellationRequested) +#pragma warning restore PS0020 + { + } } } } diff --git a/src/ServiceControl/Hosting/HostArguments.cs b/src/ServiceControl/Hosting/HostArguments.cs index 88fe15a162..bd6400df32 100644 --- a/src/ServiceControl/Hosting/HostArguments.cs +++ b/src/ServiceControl/Hosting/HostArguments.cs @@ -62,6 +62,15 @@ public HostArguments(string[] args) } }; + var migrationSourceReportOptions = new OptionSet + { + { + "migration-source-report", + "Report what a migration would read from the source persistence", + s => Command = typeof(MigrationSourceReportCommand) + } + }; + try { // Parsed before setup returns, so setup can refuse to provision an ingestion-only worker. @@ -94,6 +103,13 @@ public HostArguments(string[] args) return; } + migrationSourceReportOptions.Parse(args); + + if (Command == typeof(MigrationSourceReportCommand)) + { + return; + } + defaultOptions.Parse(args); } catch (Exception e) diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index af52f4bf88..35f2c65d75 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -16,6 +16,7 @@ using ServiceControl.Infrastructure.Settings; using ServiceControl.Infrastructure.WebApi; using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Transports; using ServicePulse; using JsonSerializer = System.Text.Json.JsonSerializer; @@ -185,6 +186,7 @@ public string InstanceId public string TransportType { get; set; } public string PersistenceType { get; private set; } + public bool MigrationEnabled => SettingsReader.Read(SettingsRootNamespace, MigrationSettings.EnabledKey, MigrationSettings.DefaultEnabled); public string ErrorLogQueue { get; set; } public string ErrorQueue { get; set; } diff --git a/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs new file mode 100644 index 0000000000..68ab286716 --- /dev/null +++ b/src/ServiceControl/Migration/Checks/MigrationIsReleasedCheck.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +/// +/// Registered by a test host that needs a copy to run on a build that does not yet carry the whole migration. +/// +class AllowUnreleasedMigration; + +/// +/// Refuses every copy until the whole migration has shipped. A build that can copy the required categories but +/// not yet the background copy, the end-of-migration guard or verification would still commit an instance to the +/// target with no way to finish or check the move. The last phase of the migration deletes this check. +/// +class MigrationIsReleasedCheck(AllowUnreleasedMigration allowUnreleasedMigration = null) : IMigrationStartupCheck +{ + public string Name => "this build carries the whole migration"; + + public Task Run(CancellationToken cancellationToken = default) + { + if (allowUnreleasedMigration is null) + { + throw new Exception( + $"This build of ServiceControl does not yet carry the whole migration from RavenDB, so it will not copy anything. Set {MigrationSettings.EnabledKey} back to false and start ServiceControl again."); + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs b/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs new file mode 100644 index 0000000000..e3ffed392a --- /dev/null +++ b/src/ServiceControl/Migration/Checks/MigrationPairIsSupportedCheck.cs @@ -0,0 +1,31 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses any target but SQL Server or PostgreSQL, since the source is always RavenDB. The seams underneath are +/// general enough to copy between any two persisters, and this is the check that says which pair is actually supported. +/// +class MigrationPairIsSupportedCheck(Settings settings) : IMigrationStartupCheck +{ + public string Name => "the source and target are the supported pair"; + + public Task Run(CancellationToken cancellationToken = default) + { + var targetName = PersistenceManifestLibrary.Find(settings.PersistenceType)?.Name; + + if (!PersistenceFactory.SqlPersistenceNames.Contains(targetName, StringComparer.OrdinalIgnoreCase)) + { + throw new Exception( + $"Migrating from '{PersistenceFactory.MigrationSourcePersistenceType}' to '{settings.PersistenceType}' is not supported. The only supported migration is from RavenDB to SQL Server or PostgreSQL, so set {Settings.SettingsRootNamespace}/PersistenceType to {string.Join(" or ", PersistenceFactory.SqlPersistenceNames)} before setting {MigrationSettings.EnabledKey}."); + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs b/src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs new file mode 100644 index 0000000000..d8f2ef87bc --- /dev/null +++ b/src/ServiceControl/Migration/Checks/OptionalCategoryWindowsAreValidCheck.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Persistence.DataMigration; + +/// +/// Reads the engine options out of the settings, which refuses an optional category window that is not a time +/// span before anything is copied rather than part way through the copy. +/// +class OptionalCategoryWindowsAreValidCheck(TimeSpan eventRetentionPeriod, TimeSpan errorRetentionPeriod) : IMigrationStartupCheck +{ + public string Name => "the optional category windows are valid"; + + /// + /// The options parsed from the settings, which is null until has returned. + /// + public MigrationEngineOptions Options { get; private set; } + + public Task Run(CancellationToken cancellationToken = default) + { + Options = MigrationEngineOptions.FromSettings(Settings.SettingsRootNamespace, eventRetentionPeriod, errorRetentionPeriod); + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs b/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs new file mode 100644 index 0000000000..00f5903bd4 --- /dev/null +++ b/src/ServiceControl/Migration/Checks/RetryHistoryDepthIsSafeCheck.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Migration.Checks; + +using System; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses a copy that the instance's own retry history setting would throw away. At a depth of zero the SQL +/// persisters empty the history table whenever a retry completes, and the migrated rows would go with it. +/// +/// The instance's ServiceControl/RetryHistoryDepth. +class RetryHistoryDepthIsSafeCheck(int retryHistoryDepth) : IMigrationStartupCheck +{ + public string Name => "the retry history depth will not empty a migrated table"; + + public Task Run(CancellationToken cancellationToken = default) + { + // The depth at which RetryHistoryDataStore.TrimHistory deletes every row rather than trimming. + if (retryHistoryDepth <= 0) + { + throw new Exception( + $"ServiceControl/RetryHistoryDepth is {retryHistoryDepth}, and at that depth the SQL persisters delete every row of HistoricRetryOperations when a retry completes. The RetryOperations category copies that table, so the first completed retry after cutover would silently discard the migrated retry history. Set ServiceControl/RetryHistoryDepth to a positive number, or clear it to use the default of 10, before setting {MigrationSettings.EnabledKey}."); + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs b/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs new file mode 100644 index 0000000000..6433658d4b --- /dev/null +++ b/src/ServiceControl/Migration/FinishedCopyBeforeAnIngestionNodeOpens.cs @@ -0,0 +1,58 @@ +namespace ServiceControl.Migration; + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using ServiceControl.Persistence.DataMigration; + +/// +/// Refuses to start a host that writes failed messages into the database, an error ingestion only host or +/// --import-failed-errors, until every required category is Done or Abandoned. An optional category, which copies +/// in the background once the main host has opened, never holds it back, but a category under an id this build +/// does not know does. A refusal throws, which fails the start and stops the host. Start such a host only after the +/// main host has opened, because a database with no checkpoint row yet reads as one no migration has touched. +/// +/// Where the checkpoint rows are read. A database no migration has touched holds none, so the host starts as before. +/// Names the host in the refusal, such as "this error ingestion only host". +sealed class FinishedCopyBeforeAnIngestionNodeOpens( + IMigrationCheckpointStore checkpointStore, + string hostDescription) : IHostedLifecycleService +{ + // It runs with the migration off too, because a node someone forgot to flag would ingest into a part-copied + // database and turn abandoning it from a clean rollback into permanent loss. A database that never migrated has + // nothing unfinished, so its nodes start as before. + public async Task StartingAsync(CancellationToken cancellationToken = default) + { + var unfinished = (await checkpointStore.ReadAll(cancellationToken)) + .Where(checkpoint => MigrationCategoryRegistry.Find(checkpoint.CategoryId)?.Kind != MigrationCategoryKind.Optional) + .Where(checkpoint => !checkpoint.State.IsFinished()) + .ToArray(); + + if (unfinished.Length == 0) + { + return; + } + + var detail = string.Join(". ", unfinished.Select(checkpoint => checkpoint.State.IsFailed() + ? MigrationStartup.FailedDetail(checkpoint) + : $"{checkpoint.CategoryId} is Copying ({checkpoint.State})")); + + var advice = unfinished.Any(checkpoint => !checkpoint.State.IsFailed()) + ? "Let the instance running the copy finish it and start this host again." + : "Start this host again once they are settled."; + + throw new Exception($"A copy into this database has not finished, so {hostDescription} will not start and nothing has been lost. {detail}. {advice}"); + } + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Migration/MigrationStartup.cs b/src/ServiceControl/Migration/MigrationStartup.cs new file mode 100644 index 0000000000..dc17bab1d1 --- /dev/null +++ b/src/ServiceControl/Migration/MigrationStartup.cs @@ -0,0 +1,607 @@ +namespace ServiceControl.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Migration.Checks; +using ServiceControl.Persistence; +using ServiceControl.Persistence.DataMigration; + +/// +/// The required copy, from the checks that decide whether it can run at all to the refusal it throws when a +/// category did not finish. Everything here happens with ServiceControl closed, which is the only window in +/// which the copy can be thrown away at no cost. +/// +static class MigrationStartup +{ + // A category needs both a reader and a writer, so a half-implemented one is never attempted. + internal static IReadOnlyCollection CopyableCategoryIds(IReadOnlyCollection sourceSupports, IReadOnlyCollection targetSupports) => + [.. sourceSupports.Intersect(targetSupports, StringComparer.Ordinal)]; + + /// + /// Runs the startup checks, copies every required category this build can copy that is not already settled, + /// and then seeds the the host reads. Once every required category this build + /// copies is settled and no row under an unknown id is unfinished, a source that will not open is logged and + /// recorded on the optional categories' rows instead of refused, nothing is copied, and the host still opens. Throws when a check refuses or a category does not + /// finish, with a message telling the operator what to do and how to go back; the caller must let that stop + /// the host. Call it after the host is built and before it starts, because the copy has to finish before + /// anything else opens on the target. + /// + /// The built host's services, which is where the target, the checkpoint store and the migration state come from. + /// The instance settings, read for the source and target persistence types. + /// Cancelled when the host is shutting down, which ends the copy without a refusal message. + public static async Task RunRequiredCopy(IServiceProvider services, Settings settings, CancellationToken cancellationToken = default) + { + await MigrationStartupCheckRunner.Run( + [ + new MigrationIsReleasedCheck(services.GetService()), + new MigrationPairIsSupportedCheck(settings) + ], cancellationToken); + + var loggerFactory = services.GetRequiredService(); + var logger = loggerFactory.CreateLogger(typeof(MigrationStartup)); + var target = services.GetRequiredService(); + var checkpointStore = services.GetRequiredService(); + var timeProvider = services.GetRequiredService(); + + await using var source = PersistenceFactory.CreateMigrationSource(settings); + + var copyable = CopyableCategoryIds(source.SupportedCategoryIds, target.SupportedCategoryIds); + + var options = await RunChecksAndOpenTarget(services, settings, cancellationToken); + + var engine = new MigrationEngine( + source, + target, + checkpointStore, + timeProvider, + options, + loggerFactory.CreateLogger()); + + // Optional categories copy beside the running services, so only the required ones hold those services back. + var holdingBack = engine.SelectCategories(MigrationCategoryKind.Required) + .Select(category => category.Id) + .ToArray(); + + var toCopy = engine.SelectCategories(MigrationCategoryKind.Required) + .Where(category => copyable.Contains(category.Id)) + .ToArray(); + + var deferred = engine.SelectCategories(MigrationCategoryKind.Required) + .Where(category => !copyable.Contains(category.Id)) + .Select(category => category.Id) + .ToArray(); + + var checkpoints = await checkpointStore.ReadAll(cancellationToken); + var unfinishedOutsideTheCopy = UnfinishedRowsOutsideTheCopy(checkpoints, toCopy); + var requiredCopySettled = RequiredCopyIsSettled(checkpoints, toCopy); + + try + { + await MigrationStartupCheckRunner.Run([SourceOpens(source)], cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Safe to open without the source, because every required category is settled and only the background copy of the optional ones reads it. + catch (Exception exception) when (requiredCopySettled) + { + // The runner's wrapper says ServiceControl will not start, which is false here and would land on the rows. + var cause = exception.InnerException ?? exception; + + logger.LogError(cause, + "The {SourcePersistenceType} migration source could not be opened, so ServiceControl opens without it: every required category this build copies is Done or Abandoned and no row under an id it does not know is unfinished, so only the background copy of the optional categories reads the source. The optional categories still copying stay as they are until a start that can open the source, and their checkpoints carry this error.", + PersistenceFactory.MigrationSourcePersistenceType); + + await RecordSourceOutage(checkpointStore, engine.SelectCategories(MigrationCategoryKind.Optional), cause, cancellationToken); + await SeedMigrationState(services, holdingBack, cancellationToken); + return; + } + catch (Exception exception) + { + throw new Exception($"{exception.Message} {NoWayOutWithoutTheSource(settings)}", exception.InnerException); + } + + await MigrationStartupCheckRunner.Run(source.ContributedChecks(), cancellationToken); + + logger.LogInformation( + "Migration mode: copying {CopyCount} required categories before ServiceControl opens ({DeferredCount} not yet implemented: {Deferred})", + toCopy.Length, deferred.Length, string.Join(", ", deferred)); + + // Captured before the copy so the report can tell a category this run finished from one an earlier run did. + var runStartedAt = timeProvider.GetUtcNow().UtcDateTime; + + var finished = await CopyOrExplainWhyItStopped( + RunRequiredCategories(engine, toCopy, checkpointStore, timeProvider, logger, cancellationToken), + settings); + + ReportWhatTheCopyLeftBehind(finished, logger, runStartedAt); + + RefuseIfAnyCategoryDidNotComplete(toCopy, finished, unfinishedOutsideTheCopy, settings); + + await SeedMigrationState(services, holdingBack, cancellationToken); + } + + static async Task SeedMigrationState(IServiceProvider services, IReadOnlyCollection holdingBack, CancellationToken cancellationToken) + { + if (services.GetRequiredService() is CheckpointMigrationState state) + { + await state.Seed(holdingBack, cancellationToken); + } + } + + /// + /// Runs every startup check in the order they have to run, and opens the target and the source as two of + /// them. The order is what the operator sees: a check that costs nothing comes before one that connects to a + /// database, and the source's own checks run last because they need it open. + /// + /// The instance settings, whose retention periods are the optional category windows when none is set, and whose retry history depth the copy must not be thrown away by. + /// The options read from the settings, which the window check parsed on its way past. + /// A check refused. The message names the check and says what to do, and this start has copied nothing. + public static async Task RunChecksAndOpen(IServiceProvider services, Settings settings, IMigrationSource source, CancellationToken cancellationToken = default) + { + var options = await RunChecksAndOpenTarget(services, settings, cancellationToken); + + await MigrationStartupCheckRunner.Run([SourceOpens(source)], cancellationToken); + + await MigrationStartupCheckRunner.Run(source.ContributedChecks(), cancellationToken); + + return options; + } + + /// + /// Runs every startup check that does not need the source, in the order they have to run, ending with + /// opening the target. A check that costs nothing comes before one that connects to a database. The target + /// is open when this returns, and the source has not been touched. + /// + /// The built host's services, which is where the target and its readiness checks come from. + /// The instance settings, whose retention periods are the optional category windows when none is set, and whose retry history depth the copy must not be thrown away by. + /// Cancelled when the host is shutting down, which stops the checks without a refusal message. + /// The options read from the settings, which the window check parsed on its way past. + /// A check refused or the target did not open. The message names the check and says what to do, and this start has copied nothing. + /// The host is shutting down. + public static async Task RunChecksAndOpenTarget(IServiceProvider services, Settings settings, CancellationToken cancellationToken = default) + { + var target = services.GetRequiredService(); + var readiness = services.GetRequiredService(); + var categories = new OptionalCategoryWindowsAreValidCheck(settings.EventsRetentionPeriod, settings.ErrorRetentionPeriod); + + await MigrationStartupCheckRunner.Run( + [ + categories, + new RetryHistoryDepthIsSafeCheck(settings.RetryHistoryDepth), + .. readiness.ContributedChecks(), + new Step("the migration target opens", target.Open) + ], cancellationToken); + + return categories.Options; + } + + /// + /// Records on each optional category's checkpoint that the source could not be opened, so status and verify + /// read the outage from the rows. The required copy calls it on a start that + /// opens without the source, and the background copier calls it when its own open fails. It changes no + /// state, cursor or count: a row still copying gains the error as its , + /// a category with no row gets a new not-started row carrying the error, and a row that is Done, Failed or + /// Abandoned is left alone. The next start that resumes a row clears the error. + /// + /// Where the rows are read and saved. + /// The optional categories this instance selected. + /// Why the source could not be opened. Its type and message go into the error. + /// Cancelled when the host is shutting down. + /// Another writer saved one of these rows since it was read. + internal static async Task RecordSourceOutage(IMigrationCheckpointStore checkpointStore, IReadOnlyList optionalCategories, Exception exception, CancellationToken cancellationToken = default) + { + var error = $"The {PersistenceFactory.MigrationSourcePersistenceType} migration source could not be opened: {exception.GetType().Name}: {exception.Message.TrimEnd('.', ' ')}. This category stays as it is, and the next start tries the source again."; + + foreach (var category in optionalCategories) + { + var checkpoint = await checkpointStore.Read(category.Id, cancellationToken); + + if (checkpoint is null) + { + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, error), cancellationToken); + } + else if (!checkpoint.State.IsFinished() && !checkpoint.State.IsFailed()) + { + await checkpointStore.Upsert(checkpoint with { LastError = error }, cancellationToken); + } + } + } + + /// + /// Copies the required categories one after another under the stall watchdog and returns where each one ended, + /// in the same order. A category that commits nothing for is + /// stopped and settled Halted, and the categories after it still get their go. Before copying anything it saves + /// a not-started checkpoint for every category that has none, as does. + /// + /// Copies each category. + /// The categories to copy, in the order to copy them. + /// Where each category's row is seeded, watched and, after a stall, settled. + /// The clock the watchdog measures a stall on. + /// Receives the watchdog's progress lines and the stall. + /// The host's token. Cancelling it stops the copy and leaves the running category as its last committed batch left it, so the next start resumes it. + /// The checkpoint each category ended on. + /// Another writer saved one of these checkpoints, which means a second instance is copying into the same database. + /// The host is shutting down. + internal static async Task> RunRequiredCategories( + MigrationEngine engine, + IReadOnlyList categories, + IMigrationCheckpointStore checkpointStore, + TimeProvider timeProvider, + ILogger logger, + CancellationToken cancellationToken = default) + { + // The gates that keep a host off an unfinished copy read only the rows that exist. + foreach (var category in categories) + { + if (await checkpointStore.Read(category.Id, cancellationToken) is null) + { + await checkpointStore.Upsert(new MigrationCheckpoint(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null), cancellationToken); + } + } + + await using var progress = new ClosedWindowProgress(checkpointStore, timeProvider, logger, [.. categories.Select(category => category.Id)], cancellationToken); + + var results = new List(categories.Count); + + foreach (var category in categories) + { + using var categoryCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + progress.Watch(category.Id, timeProvider.GetUtcNow().UtcDateTime, categoryCancellation); + + try + { + results.Add(await engine.RunCategoryAsync(category, categoryCancellation.Token)); + } + // Only the watchdog cancels this source without the host's token, so the filter tells a stall from a shutdown. +#pragma warning disable PS0020 + catch (OperationCanceledException) when (categoryCancellation.IsCancellationRequested && !cancellationToken.IsCancellationRequested) +#pragma warning restore PS0020 + { + // The host's token, because the stall has already cancelled the category's and a store call on that would fail. + var stalled = await checkpointStore.Read(category.Id, cancellationToken); + + results.Add(await checkpointStore.Upsert(stalled with + { + State = MigrationCategoryState.Halted, + SettledAt = timeProvider.GetUtcNow().UtcDateTime, + LastError = StallExplanation(category.Id) + }, cancellationToken)); + } + } + + return results; + } + + /// + /// Waits for the copy and turns a second instance writing checkpoints to the same database into a refusal the + /// operator can act on. A shutdown and every other failure come out as they are. + /// + /// The copy, already running. + /// Read for the persistence type the refusal names. + /// What the copy returned. + /// A checkpoint saved by another instance. The message says what to do and how to go back. + /// The host is shutting down. +#pragma warning disable PS0018 // The copy it waits on already runs under the host's token, so a token here would have nothing to cancel. + internal static async Task> CopyOrExplainWhyItStopped( + Task> copy, + Settings settings) +#pragma warning restore PS0018 + { + try + { + return await copy; + } + // The engine never turns a conflict into a halt, so without this the copy ends on the checkpoint store's + // own message and none of the advice every other refusal carries. + catch (MigrationCheckpointConflictException exception) + { + throw new Exception( + $"The required copy stopped because another writer saved a migration checkpoint for this instance, which is a second ServiceControl pointed at the same {settings.PersistenceType} database. ServiceControl will not start. {exception.Message} Stop the other instance, then restart with {MigrationSettings.EnabledKey} still on; the copy resumes from its last committed batch. " + + RollbackAdvice(settings), exception); + } + } + + /// + /// Throws unless every attempted category finished. A category that reported no checkpoint at all counts as + /// outstanding too, because nothing says how much of it was copied. + /// + /// A category did not finish. The message names each one with its state and counts, names the commands that move each Failed one on, and says how to go back. + internal static void RefuseIfAnyCategoryDidNotComplete( + IReadOnlyList attempted, + IReadOnlyList finished, + Settings settings) + { + var outstanding = finished + .Where(checkpoint => !checkpoint.State.IsFinished()) + .Select(checkpoint => checkpoint.State.IsFailed() ? FailedDetail(checkpoint) : CopyingDetail(checkpoint)) + .ToList(); + + var reported = finished.Select(checkpoint => checkpoint.CategoryId).ToHashSet(StringComparer.Ordinal); + + outstanding.AddRange(attempted + .Where(category => !reported.Contains(category.Id)) + .Select(category => $"{category.Id} reported no checkpoint at all, so whether it copied anything is unknown")); + + if (outstanding.Count == 0) + { + return; + } + + throw new Exception( + $"The required copy did not finish, so ServiceControl will not start and nothing has been lost. {string.Join(". ", outstanding)}. " + + RollbackAdvice(settings)); + } + + /// + /// Says whether the host may open without the source: every category this start copies is finished, and no row + /// outside the copy is unfinished. + /// + /// Every checkpoint row in the target. + /// The categories this start copies. + /// True when nothing required is still outstanding. + internal static bool RequiredCopyIsSettled(IReadOnlyList checkpoints, IReadOnlyList toCopy) => + UnfinishedRowsOutsideTheCopy(checkpoints, toCopy).Count == 0 + && toCopy.All(category => checkpoints.Any(checkpoint => checkpoint.CategoryId == category.Id && checkpoint.State.IsFinished())); + + /// + /// Throws unless every attempted category finished and no row outside the copy is unfinished, naming each + /// outstanding row in the same words. + /// + /// The categories this start copied. + /// Where each attempted category ended. + /// The rows found. + /// Read for the persistence type the refusal names. + /// Something is outstanding. The message names it and says how to go back. + internal static void RefuseIfAnyCategoryDidNotComplete( + IReadOnlyList attempted, + IReadOnlyList finished, + IReadOnlyList unfinishedOutsideTheCopy, + Settings settings) => + RefuseIfAnyCategoryDidNotComplete(attempted, [.. finished, .. unfinishedOutsideTheCopy], settings); + + /// + /// Finds the unfinished rows the copy will not run: a row under an id this build does not know, which counts as + /// required because a newer build may have written it for a category that must finish, and a required row for a + /// category this build cannot copy yet. A finished row, and a row known to be optional, hold nothing. + /// + /// Every checkpoint row in the target. + /// The categories this start copies, whose rows the copy judges itself. + /// The rows that keep the host closed, in the order they were read. + internal static IReadOnlyList UnfinishedRowsOutsideTheCopy(IReadOnlyList checkpoints, IReadOnlyList toCopy) => + [.. checkpoints + .Where(checkpoint => MigrationCategoryRegistry.Find(checkpoint.CategoryId)?.Kind != MigrationCategoryKind.Optional) + .Where(checkpoint => toCopy.All(category => category.Id != checkpoint.CategoryId)) + .Where(checkpoint => !checkpoint.State.IsFinished())]; + + internal static string FailedDetail(MigrationCheckpoint checkpoint) + { + var reasons = checkpoint.SkipReasons is { Count: > 0 } counts + ? $" ({string.Join("; ", counts.Select(reason => $"{reason.Key} {reason.Value}{(reason.Key.IsPermanent() ? ", no retry can fix" : "")}"))})" + : ""; + + return $"{checkpoint.CategoryId} is Failed ({checkpoint.State}) after copying {checkpoint.CopiedCount} and skipping {checkpoint.SkippedCount}{reasons}{LastErrorClause(checkpoint)}; " + + $"run --migration-retry {checkpoint.CategoryId} once the cause is fixed, which copies the category again from the start, or --migration-abandon {checkpoint.CategoryId} to keep what was copied and give up the rest, both with ServiceControl stopped"; + } + + static string CopyingDetail(MigrationCheckpoint checkpoint) => + $"{checkpoint.CategoryId} is {checkpoint.State} after copying {checkpoint.CopiedCount} and skipping {checkpoint.SkippedCount}{LastErrorClause(checkpoint)}"; + + // Trimmed because the refusal punctuates each category's sentence itself. + static string LastErrorClause(MigrationCheckpoint checkpoint) => + checkpoint.LastError is null ? "" : $": {checkpoint.LastError.TrimEnd('.', ' ')}"; + + /// + /// Logs one line per category saying what it copied and what it left behind. Skipped rows are warned about + /// one at a time while the copy runs, and nothing else states the total or says what becomes of them. + /// + /// When this start began, which is what tells a category this run finished from one an earlier run did. + internal static void ReportWhatTheCopyLeftBehind(IReadOnlyList finished, ILogger logger, DateTime runStartedAt) + { + foreach (var checkpoint in finished) + { + // It did not run, and the refusal that follows names it with the category it waits for. + if (checkpoint.State == MigrationCategoryState.Blocked) + { + continue; + } + + // A category already finished when this run began is returned without being run, so its counts are + // an earlier run's. A Failed one is returned untouched too, but it is not finished, so it falls through. + if (checkpoint.State.IsFinished() && checkpoint.SettledAt is { } settledAt && settledAt < runStartedAt) + { + logger.LogInformation( + "{CategoryId}: already finished before this start, by a run that copied {Copied} and skipped {Skipped}. This start copied nothing.", + checkpoint.CategoryId, checkpoint.CopiedCount, checkpoint.SkippedCount); + continue; + } + + // An exception, a stall or unbalanced counts fail a category without a skip, and that is not a clean copy. + if (checkpoint.State.IsFailed() && checkpoint.SkippedCount == 0) + { + logger.LogWarning("{CategoryId} is Failed ({State}) after {Copied} copied and {AlreadyPresent} already present. The refusal that follows says why and what to run.", + checkpoint.CategoryId, checkpoint.State, checkpoint.CopiedCount, checkpoint.AlreadyPresentCount); + continue; + } + + if (checkpoint.SkippedCount == 0) + { + logger.LogInformation("{CategoryId}: {Copied} copied, {AlreadyPresent} already present, nothing skipped", + checkpoint.CategoryId, checkpoint.CopiedCount, checkpoint.AlreadyPresentCount); + continue; + } + + var reasons = checkpoint.SkipReasons is { Count: > 0 } counts + ? string.Join(", ", counts.OrderByDescending(reason => reason.Value).Select(reason => $"{reason.Key} {reason.Value}")) + : "no reason recorded"; + + // Only a Failed category can be retried, so a Done one is never offered the command. + var whatBecomesOfThem = checkpoint.State.IsFailed() + ? WhatARetryCanDo(checkpoint) + : checkpoint.State == MigrationCategoryState.Complete + ? "They were left out as harmless, because ServiceControl would have removed them anyway, and they stay only in the source database." + : "They stay only in the source database."; + + logger.LogWarning( + "{CategoryId}: {Copied} copied, {AlreadyPresent} already present, {Skipped} skipped ({Reasons}). {WhatBecomesOfThem}", + checkpoint.CategoryId, checkpoint.CopiedCount, checkpoint.AlreadyPresentCount, checkpoint.SkippedCount, reasons, whatBecomesOfThem); + } + } + + // A retry re-reads the whole category, so it is offered only when some of the faults could come across on it. + static string WhatARetryCanDo(MigrationCheckpoint checkpoint) + { + var faults = checkpoint.SkipReasons?.Keys.Where(reason => !reason.IsBenign()).ToArray() ?? []; + var permanent = faults.Where(reason => reason.IsPermanent()).ToArray(); + + if (faults.Length > 0 && permanent.Length == faults.Length) + { + return $"They stay only in the source database, and no retry can fix them, so --migration-abandon {checkpoint.CategoryId} keeps what was copied and gives up the rest."; + } + + var cannotFix = permanent.Length == 0 ? "" : $" No retry can fix the {string.Join(" or ", permanent)} ones."; + + return $"They stay only in the source database until --migration-retry {checkpoint.CategoryId} re-reads them once the cause is fixed.{cannotFix}"; + } + + internal static string StallExplanation(string stalledCategoryId) => + $"Halted: {stalledCategoryId} committed nothing for {ClosedWindowProgress.StallLimit.TotalMinutes:0.#} minutes, so it was stopped. The limit is not configurable: check that the source and the target are both responding rather than looking for a setting to change."; + + static string RollbackAdvice(Settings settings) => + $"Nothing has opened on {settings.PersistenceType} yet, so setting {MigrationSettings.EnabledKey}=false and pointing PersistenceType back at {PersistenceFactory.MigrationSourcePersistenceType} discards the partial copy and returns the instance to {PersistenceFactory.MigrationSourcePersistenceType} with no loss."; + + static string NoWayOutWithoutTheSource(Settings settings) => + $"If the {PersistenceFactory.MigrationSourcePersistenceType} source is already gone, a required category that is Failed or has started copying can be given up with --migration-abandon , with ServiceControl stopped. " + + $"A required category that never started cannot be abandoned: while one is outstanding ServiceControl has never opened on {settings.PersistenceType}, so pointing PersistenceType back at {PersistenceFactory.MigrationSourcePersistenceType}, or starting over against an empty {settings.PersistenceType} database, loses nothing it has served."; + + static Step SourceOpens(IMigrationSource source) => new("the migration source opens", source.Open); + + /// + /// Makes opening the target or the source look like a startup check, so a failure to connect is reported in + /// the same words as a check that refused, and in its place in the order. + /// + sealed class Step(string name, Func run) : IMigrationStartupCheck + { + public string Name => name; + + public Task Run(CancellationToken cancellationToken = default) => run(cancellationToken); + } + + /// + /// Logs how far each category has got while the copy runs, and stops the running category when it commits + /// nothing for . Without it a copy that is waiting on a database nobody is watching + /// holds the instance closed for as long as the operator leaves it. + /// + internal sealed class ClosedWindowProgress : IAsyncDisposable + { + // Not configurable, and not a total timeout: a deadline would kill a copy that is working. + internal static readonly TimeSpan PollInterval = TimeSpan.FromSeconds(30); + internal static readonly TimeSpan StallLimit = TimeSpan.FromMinutes(30); + + readonly CancellationTokenSource cancellation; + readonly HashSet attempted; + readonly Task polling; + volatile RunningCategory running; + + public ClosedWindowProgress(IMigrationCheckpointStore checkpointStore, TimeProvider timeProvider, ILogger logger, IReadOnlyCollection attemptedCategoryIds, CancellationToken cancellationToken = default) + { + cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attempted = attemptedCategoryIds.ToHashSet(StringComparer.Ordinal); + polling = Poll(checkpointStore, timeProvider, logger, cancellation.Token); + } + + /// + /// Points the watchdog at the category that is starting now, in place of the one before it. Only this + /// category is judged from here on, from the later of its last committed batch and . + /// + /// The category starting now. + /// When this start began running it. A row an earlier start left carries an older stamp, which would read as a stall at once. + /// Cancelled when the category stalls. The category must run under its token, and nothing else should. + public void Watch(string categoryId, DateTime runStartedAt, CancellationTokenSource stop) => + running = new RunningCategory(categoryId, runStartedAt, stop); + + async Task Poll(IMigrationCheckpointStore checkpointStore, TimeProvider timeProvider, ILogger logger, CancellationToken cancellationToken) + { + using var timer = new PeriodicTimer(PollInterval, timeProvider); + + try + { + while (await timer.WaitForNextTickAsync(cancellationToken)) + { + try + { + // Only this run's categories, because a row an earlier run left in progress is not this run's to report. + var inProgress = (await checkpointStore.ReadAll(cancellationToken)) + .Where(checkpoint => checkpoint.State == MigrationCategoryState.InProgress && attempted.Contains(checkpoint.CategoryId)) + .ToArray(); + + foreach (var checkpoint in inProgress) + { + logger.LogInformation( + "{CategoryId}: {Copied} of {Total} copied, {Skipped} skipped, cursor {Cursor}", + checkpoint.CategoryId, + checkpoint.CopiedCount, + checkpoint.SourceTotal is { } total ? total.ToString() : "an unknown number of", + checkpoint.SkippedCount, + checkpoint.Cursor ?? "the start"); + } + + var watched = running; + + // A resumed row carries the previous run's stamp, so the window starts at whichever is + // later: that stamp, or the moment this category's run began. + var stalled = watched is not null && !watched.Stop.IsCancellationRequested && inProgress.Any(checkpoint => + checkpoint.CategoryId == watched.CategoryId + && timeProvider.GetUtcNow().UtcDateTime + - (checkpoint.LastProgressAt is { } lastProgress && lastProgress > watched.RunStartedAt ? lastProgress : watched.RunStartedAt) > StallLimit); + + if (stalled) + { + logger.LogError( + "{CategoryId} has committed nothing for {StallLimit}, so it is being stopped and settled Halted. The categories after it still get their go.", + watched.CategoryId, StallLimit); + await watched.Stop.CancelAsync(); + } + } + // The copy finished or the host is stopping: the outer catch ends the poll. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Log, don't throw: an error here would come out of DisposeAsync and hide what the copy itself failed with. + catch (Exception exception) + { + logger.LogError(exception, "The stall watchdog's poll failed, so a stalled copy will not be noticed until a later poll succeeds. The copy itself is unaffected and is still running."); + } + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Either the copy finished and disposal cancelled the poll, or the host is shutting down. + } + } + + public async ValueTask DisposeAsync() + { + await cancellation.CancelAsync(); + + try + { + await polling; + } + finally + { + cancellation.Dispose(); + } + } + + sealed record RunningCategory(string CategoryId, DateTime RunStartedAt, CancellationTokenSource Stop); + } +} diff --git a/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs b/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs new file mode 100644 index 0000000000..4ccc0626a5 --- /dev/null +++ b/src/ServiceControl/Migration/MigrationStartupCheckRunner.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +/// +/// Runs startup checks in order and stops at the first one that refuses. +/// +static class MigrationStartupCheckRunner +{ + /// + /// Runs each check in turn, and wraps whatever a failing one throws in a message naming the check and + /// saying that this start has copied nothing, because every check runs before this start's copy. + /// + /// A check failed. The check's own message is kept, and its exception is the inner one. + public static async Task Run(IReadOnlyList checks, CancellationToken cancellationToken = default) + { + foreach (var check in checks) + { + try + { + await check.Run(cancellationToken); + } + // A shutdown is not a check failing, and saying it was would send the customer after the wrong thing. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception exception) + { + throw new Exception( + $"Migration startup check '{check.Name}' failed, so ServiceControl will not start, and this start has copied nothing. {exception.Message}", exception); + } + } + } +} diff --git a/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs b/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs new file mode 100644 index 0000000000..676eb714fa --- /dev/null +++ b/src/ServiceControl/Migration/RecordHostOpenedOnTarget.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.Migration; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using ServiceControl.Persistence.DataMigration; + +/// +/// Calls when a host started with the migration on opens +/// on a database a migration has already written to. +/// +sealed class RecordHostOpenedOnTarget(IServiceProvider services) : IHostedLifecycleService +{ + // StartedAsync, not StartAsync: the web server binds its port during StartAsync, and a start that dies there + // served nothing. The stores are resolved here rather than injected, because a RavenDB host registers neither + // and is refused before it gets this far. + public async Task StartedAsync(CancellationToken cancellationToken = default) + { + var checkpointStore = services.GetRequiredService(); + + // Only a database a migration has written to gets the stamp, so the marker answers whether a host + // has opened since the copy began rather than whether one ever ran on this database. + if ((await checkpointStore.ReadAll(cancellationToken)).Count > 0) + { + await services.GetRequiredService().RecordHostOpened(cancellationToken); + } + } + + public Task StartingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs b/src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs new file mode 100644 index 0000000000..9ca9388246 --- /dev/null +++ b/src/ServiceControl/Migration/RequiredCopyBeforeTheHostOpens.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Migration; + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using ServiceBus.Management.Infrastructure.Settings; + +/// +/// Runs the required copy as the host starts, before any hosted service of its own does. +/// A refusal throws, which fails the start and stops the host, as the copy requires. +/// +sealed class RequiredCopyBeforeTheHostOpens(IServiceProvider services, Settings settings) : IHostedLifecycleService +{ + // Inside the host rather than before it, because a Windows service reports itself started only once the host + // starts, and the Service Control Manager kills a process that has said nothing for 30 seconds. Every + // StartingAsync runs before any hosted service, so nothing has bound a port or begun ingesting. + public Task StartingAsync(CancellationToken cancellationToken = default) => + MigrationStartup.RunRequiredCopy(services, settings, cancellationToken); + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StartedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppingAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StopAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task StoppedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/src/ServiceControl/Persistence/PersistenceFactory.cs b/src/ServiceControl/Persistence/PersistenceFactory.cs index 892920cf18..5fd045058d 100644 --- a/src/ServiceControl/Persistence/PersistenceFactory.cs +++ b/src/ServiceControl/Persistence/PersistenceFactory.cs @@ -2,13 +2,23 @@ namespace ServiceControl.Persistence { using System; using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence.DataMigration; static class PersistenceFactory { + /// + /// The manifest names of the two persisters built on EF Core. Anything that is true of both of them and + /// of neither RavenDB nor a future persister is decided by this list. + /// + public static readonly string[] SqlPersistenceNames = ["SQLServer", "PostgreSQL"]; + public static IPersistence Create(Settings settings, bool maintenanceMode = false) { - var persistenceConfiguration = CreatePersistenceConfiguration(settings); + var persistenceConfiguration = CreatePersistenceConfiguration(settings.PersistenceType, settings); if (maintenanceMode && !persistenceConfiguration.SupportsMaintenanceMode) { @@ -24,11 +34,51 @@ public static IPersistence Create(Settings settings, bool maintenanceMode = fals return persistence; } - static IPersistenceConfiguration CreatePersistenceConfiguration(Settings settings) + /// + /// The persistence a migration copies from. RavenDB is the only source supported today. + /// + public const string MigrationSourcePersistenceType = "RavenDB"; + + public static IMigrationSource CreateMigrationSource(Settings settings) + { + var sourceFactory = (IMigrationSourceFactory)CreatePersistenceConfiguration(MigrationSourcePersistenceType, settings); + + return sourceFactory.CreateSource(Settings.SettingsRootNamespace); + } + + public static async Task OpenMigrationSource(Settings settings, CancellationToken cancellationToken = default) + { + var source = CreateMigrationSource(settings); + var opened = false; + + try + { + await source.Open(cancellationToken); + opened = true; + } + finally + { + if (!opened) + { + await source.DisposeAsync(); + } + } + + return source; + } + + static IPersistenceConfiguration CreatePersistenceConfiguration(string persistenceType, Settings settings) { + var persistenceManifest = PersistenceManifestLibrary.Find(persistenceType) + ?? throw new Exception($"There is no persistence named '{persistenceType}'. Available: {string.Join(", ", PersistenceManifestLibrary.PersistenceManifests.Where(m => m.IsSupported).Select(m => m.Name))}."); + + if (persistenceManifest.TypeName is null) + { + throw new Exception($"The '{persistenceManifest.DisplayName}' persistence no longer ships an assembly and cannot be loaded."); + } + try { - var persistenceManifest = PersistenceManifestLibrary.Find(settings.PersistenceType); var assemblyPath = Path.Combine(persistenceManifest.Location, $"{persistenceManifest.AssemblyName}.dll"); var loadContext = settings.AssemblyLoadContextResolver(assemblyPath); var customizationType = Type.GetType(persistenceManifest.TypeName, loadContext.LoadFromAssemblyName, null, true); @@ -37,7 +87,7 @@ static IPersistenceConfiguration CreatePersistenceConfiguration(Settings setting } catch (Exception e) { - throw new Exception($"Could not load persistence customization type {settings.PersistenceType}.", e); + throw new Exception($"Could not load persistence customization type {persistenceType}.", e); } } } diff --git a/src/ServiceControl/Program.cs b/src/ServiceControl/Program.cs index 7234479cd3..96fb2359df 100644 --- a/src/ServiceControl/Program.cs +++ b/src/ServiceControl/Program.cs @@ -39,7 +39,7 @@ await new CommandRunner(arguments.Command).Execute(arguments, settings); - return 0; + return Environment.ExitCode; } catch (Exception ex) { @@ -52,7 +52,13 @@ LoggingConfigurator.ConfigureNLog("bootstrap.txt", "./", NLog.LogLevel.Fatal); NLog.LogManager.GetCurrentClassLogger().Fatal(ex, "Unrecoverable error"); } - throw; + + // The message goes to the console on its own, because the log above already holds the whole exception. + // Rethrowing instead would print the stack trace a second time and fire the unhandled-exception handler + // for a third, which buries a configuration mistake in what reads like a crash. + await Console.Error.WriteLineAsync(ex.Message); + + return 1; } finally { diff --git a/src/ServiceControl/ServiceControl.csproj b/src/ServiceControl/ServiceControl.csproj index 7afd8f88cf..d9b9958c43 100644 --- a/src/ServiceControl/ServiceControl.csproj +++ b/src/ServiceControl/ServiceControl.csproj @@ -68,6 +68,7 @@ +