feat(db): move every query into one database layer and support MySQL and MariaDB

.github/workflows/python-tests.yml

@@ -155,3 +155,96 @@ jobs:
         # one worker per vCPU is the right level where the runner has the headroom for it. The
         # matrix carries the count, because windows needs a lower one.
         run: uv run --no-sync pytest -n ${{ matrix.pytest-workers }} --max-worker-restart=0
+
+  # Decides whether the external-database job runs. That job's service containers start before any of its
+  # steps, so the decision has to come from a job of its own: otherwise every pull request, frontend-only ones
+  # included, would pull both server images and wait for them to become healthy.
+  external-database-changes:
+    name: 'external database: change detection'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    outputs:
+      run: ${{ steps.changed-files.outputs.python_any_changed == 'true' || inputs.always_run == true }}
+    steps:
+      - name: checkout
+        if: ${{ inputs.always_run != true }}
+        uses: nschloe/action-cached-lfs-checkout@385a8ecc719e50b8c71af6ab01a624b486b7c3bc
+
+      - name: check for changed python files
+        if: ${{ inputs.always_run != true }}
+        id: changed-files
+        # Pinned to the _hash_ for v45.0.9, see the matrix job above.
+        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96
+        with:
+          files_yaml: |
+            python:
+              - 'pyproject.toml'
+              - 'uv.lock'
+              - '.github/workflows/python-tests.yml'
+              - 'invokeai/**/*.py'
+              # The library's tests store the bundled workflows and style presets.
+              - 'invokeai/app/services/workflow_records/default_workflows/**'
+              - 'invokeai/app/services/style_preset_records/default_style_presets.json'
+              - 'tests/**'
+
+  # The database layer against MySQL and MariaDB, at the oldest release of each that InvokeAI supports.
+  # Only the tests on the database fixtures (marked `uses_database`) target the server; the matrix above
+  # runs the whole suite, those tests included, on SQLite.
+  external-database:
+    needs: external-database-changes
+    if: ${{ needs.external-database-changes.outputs.run == 'true' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - server: mysql
+            image: 'mysql:8.4.11'
+            health-cmd: 'mysqladmin ping -h 127.0.0.1 --silent'
+            url: 'mysql+pymysql://root:invokeai@127.0.0.1:3306'
+          - server: mariadb
+            image: 'mariadb:10.11.19'
+            health-cmd: 'mariadb-admin ping -h 127.0.0.1 --silent'
+            url: 'mariadb+pymysql://root:invokeai@127.0.0.1:3306'
+    name: 'py3.12: linux-${{ matrix.server }}'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    services:
+      database:
+        image: ${{ matrix.image }}
+        env:
+          MYSQL_ROOT_PASSWORD: invokeai
+          MARIADB_ROOT_PASSWORD: invokeai
+        ports:
+          - '3306:3306'
+        options: >-
+          --health-cmd="${{ matrix.health-cmd }}"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=30
+    env:
+      OMP_NUM_THREADS: '1'
+      MKL_NUM_THREADS: '1'
+      INVOKEAI_TEST_DB_URL: ${{ matrix.url }}
+
+    steps:
+      - name: checkout
+        uses: nschloe/action-cached-lfs-checkout@385a8ecc719e50b8c71af6ab01a624b486b7c3bc
+
+      - name: setup uv
+        uses: astral-sh/setup-uv@v8.1.0
+        with:
+          version: '0.11.28'  # When updating this, make sure the version in launcher's package.json gets updated too.
+          enable-cache: true
+          python-version: '3.12'
+
+      - name: install dependencies
+        env:
+          UV_INDEX: 'https://download.pytorch.org/whl/cpu'
+        run: uv sync --no-progress --locked --extra test --extra mysql
+
+      - name: run pytest
+        run: uv run --no-sync pytest -n logical --max-worker-restart=0 -m "uses_database and not sqlite_only and not slow"
+
+      - name: server log
+        if: ${{ failure() }}
+        run: docker logs "${{ job.services.database.id }}"

docker/Dockerfile

@@ -84,7 +84,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     # this is just to get the package manager to recognize that the project exists, without making changes to the docker layer
     --mount=type=bind,source=invokeai/version,target=invokeai/version \
     ulimit -n 30000 && \
-    uv sync --extra $GPU_DRIVER --frozen
+    uv sync --extra $GPU_DRIVER --extra mysql --frozen
 
 # Link amdgpu.ids for ROCm builds
 # contributed by https://github.com/Rubonnek
@@ -114,5 +114,5 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
     --mount=type=bind,source=uv.lock,target=uv.lock \
     ulimit -n 30000 && \
-    uv pip install -e .[$GPU_DRIVER]
+    uv pip install -e .[$GPU_DRIVER,mysql]
 

docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx

@@ -235,16 +235,18 @@ Both actions handle partial failures: if some models succeed and others fail, th
 
 ## Finding orphaned models
 
-If a model file is deleted or moved outside the Model Manager, its
-database entry sticks around. To find these orphaned entries:
+Model files can stay in the models directory after their database entry is gone, for example when the
+database was reset, or files were copied into the directory by hand. To find these orphaned models:
 
 1. Open the Model Manager.
 2. Click the "..." menu at the top right of the installed model list, and pick **"Clean up orphaned models..."**
-3. If there are orphaned models, a dialogue box appears that lists them and offers to remove their database entries.
+3. If there are orphaned models, a dialogue box appears that lists their folders and offers to delete them.
 
-Orphaned models are automatically excluded from selection dropdowns
-(main model, LoRA, VAE, etc.), so you cannot accidentally pick one for
-generation.
+On an install without the web UI at hand, an administrator can do the same through the API:
+`GET /api/v2/models/sync/orphaned` lists the folders and `DELETE /api/v2/models/sync/orphaned` deletes those given.
+
+The opposite case, a database entry whose files are missing from disk, is marked in the Model Manager's
+model list, and its filter can show only those models.
 
 ## Exporting and Importing Model Settings
 

docs/src/content/docs/configuration/01.invokeai-yaml.mdx

@@ -249,6 +249,8 @@ If you serve InvokeAI through nginx, Caddy, or similar, set `http_compression_le
 
 #### Database Durability
 
+This section concerns the default SQLite database. A [MySQL or MariaDB database](/configuration/external-database/), set with `db_url`, has its durability set on the server.
+
 Every write to InvokeAI's SQLite database is committed to disk before the request returns. How hard that commit is pushed to the physical disk is controlled by `db_synchronous`:
 
 ```yaml
@@ -268,7 +270,7 @@ Running the database on a network share is not a supported configuration -- SQLi
 
 The safeguard exists anyway, because WAL needs shared memory and a few setups cannot provide it: a database on a network share, some container volume drivers, and SQLite builds compiled without WAL all leave a rollback journal in place instead — and on a rollback journal, `normal` _can_ corrupt the database on power loss. InvokeAI reads back the journal mode it actually got, so in that case it ignores `normal`, uses `full`, and logs a warning naming the mode. An in-memory database (`use_memory_db`) has no durability to trade and is unaffected.
 
-`db_synchronous` applies to the application's own database connection. Gallery maintenance is an administrator action in Settings and uses that same application database connection, so its database work follows this setting. The image import script, `invokeai/frontend/install/import_images.py`, still opens the database independently and leaves SQLite's own default in place; the measurements below describe the running app, not that script.
+`db_synchronous` applies to the application's own database connection, which gallery maintenance (an administrator action in Settings) uses too, and to the image import script, `invokeai/frontend/install/import_images.py`, which reads the same `invokeai.yaml`. The measurements below describe the running app.
 
 Measured on a copy of a real library, 300 single-row inserts each committed on its own:
 

docs/src/content/docs/configuration/02.docker.mdx

@@ -60,6 +60,8 @@ Any environment variables supported by InvokeAI can be set here - please see the
 At the very least, you might want to set the `INVOKEAI_ROOT` environment variable
 to point to the location where you wish to store your InvokeAI models, configuration, and outputs.
 
+The image includes the driver for a [MySQL or MariaDB database](/configuration/external-database/): set `INVOKEAI_DB_URL` to use one instead of the SQLite file in the root directory.
+
 | Environment Variable | Default value | Description |
 | --- | --- | --- |
 | `INVOKEAI_ROOT` | `~/invokeai` | **Required** - the location of your InvokeAI root directory. It will be created if it does not exist. |

docs/src/content/docs/configuration/03.external-database.mdx

@@ -0,0 +1,110 @@
+---
+title: MySQL and MariaDB
+sidebar:
+  order: 3
+---
+
+InvokeAI keeps its boards, image records, workflows, projects, queue and accounts in a database. By default that is a SQLite file in `db_dir`, which needs no setup. On a server that hosts InvokeAI for several people, the database can be a MySQL or MariaDB database instead, run and backed up alongside the server's other databases.
+
+## Requirements
+
+- MySQL 8.4 or newer, or MariaDB 10.11 or newer, with the InnoDB storage engine.
+- `max_allowed_packet` of at least 64 MiB on the server. MariaDB's default is 16 MiB; InvokeAI refuses to start below 64 MiB, because a large project is stored in one statement. Set it in the server's configuration and restart the server:
+
+  ```ini
+  [mysqld]
+  max_allowed_packet=64M
+  ```
+
+- The `mysql` extra of the `invokeai` package, which installs the PyMySQL driver: `pip install "invokeai[mysql]"`, or `uv sync --extra mysql` in a source checkout. The Docker image includes it.
+
+## Creating the database
+
+Create an empty database and an account for InvokeAI. InvokeAI creates its tables, each with the character set and collations it needs, when it first starts.
+
+MySQL:
+
+```sql
+CREATE DATABASE invokeai CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin;
+CREATE USER 'invokeai'@'%' IDENTIFIED BY 'a-strong-password';
+GRANT ALL PRIVILEGES ON invokeai.* TO 'invokeai'@'%';
+```
+
+MariaDB:
+
+```sql
+CREATE DATABASE invokeai CHARACTER SET utf8mb4 COLLATE utf8mb4_nopad_bin;
+CREATE USER 'invokeai'@'%' IDENTIFIED BY 'a-strong-password';
+GRANT ALL PRIVILEGES ON invokeai.* TO 'invokeai'@'%';
+```
+
+## Configuring InvokeAI
+
+Set `db_url` in `invokeai.yaml`, or the `INVOKEAI_DB_URL` environment variable:
+
+```yaml
+db_url: mariadb+pymysql://invokeai:a-strong-password@db.example:3306/invokeai
+```
+
+Use `mysql+pymysql://` for MySQL and `mariadb+pymysql://` for MariaDB. Characters in the password that have a meaning in a URL (`@`, `:`, `/`, `%`, `#`) must be percent-encoded, e.g. `@` as `%40`. InvokeAI reads `db_url` when it starts, and shows it without the password in its settings. `db_dir` and `db_synchronous` apply to a SQLite database only.
+
+On its first start InvokeAI creates the tables. On later starts it updates them to its version, as it does a SQLite database. It makes no backup of a server database before an update: back the database up yourself, e.g. with `mysqldump` or `mariadb-dump`, before upgrading InvokeAI.
+
+## Moving an existing install
+
+`invoke-db-copy` copies an install's SQLite database into a new, empty server database. It copies the database only: images, models and the other files stay where they are, in the root directory. The SQLite database is not changed.
+
+1. Stop InvokeAI. What it writes during the copy is not copied.
+2. Set `db_url` in `invokeai.yaml` (or `INVOKEAI_DB_URL`) to the new database. Do not start InvokeAI yet: it would start the new database empty.
+3. Check the copy, which copies nothing yet:
+
+   ```bash
+   invoke-db-copy --check
+   ```
+
+4. Copy:
+
+   ```bash
+   invoke-db-copy
+   ```
+
+5. Start InvokeAI.
+
+`invoke-db-copy` takes the install it runs in, or the one `--root` names, and copies into the database its `db_url` names. `--target URL` names another one; a password on the command line is visible to the other users of the computer and stays in the shell's history.
+
+It copies from a snapshot of the SQLite database, which it makes in the database's directory and deletes afterwards: that needs free space as large as the database. It brings the snapshot to no newer version: if InvokeAI has not started since an upgrade, start it once first. The target must be empty, for the check too, and no InvokeAI process may use it.
+
+It stops before copying anything when the server would refuse a row:
+
+- **Rows whose foreign keys name a row that is gone**, which a SQLite database can hold and a server refuses. `--orphans skip` copies them without the reference where the database would clear it anyway (a board whose cover image is gone keeps no cover), and leaves out the others, together with what the database deletes with them. The report lists what it left out, per table.
+- **Values longer than the server's column holds**, and rows larger than half of `max_allowed_packet`.
+
+Two kinds of value are copied in the form a server stores:
+
+- **Vocabulary terms of the image index that differ only in the case of a letter outside A-Z** (`Äpfel` and `äpfel`), which the server compares as equal: it keeps one of them.
+- **A model's file size stored with a fraction**, which is written as a whole number of bytes.
+
+After copying, `invoke-db-copy` compares every table of the copy with its source, row by row, and only then writes the records that make it a database InvokeAI takes. If the copy fails or does not match, InvokeAI refuses the target: drop the database, create it again empty, and run `invoke-db-copy` again.
+
+## Going back to SQLite
+
+`invoke-db-copy --to-sqlite FILE` copies the server database back into a new SQLite file. Removing `db_url` alone is not a way back: InvokeAI then opens the SQLite database it used before the move, as it was then. The boards, image records, workflows and accounts made since are not in it, and the gallery no longer shows the images made since, although their files are in the outputs folder.
+
+1. Stop InvokeAI. `invoke-db-copy` refuses to copy a database an InvokeAI process uses.
+2. Copy into a new file:
+
+   ```bash
+   invoke-db-copy --to-sqlite invokeai-from-server.db
+   ```
+
+3. In the install's database directory (`db_dir`, by default `databases` in the root), rename `invokeai.db`, which holds the state before the move, and put the new file in its place, named `invokeai.db`.
+4. Remove `db_url` from `invokeai.yaml`, and `INVOKEAI_DB_URL` from the environment, and start InvokeAI.
+
+The source is the database `db_url` names, unless `--source URL` names another; it is not changed. The new file appears only once every table matches its source, so a copy that fails leaves no file behind, and `invoke-db-copy` never overwrites an existing file. Ids continue as they did on the server: a queue item gets no id the server already gave out.
+
+## Limits
+
+- **One InvokeAI process per database.** At startup a process cancels the queue items left running and updates its bundled workflows and style presets, which a second process would undo. The first process holds a lock on the database; a second one refuses to start. If a process ends without closing its connection, because its computer crashed or lost the network, the server releases its lock within ten minutes; the refusal names the query that finds the connection to end sooner with `KILL`.
+- **One database server.** Setups with several writing servers (Galera, group replication with several primaries) are not supported.
+- **Busy database.** When a request waits too long for a lock held by another, finds no free connection, or cannot reach the server, it fails with HTTP 503 and `Retry-After: 1`, and can be repeated. If this happens often, the server is overloaded or another program holds locks in InvokeAI's database.
+- **Command-line tools.** The account commands (`invoke-useradd`, `invoke-userdel`, `invoke-usermod`, `invoke-userlist`) and the maintenance scripts use the database `invokeai.yaml` names. They refuse a database InvokeAI has not updated to their version: start InvokeAI once after an upgrade before using them.

docs/src/content/docs/configuration/Multi-User Mode/admin-guide.mdx

@@ -381,7 +381,7 @@ Then cut and paste the printed secret into this Sqlite3 command:
 sqlite3 INVOKE_ROOT/databases/invokeai.db 'update app_settings set value="THE_SECRET" where key="jwt_secret"'
 ```
 
-(replace INVOKE_ROOT with your InvokeAI root directory and THE_SECRET with the new secret).
+(replace INVOKE_ROOT with your InvokeAI root directory and THE_SECRET with the new secret). With a [MySQL or MariaDB database](/configuration/external-database/), run the same `UPDATE` in the server's client instead, with the value in single quotes.
 
 After this, restart the server. All logged in users will be logged out and will need to provide their usernames and passwords again.
 
@@ -451,7 +451,7 @@ It is best to restrict access to trusted networks and remote IP addresses, or us
 **Backup and Recovery:**
 
 It is always a good idea to periodically backup your InvokeAI database and images, but especially
-so if the server is publicly accessible to the Internet.
+so if the server is publicly accessible to the Internet. The commands below back up a SQLite database; back up a [MySQL or MariaDB database](/configuration/external-database/) with the server's own tools, such as `mysqldump` or `mariadb-dump`.
 
 **Manual Backup:**