test: replace MinIO with SeaweedFS for object-storage testing (#1647)

* test: run object-storage tests against SeaweedFS instead of MinIO

MinIO withdrew its public images and archived its source. SeaweedFS
4.48 matches MinIO on s3fs directory emulation, which GC relies on, and
passes the object-storage suites from a cold pull. The image is
overridable through DJ_TEST_S3_IMAGE.

Refs #1560

* test: finish moving the dev and test setup from MinIO to SeaweedFS

- docker-compose: replace the minio service with s3 (SeaweedFS 4.48,
  the image the fixture uses); the old minio/minio image no longer pulls
- rename the requires_minio marker to requires_s3 and the minio_client
  fixture to s3_client
- default S3_ENDPOINT for external containers is localhost:8333
- drop the unused minio extra from testcontainers; relock pixi
- update CONTRIBUTING.md

Refs #1560

Author: dimitri-yatsenko

CONTRIBUTING.md

@@ -36,7 +36,7 @@ pytest tests/
 
 ## Running Tests
 
-Tests use [testcontainers](https://testcontainers.com/) to automatically manage MySQL, PostgreSQL, and MinIO containers. No manual `docker-compose up` required.
+Tests use [testcontainers](https://testcontainers.com/) to automatically manage MySQL, PostgreSQL, and S3 ([SeaweedFS](https://github.com/seaweedfs/seaweedfs)) containers. No manual `docker-compose up` required.
 
 Integration tests are **backend-parameterized** — tests using the `backend` fixture run automatically against both MySQL and PostgreSQL.
 
@@ -67,13 +67,13 @@ Tests automatically spin up both MySQL and PostgreSQL containers via testcontain
 ### External Containers (for debugging)
 
 ```bash
-# MySQL + MinIO
-docker compose up -d db minio
+# MySQL + S3
+docker compose up -d db s3
 DJ_USE_EXTERNAL_CONTAINERS=1 pixi run -e test test
 docker compose down
 
-# MySQL + PostgreSQL + MinIO
-docker compose up -d db postgres minio
+# MySQL + PostgreSQL + S3
+docker compose up -d db postgres s3
 DJ_USE_EXTERNAL_CONTAINERS=1 pixi run -e test test
 docker compose down
 ```
@@ -133,7 +133,7 @@ For `DJ_USE_EXTERNAL_CONTAINERS=1`:
 
 | Variable | Default | Description |
 |----------|---------|-------------|
-| `S3_ENDPOINT` | `localhost:9000` | MinIO endpoint |
+| `S3_ENDPOINT` | `localhost:8333` | S3 endpoint |
 
 ---
 

docker-compose.yaml

@@ -1,12 +1,12 @@
-# Development environment with MySQL and MinIO services
+# Development environment with MySQL, PostgreSQL, and S3 (SeaweedFS) services
 #
 # NOTE: docker-compose is OPTIONAL for running tests.
 # Tests use testcontainers to automatically manage containers.
 # Just run: pytest tests/
 #
 # Use docker-compose for development/debugging when you want
 # persistent containers that survive test runs:
-#   docker compose up -d db minio   # Start services manually
+#   docker compose up -d db s3      # Start services manually
 #   pytest tests/                    # Tests will use these containers
 #
 # Full Docker testing (CI):
@@ -37,20 +37,17 @@ services:
       timeout: 30s
       retries: 5
       interval: 15s
-  minio:
-    image: minio/minio:${MINIO_VER:-RELEASE.2025-02-28T09-55-16Z}
+  s3:
+    # Same image as the test fixture in tests/conftest.py
+    image: chrislusf/seaweedfs:${SEAWEEDFS_VER:-4.48}
     environment:
-      - MINIO_ACCESS_KEY=datajoint
-      - MINIO_SECRET_KEY=datajoint
+      - AWS_ACCESS_KEY_ID=datajoint
+      - AWS_SECRET_ACCESS_KEY=datajoint
     ports:
-      - "9000:9000"
-    command: server --address ":9000" /data
+      - "8333:8333"
+    command: server -s3 -ip.bind=0.0.0.0
     healthcheck:
-      test:
-        - "CMD"
-        - "curl"
-        - "--fail"
-        - "http://localhost:9000/minio/health/live"
+      test: [ "CMD", "nc", "-z", "localhost", "8333" ]
       timeout: 30s
       retries: 5
       interval: 15s
@@ -67,7 +64,7 @@ services:
         condition: service_healthy
       postgres:
         condition: service_healthy
-      minio:
+      s3:
         condition: service_healthy
     environment:
       - DJ_HOST=db
@@ -80,7 +77,7 @@ services:
       - DJ_PG_USER=postgres
       - DJ_PG_PASS=password
       - DJ_PG_PORT=5432
-      - S3_ENDPOINT=minio:9000
+      - S3_ENDPOINT=s3:8333
       - S3_ACCESS_KEY=datajoint
       - S3_SECRET_KEY=datajoint
       - S3_BUCKET=datajoint.test

pixi.lock

@@ -96,7 +96,7 @@ test = [
   "matplotlib",
   "ipython",
   "graphviz",
-  "testcontainers[mysql,minio,postgres]>=4.0",
+  "testcontainers[mysql,postgres]>=4.0",
   "polars>=0.20.0",
   "pyarrow>=14.0.0",
 ]
@@ -117,7 +117,7 @@ test = [
   "matplotlib",
   "ipython",
   "s3fs>=2023.1.0",
-  "testcontainers[mysql,minio,postgres]>=4.0",
+  "testcontainers[mysql,postgres]>=4.0",
   "psycopg2-binary>=2.9.0",
   "polars>=0.20.0",
   "pyarrow>=14.0.0",
@@ -245,7 +245,7 @@ ignore-words-list = "rever,numer,astroid"
 [tool.pytest.ini_options]
 markers = [
     "requires_mysql: marks tests as requiring MySQL database (deselect with '-m \"not requires_mysql\"')",
-    "requires_minio: marks tests as requiring MinIO object storage (deselect with '-m \"not requires_minio\"')",
+    "requires_s3: marks tests as requiring S3-compatible object storage (deselect with '-m \"not requires_s3\"')",
     "mysql: marks tests that run on MySQL backend (select with '-m mysql')",
     "postgresql: marks tests that run on PostgreSQL backend (select with '-m postgresql')",
     "backend_agnostic: marks tests that should pass on all backends (auto-marked for parameterized tests)",
@@ -293,7 +293,7 @@ test-py315 = { features = ["test", "py315"] }
 test = "pytest tests/"
 test-cov = "pytest --cov-report term-missing --cov=datajoint tests/"
 # Optional: use external containers (docker-compose) instead of testcontainers
-services-up = "docker compose up -d db minio"
+services-up = "docker compose up -d db s3"
 services-down = "docker compose down"
 test-external = { cmd = "DJ_USE_EXTERNAL_CONTAINERS=1 pytest tests/", depends-on = ["services-up"] }
 

pyproject.toml

@@ -3,7 +3,7 @@
 
 Tests are organized by their dependencies:
 - Unit tests: No external dependencies, run with `pytest -m "not requires_mysql"`
-- Integration tests: Require MySQL/MinIO, marked with @pytest.mark.requires_mysql
+- Integration tests: Require MySQL/S3, marked with @pytest.mark.requires_mysql
 
 Containers are automatically started via testcontainers when needed.
 Just run: pytest tests/
@@ -58,9 +58,9 @@ def pytest_collection_modifyitems(config, items):
         "db_creds_root",
         "db_creds_test",
     }
-    # Tests that use these fixtures require MinIO
-    minio_fixtures = {
-        "minio_client",
+    # Tests that use these fixtures require S3 object storage
+    s3_fixtures = {
+        "s3_client",
         "s3fs_client",
         "s3_creds",
         "stores_config",
@@ -83,8 +83,8 @@ def pytest_collection_modifyitems(config, items):
         # Auto-add marks based on fixture usage
         if fixturenames & mysql_fixtures:
             item.add_marker(pytest.mark.requires_mysql)
-        if fixturenames & minio_fixtures:
-            item.add_marker(pytest.mark.requires_minio)
+        if fixturenames & s3_fixtures:
+            item.add_marker(pytest.mark.requires_s3)
 
         # Auto-mark backend-parameterized tests
         if fixturenames & backend_fixtures:
@@ -95,7 +95,7 @@ def pytest_collection_modifyitems(config, items):
 
 
 # =============================================================================
-# Container Fixtures - Auto-start MySQL and MinIO via testcontainers
+# Container Fixtures - Auto-start MySQL, PostgreSQL, and S3 via testcontainers
 # =============================================================================
 
 # Check if we should use external containers (for CI or manual docker-compose)
@@ -164,36 +164,50 @@ def postgres_container():
 
 
 @pytest.fixture(scope="session")
-def minio_container():
-    """Start MinIO container for the test session (or use external)."""
+def s3_container():
+    """Start an S3-compatible object store for the test session (or use external)."""
     if USE_EXTERNAL_CONTAINERS:
         # Use external container - return None, credentials come from env
-        logger.info("Using external MinIO container")
+        logger.info("Using external S3 container")
         yield None
         return
 
-    from testcontainers.minio import MinioContainer
-
-    # MinIO withdrew its public images: `minio/minio` on Docker Hub is gone
-    # entirely (any tag, not just `latest`) and `quay.io/minio/minio` now
-    # requires authentication. Chainguard publishes a maintained build that
-    # needs no credentials and is drop-in for this fixture. Overridable so a
-    # deployment can point at its own mirror without editing tests.
-    container = MinioContainer(
-        image=os.environ.get("DJ_TEST_MINIO_IMAGE", "cgr.dev/chainguard/minio:latest"),
-        access_key="datajoint",
-        secret_key="datajoint",
+    import time
+
+    from testcontainers.core.container import DockerContainer
+
+    # SeaweedFS replaces MinIO, which withdrew its public images and archived
+    # its source (#1560). It matched MinIO on s3fs directory emulation
+    # (isdir/walk/find over prefixes and zero-byte markers), which GC depends
+    # on. Overridable so a deployment can point at its own mirror.
+    container = (
+        DockerContainer(os.environ.get("DJ_TEST_S3_IMAGE", "chrislusf/seaweedfs:4.48"))
+        .with_command("server -s3 -ip.bind=0.0.0.0")
+        .with_env("AWS_ACCESS_KEY_ID", "datajoint")
+        .with_env("AWS_SECRET_ACCESS_KEY", "datajoint")
+        .with_exposed_ports(8333)
     )
     container.start()
 
     host = container.get_container_host_ip()
-    port = container.get_exposed_port(9000)
-    logger.info(f"MinIO container started at {host}:{port}")
+    port = container.get_exposed_port(8333)
+    # The S3 gateway starts after the master, volume, and filer servers
+    deadline = time.monotonic() + 60
+    while True:
+        try:
+            urllib3.PoolManager().request("GET", f"http://{host}:{port}/", retries=False, timeout=1)
+            break
+        except urllib3.exceptions.HTTPError:
+            if time.monotonic() > deadline:
+                container.stop()
+                raise
+            time.sleep(0.5)
+    logger.info(f"S3 container started at {host}:{port}")
 
     yield container
 
     container.stop()
-    logger.info("MinIO container stopped")
+    logger.info("S3 container stopped")
 
 
 # =============================================================================
@@ -253,12 +267,12 @@ def db_creds_test(mysql_container) -> Dict:
 
 
 @pytest.fixture(scope="session")
-def s3_creds(minio_container) -> Dict:
-    """S3/MinIO credentials from container or environment."""
-    if minio_container is not None:
+def s3_creds(s3_container) -> Dict:
+    """S3 credentials from container or environment."""
+    if s3_container is not None:
         # From testcontainer
-        host = minio_container.get_container_host_ip()
-        port = minio_container.get_exposed_port(9000)
+        host = s3_container.get_container_host_ip()
+        port = s3_container.get_exposed_port(8333)
         return dict(
             endpoint=f"{host}:{port}",
             access_key="datajoint",
@@ -268,7 +282,7 @@ def s3_creds(minio_container) -> Dict:
     else:
         # From environment (external container)
         return dict(
-            endpoint=os.environ.get("S3_ENDPOINT", "localhost:9000"),
+            endpoint=os.environ.get("S3_ENDPOINT", "localhost:8333"),
             access_key=os.environ.get("S3_ACCESS_KEY", "datajoint"),
             secret_key=os.environ.get("S3_SECRET_KEY", "datajoint"),
             bucket=os.environ.get("S3_BUCKET", "datajoint.test"),
@@ -487,7 +501,7 @@ def connection_test(connection_root, prefix, db_creds_test):
 
 
 # =============================================================================
-# S3/MinIO Fixtures
+# S3 Fixtures
 # =============================================================================
 
 
@@ -509,7 +523,7 @@ def stores_config(s3_creds, tmpdir_factory):
             bucket=s3_creds.get("bucket", "datajoint-test"),
             location="dj/repo",
             stage=str(tmpdir_factory.mktemp("repo-s3")),
-            secure=False,  # MinIO runs without SSL in tests
+            secure=False,  # The test S3 server runs without SSL
         ),
         "local": dict(protocol="file", location=str(tmpdir_factory.mktemp("local"))),
         "share": dict(
@@ -519,7 +533,7 @@ def stores_config(s3_creds, tmpdir_factory):
             secret_key=s3_creds["secret_key"],
             bucket=s3_creds.get("bucket", "datajoint-test"),
             location="dj/store/repo",
-            secure=False,  # MinIO runs without SSL in tests
+            secure=False,  # The test S3 server runs without SSL
         ),
     }
 
@@ -566,7 +580,7 @@ def http_client():
 
 @pytest.fixture(scope="session")
 def s3fs_client(s3_creds):
-    """Initialize s3fs filesystem for MinIO."""
+    """Initialize s3fs filesystem for the test S3 server."""
     import s3fs
 
     return s3fs.S3FileSystem(
@@ -577,8 +591,8 @@ def s3fs_client(s3_creds):
 
 
 @pytest.fixture(scope="session")
-def minio_client(s3_creds, s3fs_client, teardown=False):
-    """S3 filesystem with test bucket created (legacy name for compatibility)."""
+def s3_client(s3_creds, s3fs_client, teardown=False):
+    """S3 filesystem with the test bucket created."""
     bucket = s3_creds["bucket"]
 
     # Create bucket if it doesn't exist

tests/conftest.py

@@ -5,7 +5,7 @@
 from tests.schema_external import Attach
 
 
-def test_attach_attributes(schema_ext, minio_client, tmpdir_factory):
+def test_attach_attributes(schema_ext, s3_client, tmpdir_factory):
     """Test saving files in attachments"""
     import datajoint as dj
 
@@ -46,7 +46,7 @@ def test_attach_attributes(schema_ext, minio_client, tmpdir_factory):
         assert p2 == path2[0]
 
 
-def test_return_string(schema_ext, minio_client, tmpdir_factory):
+def test_return_string(schema_ext, s3_client, tmpdir_factory):
     """Test returning string on fetch"""
     import datajoint as dj
 

tests/integration/test_attach.py

@@ -72,7 +72,7 @@ def test_codec_graph(schema_codec):
     c.delete()
 
 
-def test_codec_chained(schema_codec, minio_client):
+def test_codec_chained(schema_codec, s3_client):
     """Test codec chaining (layout -> blob)."""
     c = Connectivity()
     c.delete()