From 376224405be113159d5bac28ba1818dcf7740f75 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 19 Sep 2026 23:59:26 +0200 Subject: [PATCH] CI: run the S3 tests against moto instead of MinIO dl.min.io answers "410 Gone" for the minio and mc release binaries now, so the "Install and configure MinIO S3 server" step failed in every test job (and fail-fast cancelled the rest of the matrix). Use moto in server mode (moto_server) instead, like borg does since borgbackup/borg#10386: it comes from PyPI, so there is no server binary to download. The versions are pinned in requirements.d/s3-test-server.txt, which dependabot watches. moto's "server" extra pulls in the dependencies of all the AWS services it can mock; for S3, flask and flask-cors on top of the "s3" extra are enough. The step moved behind the Python setup, because it needs pip. moto keeps the objects in memory and accepts any credentials; the bucket is created with boto3. curl is not needed anymore. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 45 +++++++++++++------------------ requirements.d/s3-test-server.txt | 6 +++++ 2 files changed, 24 insertions(+), 27 deletions(-) create mode 100644 requirements.d/s3-test-server.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 216473c..4bcf08c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install -y rclone openssh-server curl + sudo apt-get install -y rclone openssh-server - name: Install macOS packages if: runner.os == 'macOS' run: | @@ -118,32 +118,6 @@ jobs: ssh-add ~/.ssh/id_ed25519 # Export SFTP test URL for tox via GITHUB_ENV echo "BORGSTORE_TEST_SFTP_URL=sftp://sftpuser@localhost:22/borgstore/temp-store" >> $GITHUB_ENV - - name: Install and configure MinIO S3 server (test only) - if: runner.os == 'Linux' && matrix.toxenv != 'flake8' && matrix.toxenv != 'mypy' - run: | - set -e - arch=$(uname -m) - case "$arch" in - x86_64|amd64) srv_url=https://dl.min.io/server/minio/release/linux-amd64/minio; cli_url=https://dl.min.io/client/mc/release/linux-amd64/mc ;; - aarch64|arm64) srv_url=https://dl.min.io/server/minio/release/linux-arm64/minio; cli_url=https://dl.min.io/client/mc/release/linux-arm64/mc ;; - *) echo "Unsupported arch: $arch"; exit 1 ;; - esac - curl -fsSL -o /usr/local/bin/minio "$srv_url" - curl -fsSL -o /usr/local/bin/mc "$cli_url" - sudo chmod +x /usr/local/bin/minio /usr/local/bin/mc - export PATH=/usr/local/bin:$PATH - # Start MinIO on :9000 with default credentials (minioadmin/minioadmin) - MINIO_DIR="$GITHUB_WORKSPACE/.minio-data" - MINIO_LOG="$GITHUB_WORKSPACE/.minio.log" - mkdir -p "$MINIO_DIR" - nohup minio server "$MINIO_DIR" --address ":9000" >"$MINIO_LOG" 2>&1 & - # Wait for MinIO port to be ready - for i in $(seq 1 60); do (echo > /dev/tcp/127.0.0.1/9000) >/dev/null 2>&1 && break; sleep 1; done - # Configure client and create bucket - mc alias set local http://127.0.0.1:9000 minioadmin minioadmin - mc mb --ignore-existing local/test - # Export S3 test URL for tox via GITHUB_ENV - echo "BORGSTORE_TEST_S3_URL=s3:minioadmin:minioadmin@http://127.0.0.1:9000/test/path" >> $GITHUB_ENV - name: Configure nginx + borgstore REST server via systemd socket activation (test only) if: runner.os == 'Linux' && matrix.toxenv != 'flake8' && matrix.toxenv != 'mypy' run: | @@ -211,6 +185,21 @@ jobs: run: | python -m pip install --upgrade pip setuptools pip install -r requirements.d/dev.txt + - name: Install and start moto S3 server (test only) + # The S3 tests run against moto in server mode: it comes from PyPI (pinned in + # requirements.d/s3-test-server.txt), so there is no server binary to download. + # It keeps the objects in memory and accepts any credentials. + if: runner.os == 'Linux' && matrix.toxenv != 'flake8' && matrix.toxenv != 'mypy' + run: | + set -e + pip install -r requirements.d/s3-test-server.txt + nohup moto_server -H 127.0.0.1 -p 9000 > "$RUNNER_TEMP/moto.log" 2>&1 & + # Wait for the moto port to be ready + for i in $(seq 1 60); do (echo > /dev/tcp/127.0.0.1/9000) >/dev/null 2>&1 && break; sleep 1; done + # Create the bucket (boto3 is a moto dependency) + python -c 'import boto3; boto3.client("s3", endpoint_url="http://127.0.0.1:9000", aws_access_key_id="test", aws_secret_access_key="test").create_bucket(Bucket="test")' + # Export S3 test URL for tox via GITHUB_ENV + echo "BORGSTORE_TEST_S3_URL=s3:test:test@http://127.0.0.1:9000/test/path" >> $GITHUB_ENV - name: Install borgstore (Linux) if: runner.os == 'Linux' run: pip install -ve ".[s3,sftp,rest,rclone,blake3]" @@ -236,3 +225,5 @@ jobs: sudo journalctl -u borgstore@repo1.service --no-pager -n 100 || true echo "--- borgstore@repo1 status ---" sudo systemctl status borgstore@repo1.service --no-pager || true + echo "--- moto S3 server log ---" + cat "$RUNNER_TEMP/moto.log" || true diff --git a/requirements.d/s3-test-server.txt b/requirements.d/s3-test-server.txt new file mode 100644 index 0000000..3daa7d1 --- /dev/null +++ b/requirements.d/s3-test-server.txt @@ -0,0 +1,6 @@ +# The S3 server the CI tests run against: moto in server mode ("moto_server"). +# moto's own "server" extra pulls in the dependencies of all the AWS services it can +# mock, for S3 the server only needs flask and flask-cors on top of the "s3" extra. +moto[s3]==5.2.3 +flask==3.1.3 +flask-cors==6.0.5