CLDSRV-986: Pass overhead fields when abortMPU cleans up after completeMPU (hotfix/9.0.32)

.github/workflows/tests.yaml

@@ -816,98 +816,6 @@ jobs:
         source: /tmp/artifacts
       if: always()
 
-  ceph-backend-test:
-    runs-on: ubuntu-24.04
-    needs: build
-    env:
-      S3BACKEND: mem
-      S3DATA: multiple
-      S3KMS: file
-      CI_CEPH: 'true'
-      MPU_TESTING: "yes" 
-      S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
-      MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
-      CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage
-      JOB_NAME: ${{ github.job }}
-      ENABLE_NULL_VERSION_COMPAT_MODE: true # needed with mongodb backend
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Login to GitHub Registry
-      uses: docker/login-action@v3
-      with:
-        registry: ghcr.io
-        username: ${{ github.repository_owner }}
-        password: ${{ github.token }}
-    - name: Setup CI environment
-      uses: ./.github/actions/setup-ci
-    - uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: '3.2'
-    - name: Install Ruby dependencies
-      run: |
-        gem install nokogiri:1.15.5 excon:0.111.0 fog-aws:3.19.0 json:2.7.6 mime-types:3.5.2 rspec:3.12.0
-    - name: Install Java dependencies
-      run: |
-        sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
-    - name: Setup CI services
-      run: docker compose --profile ceph up -d
-      working-directory: .github/docker
-      env: 
-        S3METADATA: mongodb
-    - name: Run Ceph multiple backend tests
-      run: |-
-        set -ex -o pipefail;
-        bash .github/ceph/wait_for_ceph.sh
-        bash wait_for_local_port.bash 27018 40
-        bash wait_for_local_port.bash 8000 40
-        yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
-      env:
-        S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
-        S3METADATA: mem
-    - name: Run Java tests
-      run: |-
-        set -ex -o pipefail;
-        mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
-      working-directory: tests/functional/jaws
-    - name: Run Ruby tests
-      run: |-
-        set -ex -o pipefail;
-        rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
-      working-directory: tests/functional/fog
-    - name: Run Javascript AWS SDK tests
-      run: |-
-        set -ex -o pipefail;
-        yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
-        yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
-      env:
-        S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
-        S3BACKEND: file
-        S3VAULT: mem
-        S3METADATA: mongodb
-    - name: Cleanup and upload coverage
-      uses: ./.github/actions/cleanup-and-coverage
-      with:
-        profiles: ceph
-        codecov-token: ${{ secrets.CODECOV_TOKEN }}
-      if: always()
-    - name: Upload test results to Codecov
-      uses: codecov/test-results-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        files: '**/junit/*junit*.xml'
-        flags: ceph-backend-test
-      if: always() && !cancelled()
-    - name: Upload logs to artifacts
-      uses: scality/action-artifacts@v4
-      with:
-        method: upload
-        url: https://artifacts.scality.net
-        user: ${{ secrets.ARTIFACTS_USER }}
-        password: ${{ secrets.ARTIFACTS_PASSWORD }}
-        source: /tmp/artifacts
-      if: always()
-
   # This test with the final yarn run ft_sse_arn covers more code than the kmip tests
   sse-kms-migration-tests:
     strategy:

lib/api/apiUtils/object/abortMultipartUpload.js

@@ -167,6 +167,7 @@ function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
             const options = preprocessingVersioningDelete(
                 bucketName, destBucket, objectMD, objectMD.versionId, config.nullVersionCompatMode);
             options.replayId = uploadId;
+            options.overheadField = constants.overheadField;
             // This is a ghost metadata, without data loss, so skip oplog
             options.doesNotNeedOpogUpdate = true;
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenko/cloudserver",
-  "version": "9.0.32-2",
+  "version": "9.0.32-3",
   "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
   "main": "index.js",
   "engines": {

tests/functional/sse-kms-migration/load.js

@@ -16,11 +16,11 @@ const KMS_NODES = helpers.config.kmip.transport.length;
 const TOTAL_OBJECTS_PER_NODE = Math.floor(TOTAL_OBJECTS / KMS_NODES);
 
 /**
- * 10% approximation for the number of packets per IP
+ * 20% approximation for the number of packets per IP
  * As we might not have an exact match of packets and the
  * round robin is confined to each nodejs cluster processes
  */
-const APPROX = Math.floor(0.1 * TOTAL_OBJECTS_PER_NODE);
+const APPROX = Math.floor(0.2 * TOTAL_OBJECTS_PER_NODE);
 const EXPECTED_MIN = TOTAL_OBJECTS_PER_NODE - APPROX;
 const EXPECTED_MAX = TOTAL_OBJECTS_PER_NODE + APPROX;
 
@@ -177,7 +177,8 @@ describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER
         assert(repartition.length === KMS_NODES, `Expected ${KMS_NODES} IPs but got ${repartition.length}`);
         assert(repartitionCount.every(count =>
             count >= EXPECTED_MIN && count <= EXPECTED_MAX),
-            `Repartition counts should be around ${TOTAL_OBJECTS_PER_NODE} but got ${repartitionCount}`);
+            `Repartition counts should be around ${TOTAL_OBJECTS_PER_NODE} ` +
+            `(±${APPROX}, min: ${EXPECTED_MIN}, max: ${EXPECTED_MAX}) but got ${repartitionCount}`);
     }
 
     it(`should encrypt ${TOTAL_OBJECTS} times in parallel, ~${TOTAL_OBJECTS_PER_NODE} per node`, async () => {

tests/unit/api/apiUtils/object/abortMultipartUpload.spec.js

@@ -5,6 +5,7 @@ const { errors } = require('arsenal');
 const async = require('async');
 const crypto = require('crypto');
 
+const { overheadField } = require('../../../../../constants');
 const abortMultipartUpload = require('../../../../../lib/api/apiUtils/object/abortMultipartUpload');
 const { bucketPut } = require('../../../../../lib/api/bucketPut');
 const initiateMultipartUpload = require('../../../../../lib/api/initiateMultipartUpload');
@@ -305,6 +306,8 @@ describe('abortMultipartUpload', () => {
                 assert.ifError(err);
                 sinon.assert.calledOnce(deleteObjectMDStub);
                 assert.strictEqual(deleteObjectMDStub.getCall(0).args[2].versionId, 'orphan-vid');
+                assert.deepStrictEqual(
+                    deleteObjectMDStub.getCall(0).args[2].overheadField, overheadField);
                 done();
             }, { ...abortRequest, query: { uploadId: 'abort-id' } });
         });