feat(services/s3): add R2 and MinIO presets

.github/scripts/test_behavior/plan.py

@@ -35,6 +35,10 @@
 
 INTEGRATIONS = ["object_store"]
 
+SHARED_SERVICE_SCHEMES = {
+    "s3": {"s3", "minio", "r2"},
+}
+
 
 def provided_cases() -> list[dict[str, str]]:
     root_dir = f"{GITHUB_DIR}/services"
@@ -116,9 +120,10 @@ def mark_service_affected(service: str) -> None:
         for integration in INTEGRATIONS:
             setattr(hint, f"integration_{integration}", True)
 
-        hint.services.add(service)
-        hint.services.add(service.replace("-", "_"))
-        hint.services.add(service.replace("_", "-"))
+        for scheme in SHARED_SERVICE_SCHEMES.get(service, {service}):
+            hint.services.add(scheme)
+            hint.services.add(scheme.replace("-", "_"))
+            hint.services.add(scheme.replace("_", "-"))
 
     for p in changed_files:
         # workflow behavior tests affected
@@ -290,6 +295,10 @@ def generate_language_binding_cases(
     # Remove invalid cases for go.
     if language == "go":
         cases = [v for v in cases if v["service"] not in [
+            # Provider presets share the Rust S3 crate and don't have separate
+            # opendal-go-services packages.
+            "minio",
+            "r2",
             # opendal-go-services doesn't provide TOS yet.
             "tos",
         ]]
@@ -309,6 +318,8 @@ def generate_language_binding_cases(
             "memory",
             "obs",
             "oss",
+            "minio",
+            "r2",
             "s3",
             "webdav",
             "webhdfs",

.github/scripts/test_behavior/test_plan.py

@@ -53,6 +53,36 @@ def test_core_services_hdfs_native_mapping(self):
         self.assertTrue("hdfs_native" in cases)
         self.assertFalse("fs" in cases)
 
+    @patch.dict("os.environ", {"GITHUB_HAS_SECRETS": "true"})
+    def test_s3_crate_schedules_provider_cases(self):
+        result = plan(["core/services/s3/src/lib.rs"])
+        core_cases = {
+            (case["service"], case["feature"])
+            for target in result["core"]
+            for case in target["cases"]
+        }
+        self.assertEqual(
+            core_cases,
+            {
+                ("s3", "services-s3"),
+                ("minio", "services-minio"),
+                ("r2", "services-r2"),
+            },
+        )
+
+        go_services = {
+            case["service"]
+            for target in result["binding_go"]
+            for case in target["cases"]
+        }
+        ruby_services = {
+            case["service"]
+            for target in result["binding_ruby"]
+            for case in target["cases"]
+        }
+        self.assertTrue({"minio", "r2"}.isdisjoint(go_services))
+        self.assertTrue({"minio", "r2"} <= ruby_services)
+
     def test_binding_java(self):
         result = plan(["bindings/java/pom.xml"])
         self.assertFalse(result["components"]["core"])

.github/services/s3/0_minio_s3/action.yml → .github/services/minio/0_minio_s3/action.yml

@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: minio_s3
-description: 'Behavior test for Minio S3.'
+name: minio
+description: 'Behavior test for the MinIO provider preset.'
 
 runs:
   using: "composite"
@@ -38,10 +38,9 @@ runs:
       shell: bash
       run: |
         cat << EOF >> $GITHUB_ENV
-        OPENDAL_S3_BUCKET=test
-        OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000
-        OPENDAL_S3_ACCESS_KEY_ID=minioadmin
-        OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin
-        OPENDAL_S3_REGION=us-east-1
+        OPENDAL_MINIO_BUCKET=test
+        OPENDAL_MINIO_ENDPOINT=http://127.0.0.1:9000
+        OPENDAL_MINIO_ACCESS_KEY_ID=minioadmin
+        OPENDAL_MINIO_SECRET_ACCESS_KEY=minioadmin
         OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,write_can_append=false,copy_with_if_not_exists=false,copy_with_if_match=false
         EOF

.github/services/s3/minio_s3_with_anonymous/action.yml → .github/services/minio/minio_s3_with_anonymous/action.yml

@@ -15,8 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: minio_s3_with_anonymous
-description: 'Behavior test for Minio S3 with anonymous access.'
+name: minio_with_anonymous
+description: 'Behavior test for the MinIO provider preset with anonymous access.'
 
 runs:
   using: "composite"
@@ -44,10 +44,8 @@ runs:
       shell: bash
       run: |
         cat << EOF >> $GITHUB_ENV
-        OPENDAL_S3_BUCKET=test
-        OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000
-        OPENDAL_S3_REGION=us-east-1
-        OPENDAL_S3_ALLOW_ANONYMOUS=on
-        OPENDAL_S3_DISABLE_EC2_METADATA=on
+        OPENDAL_MINIO_BUCKET=test
+        OPENDAL_MINIO_ENDPOINT=http://127.0.0.1:9000
+        OPENDAL_MINIO_SKIP_SIGNATURE=true
         OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,write_can_append=false,copy_with_if_not_exists=false,copy_with_if_match=false
         EOF

.github/services/s3/r2/disabled_action.yml → .github/services/r2/r2/action.yml

@@ -16,7 +16,7 @@
 # under the License.
 
 name: r2
-description: "Behavior test for Cloudflare R2. This service is sponsored by @Xuanwo."
+description: "Behavior test for the Cloudflare R2 provider preset. This service is sponsored by @Xuanwo."
 
 runs:
   using: "composite"
@@ -26,17 +26,14 @@ runs:
       with:
         export-env: true
       env:
-        OPENDAL_S3_BUCKET: op://services/r2/bucket
-        OPENDAL_S3_ENDPOINT: op://services/r2/endpoint
-        OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
-        OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
+        OPENDAL_R2_BUCKET: op://services/r2/bucket
+        OPENDAL_R2_ENDPOINT: op://services/r2/endpoint
+        OPENDAL_R2_ACCESS_KEY_ID: op://services/r2/access_key_id
+        OPENDAL_R2_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
 
-    # R2 has a lower delete batch limit and doesn't support stat override response queries.
-    # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
-    - name: Add extra settings
+    - name: Add test overrides
       shell: bash
       run: |
         cat << EOF >> $GITHUB_ENV
-        OPENDAL_S3_REGION=auto
         OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,write_can_append=false,delete_max_size=700,stat_with_override_cache_control=false,stat_with_override_content_disposition=false,stat_with_override_content_type=false
         EOF

bindings/dart/rust/Cargo.toml

@@ -36,8 +36,10 @@ opendal = { path = "../../../core", features = [
   "services-http",
   "services-ipmfs",
   "services-memory",
+  "services-minio",
   "services-obs",
   "services-oss",
+  "services-r2",
   "services-s3",
   "services-webdav",
   "services-webhdfs",

bindings/dotnet/Cargo.toml

@@ -55,8 +55,10 @@ opendal = { version = ">=0", path = "../../core", features = [
   "services-http",
   "services-ipmfs",
   "services-memory",
+  "services-minio",
   "services-obs",
   "services-oss",
+  "services-r2",
   "services-s3",
   "services-tos",
   "services-webdav",

bindings/haskell/Cargo.toml

@@ -45,8 +45,10 @@ opendal = { version = ">=0", path = "../../core", features = [
   "services-http",
   "services-ipmfs",
   "services-memory",
+  "services-minio",
   "services-obs",
   "services-oss",
+  "services-r2",
   "services-s3",
   "services-webdav",
   "services-webhdfs",

bindings/java/Cargo.toml

@@ -54,8 +54,10 @@ opendal = { version = ">=0", path = "../../core", default-features = false, feat
   "services-http",
   "services-ipmfs",
   "services-memory",
+  "services-minio",
   "services-obs",
   "services-oss",
+  "services-r2",
   "services-s3",
   "services-tos",
   "services-webdav",

bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java

@@ -2023,6 +2023,88 @@ public Map<String, String> configMap() {
         }
     }
 
+    /**
+     * Configuration for service minio.
+     */
+    @Builder
+    @Data
+    @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+    class Minio implements ServiceConfig {
+        /**
+         * <p>Access key ID.</p>
+         * <p>Set this field together with <code>secret_access_key</code>.</p>
+         */
+        public final String accessKeyId;
+        /**
+         * <p>Bucket name.</p>
+         * <p>This field is required.</p>
+         */
+        public final @NonNull String bucket;
+        /**
+         * <p>MinIO endpoint.</p>
+         * <p>This field is required because MinIO deployments do not share a
+         * universal endpoint.</p>
+         */
+        public final @NonNull String endpoint;
+        /**
+         * <p>Signing region.</p>
+         * <p>The default is <code>auto</code>. Set this field when the deployment requires a
+         * configured region.</p>
+         */
+        public final String region;
+        /**
+         * <p>Root within the bucket.</p>
+         * <p>All operations happen under this root. The default is <code>/</code>.</p>
+         */
+        public final String root;
+        /**
+         * <p>Secret access key.</p>
+         * <p>Set this field together with <code>access_key_id</code>.</p>
+         */
+        public final String secretAccessKey;
+        /**
+         * <p>Session token for temporary credentials.</p>
+         * <p>This field requires <code>access_key_id</code> and <code>secret_access_key</code>.</p>
+         */
+        public final String sessionToken;
+        /**
+         * <p>Send requests without signing them.</p>
+         * <p>This option cannot be combined with direct credentials.</p>
+         */
+        public final Boolean skipSignature;
+
+        @Override
+        public String scheme() {
+            return "minio";
+        }
+
+        @Override
+        public Map<String, String> configMap() {
+            final HashMap<String, String> map = new HashMap<>();
+            if (accessKeyId != null) {
+                map.put("access_key_id", accessKeyId);
+            }
+            map.put("bucket", bucket);
+            map.put("endpoint", endpoint);
+            if (region != null) {
+                map.put("region", region);
+            }
+            if (root != null) {
+                map.put("root", root);
+            }
+            if (secretAccessKey != null) {
+                map.put("secret_access_key", secretAccessKey);
+            }
+            if (sessionToken != null) {
+                map.put("session_token", sessionToken);
+            }
+            if (skipSignature != null) {
+                map.put("skip_signature", String.valueOf(skipSignature));
+            }
+            return map;
+        }
+    }
+
     /**
      * Configuration for service moka.
      */
@@ -2752,6 +2834,90 @@ public Map<String, String> configMap() {
         }
     }
 
+    /**
+     * Configuration for service r2.
+     */
+    @Builder
+    @Data
+    @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
+    class R2 implements ServiceConfig {
+        /**
+         * <p>Access key ID.</p>
+         * <p>Set this field together with <code>secret_access_key</code>.</p>
+         */
+        public final String accessKeyId;
+        /**
+         * <p>Cloudflare account ID used to derive the R2 endpoint.</p>
+         * <p>Set exactly one of <code>account_id</code> and <code>endpoint</code>.</p>
+         */
+        public final String accountId;
+        /**
+         * <p>Bucket name.</p>
+         * <p>This field is required.</p>
+         */
+        public final @NonNull String bucket;
+        /**
+         * <p>Explicit R2-compatible endpoint.</p>
+         * <p>Use this field for a proxy, gateway, or test server. Set exactly one of
+         * <code>endpoint</code> and <code>account_id</code>.</p>
+         */
+        public final String endpoint;
+        /**
+         * <p>R2 jurisdiction.</p>
+         * <p>Supported values are <code>eu</code> and <code>fedramp</code>. This field requires
+         * <code>account_id</code> and cannot be used with <code>endpoint</code>.</p>
+         */
+        public final String jurisdiction;
+        /**
+         * <p>Root within the bucket.</p>
+         * <p>All operations happen under this root. The default is <code>/</code>.</p>
+         */
+        public final String root;
+        /**
+         * <p>Secret access key.</p>
+         * <p>Set this field together with <code>access_key_id</code>.</p>
+         */
+        public final String secretAccessKey;
+        /**
+         * <p>Session token for temporary credentials.</p>
+         * <p>This field requires <code>access_key_id</code> and <code>secret_access_key</code>.</p>
+         */
+        public final String sessionToken;
+
+        @Override
+        public String scheme() {
+            return "r2";
+        }
+
+        @Override
+        public Map<String, String> configMap() {
+            final HashMap<String, String> map = new HashMap<>();
+            if (accessKeyId != null) {
+                map.put("access_key_id", accessKeyId);
+            }
+            if (accountId != null) {
+                map.put("account_id", accountId);
+            }
+            map.put("bucket", bucket);
+            if (endpoint != null) {
+                map.put("endpoint", endpoint);
+            }
+            if (jurisdiction != null) {
+                map.put("jurisdiction", jurisdiction);
+            }
+            if (root != null) {
+                map.put("root", root);
+            }
+            if (secretAccessKey != null) {
+                map.put("secret_access_key", secretAccessKey);
+            }
+            if (sessionToken != null) {
+                map.put("session_token", sessionToken);
+            }
+            return map;
+        }
+    }
+
     /**
      * Configuration for service redb.
      */

bindings/lua/Cargo.toml

@@ -51,8 +51,10 @@ opendal = { version = ">=0", path = "../../core", features = [
   "services-http",
   "services-ipmfs",
   "services-memory",
+  "services-minio",
   "services-obs",
   "services-oss",
+  "services-r2",
   "services-s3",
   "services-webdav",
   "services-webhdfs",