From dfa837e3b4b9ae26f6526eaebf2675496f88d248 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 3 Aug 2026 00:39:51 +0800 Subject: [PATCH] feat(services/s3): add R2 and MinIO presets --- .github/scripts/test_behavior/plan.py | 17 +- .github/scripts/test_behavior/test_plan.py | 30 ++ .../{s3 => minio}/0_minio_s3/action.yml | 13 +- .../minio_s3_with_anonymous/action.yml | 12 +- .../disabled_action.yml => r2/r2/action.yml} | 15 +- bindings/dart/rust/Cargo.toml | 2 + bindings/dotnet/Cargo.toml | 2 + bindings/haskell/Cargo.toml | 2 + bindings/java/Cargo.toml | 2 + .../org/apache/opendal/ServiceConfig.java | 166 ++++++++++ bindings/lua/Cargo.toml | 2 + bindings/nodejs/Cargo.toml | 4 + bindings/ocaml/Cargo.toml | 2 + bindings/php/Cargo.toml | 2 + bindings/python/Cargo.toml | 4 + bindings/python/python/opendal/config.py | 50 +++ bindings/python/src/services.rs | 8 + bindings/ruby/Cargo.toml | 2 + core/Cargo.toml | 2 + .../src/docs/rfcs/7948_s3_provider_presets.md | 39 ++- core/fuzz/Cargo.toml | 2 + core/services/s3/README.md | 25 +- core/services/s3/src/backend.rs | 113 ++++--- core/services/s3/src/compatible_services.md | 46 +-- core/services/s3/src/docs.md | 2 +- core/services/s3/src/lib.rs | 66 +++- core/services/s3/src/minio.md | 51 +++ core/services/s3/src/minio.rs | 257 +++++++++++++++ core/services/s3/src/minio_config.rs | 161 ++++++++++ core/services/s3/src/preset.rs | 180 +++++++++++ core/services/s3/src/r2.md | 53 ++++ core/services/s3/src/r2.rs | 298 ++++++++++++++++++ core/services/s3/src/r2_config.rs | 158 ++++++++++ core/src/lib.rs | 51 ++- dev/src/generate/docs.rs | 54 +++- dev/src/generate/parser.rs | 112 ++++++- dev/src/generate/python.rs | 8 +- justfile | 2 +- website/data/services.json | 205 ++++++++++++ 39 files changed, 2060 insertions(+), 160 deletions(-) rename .github/services/{s3 => minio}/0_minio_s3/action.yml (85%) rename .github/services/{s3 => minio}/minio_s3_with_anonymous/action.yml (86%) rename .github/services/{s3/r2/disabled_action.yml => r2/r2/action.yml} (71%) create mode 100644 core/services/s3/src/minio.md create mode 100644 core/services/s3/src/minio.rs create mode 100644 core/services/s3/src/minio_config.rs create mode 100644 core/services/s3/src/preset.rs create mode 100644 core/services/s3/src/r2.md create mode 100644 core/services/s3/src/r2.rs create mode 100644 core/services/s3/src/r2_config.rs diff --git a/.github/scripts/test_behavior/plan.py b/.github/scripts/test_behavior/plan.py index 3481be704dcc..bfa71585d69e 100755 --- a/.github/scripts/test_behavior/plan.py +++ b/.github/scripts/test_behavior/plan.py @@ -35,6 +35,10 @@ INTEGRATIONS = ["object_store"] +SHARED_SERVICE_SCHEMES = { + "s3": {"s3", "minio", "r2"}, +} + def provided_cases() -> list[dict[str, str]]: root_dir = f"{GITHUB_DIR}/services" @@ -116,9 +120,10 @@ def mark_service_affected(service: str) -> None: for integration in INTEGRATIONS: setattr(hint, f"integration_{integration}", True) - hint.services.add(service) - hint.services.add(service.replace("-", "_")) - hint.services.add(service.replace("_", "-")) + for scheme in SHARED_SERVICE_SCHEMES.get(service, {service}): + hint.services.add(scheme) + hint.services.add(scheme.replace("-", "_")) + hint.services.add(scheme.replace("_", "-")) for p in changed_files: # workflow behavior tests affected @@ -290,6 +295,10 @@ def generate_language_binding_cases( # Remove invalid cases for go. if language == "go": cases = [v for v in cases if v["service"] not in [ + # Provider presets share the Rust S3 crate and don't have separate + # opendal-go-services packages. + "minio", + "r2", # opendal-go-services doesn't provide TOS yet. "tos", ]] @@ -309,6 +318,8 @@ def generate_language_binding_cases( "memory", "obs", "oss", + "minio", + "r2", "s3", "webdav", "webhdfs", diff --git a/.github/scripts/test_behavior/test_plan.py b/.github/scripts/test_behavior/test_plan.py index 76abd651163b..82cac761841e 100644 --- a/.github/scripts/test_behavior/test_plan.py +++ b/.github/scripts/test_behavior/test_plan.py @@ -53,6 +53,36 @@ def test_core_services_hdfs_native_mapping(self): self.assertTrue("hdfs_native" in cases) self.assertFalse("fs" in cases) + @patch.dict("os.environ", {"GITHUB_HAS_SECRETS": "true"}) + def test_s3_crate_schedules_provider_cases(self): + result = plan(["core/services/s3/src/lib.rs"]) + core_cases = { + (case["service"], case["feature"]) + for target in result["core"] + for case in target["cases"] + } + self.assertEqual( + core_cases, + { + ("s3", "services-s3"), + ("minio", "services-minio"), + ("r2", "services-r2"), + }, + ) + + go_services = { + case["service"] + for target in result["binding_go"] + for case in target["cases"] + } + ruby_services = { + case["service"] + for target in result["binding_ruby"] + for case in target["cases"] + } + self.assertTrue({"minio", "r2"}.isdisjoint(go_services)) + self.assertTrue({"minio", "r2"} <= ruby_services) + def test_binding_java(self): result = plan(["bindings/java/pom.xml"]) self.assertFalse(result["components"]["core"]) diff --git a/.github/services/s3/0_minio_s3/action.yml b/.github/services/minio/0_minio_s3/action.yml similarity index 85% rename from .github/services/s3/0_minio_s3/action.yml rename to .github/services/minio/0_minio_s3/action.yml index f74a07701c56..6b3c8204b5fd 100644 --- a/.github/services/s3/0_minio_s3/action.yml +++ b/.github/services/minio/0_minio_s3/action.yml @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -name: minio_s3 -description: 'Behavior test for Minio S3.' +name: minio +description: 'Behavior test for the MinIO provider preset.' runs: using: "composite" @@ -38,10 +38,9 @@ runs: shell: bash run: | cat << EOF >> $GITHUB_ENV - OPENDAL_S3_BUCKET=test - OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000 - OPENDAL_S3_ACCESS_KEY_ID=minioadmin - OPENDAL_S3_SECRET_ACCESS_KEY=minioadmin - OPENDAL_S3_REGION=us-east-1 + OPENDAL_MINIO_BUCKET=test + OPENDAL_MINIO_ENDPOINT=http://127.0.0.1:9000 + OPENDAL_MINIO_ACCESS_KEY_ID=minioadmin + OPENDAL_MINIO_SECRET_ACCESS_KEY=minioadmin OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,write_can_append=false,copy_with_if_not_exists=false,copy_with_if_match=false EOF diff --git a/.github/services/s3/minio_s3_with_anonymous/action.yml b/.github/services/minio/minio_s3_with_anonymous/action.yml similarity index 86% rename from .github/services/s3/minio_s3_with_anonymous/action.yml rename to .github/services/minio/minio_s3_with_anonymous/action.yml index 5bd46d42f2c0..d6cf441f1bbe 100644 --- a/.github/services/s3/minio_s3_with_anonymous/action.yml +++ b/.github/services/minio/minio_s3_with_anonymous/action.yml @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -name: minio_s3_with_anonymous -description: 'Behavior test for Minio S3 with anonymous access.' +name: minio_with_anonymous +description: 'Behavior test for the MinIO provider preset with anonymous access.' runs: using: "composite" @@ -44,10 +44,8 @@ runs: shell: bash run: | cat << EOF >> $GITHUB_ENV - OPENDAL_S3_BUCKET=test - OPENDAL_S3_ENDPOINT=http://127.0.0.1:9000 - OPENDAL_S3_REGION=us-east-1 - OPENDAL_S3_ALLOW_ANONYMOUS=on - OPENDAL_S3_DISABLE_EC2_METADATA=on + OPENDAL_MINIO_BUCKET=test + OPENDAL_MINIO_ENDPOINT=http://127.0.0.1:9000 + OPENDAL_MINIO_SKIP_SIGNATURE=true OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,write_can_append=false,copy_with_if_not_exists=false,copy_with_if_match=false EOF diff --git a/.github/services/s3/r2/disabled_action.yml b/.github/services/r2/r2/action.yml similarity index 71% rename from .github/services/s3/r2/disabled_action.yml rename to .github/services/r2/r2/action.yml index f9952414e5be..f4f91231865a 100644 --- a/.github/services/s3/r2/disabled_action.yml +++ b/.github/services/r2/r2/action.yml @@ -16,7 +16,7 @@ # under the License. name: r2 -description: "Behavior test for Cloudflare R2. This service is sponsored by @Xuanwo." +description: "Behavior test for the Cloudflare R2 provider preset. This service is sponsored by @Xuanwo." runs: using: "composite" @@ -26,17 +26,14 @@ runs: with: export-env: true env: - OPENDAL_S3_BUCKET: op://services/r2/bucket - OPENDAL_S3_ENDPOINT: op://services/r2/endpoint - OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id - OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key + OPENDAL_R2_BUCKET: op://services/r2/bucket + OPENDAL_R2_ENDPOINT: op://services/r2/endpoint + OPENDAL_R2_ACCESS_KEY_ID: op://services/r2/access_key_id + OPENDAL_R2_SECRET_ACCESS_KEY: op://services/r2/secret_access_key - # R2 has a lower delete batch limit and doesn't support stat override response queries. - # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information - - name: Add extra settings + - name: Add test overrides shell: bash run: | cat << EOF >> $GITHUB_ENV - OPENDAL_S3_REGION=auto OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,copy_with_source_version=false,write_can_append=false,delete_max_size=700,stat_with_override_cache_control=false,stat_with_override_content_disposition=false,stat_with_override_content_type=false EOF diff --git a/bindings/dart/rust/Cargo.toml b/bindings/dart/rust/Cargo.toml index 84c15d573e1f..60e89cb439df 100644 --- a/bindings/dart/rust/Cargo.toml +++ b/bindings/dart/rust/Cargo.toml @@ -36,8 +36,10 @@ opendal = { path = "../../../core", features = [ "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-webdav", "services-webhdfs", diff --git a/bindings/dotnet/Cargo.toml b/bindings/dotnet/Cargo.toml index 56ab13bb1a8e..41070aeb96a2 100644 --- a/bindings/dotnet/Cargo.toml +++ b/bindings/dotnet/Cargo.toml @@ -55,8 +55,10 @@ opendal = { version = ">=0", path = "../../core", features = [ "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-tos", "services-webdav", diff --git a/bindings/haskell/Cargo.toml b/bindings/haskell/Cargo.toml index 1dacd260227a..96caaf26a8ee 100644 --- a/bindings/haskell/Cargo.toml +++ b/bindings/haskell/Cargo.toml @@ -45,8 +45,10 @@ opendal = { version = ">=0", path = "../../core", features = [ "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-webdav", "services-webhdfs", diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index f75d42e76780..a246c17dfb7a 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -54,8 +54,10 @@ opendal = { version = ">=0", path = "../../core", default-features = false, feat "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-tos", "services-webdav", diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java index d545a3ee5fcc..560dd7c15f14 100644 --- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java +++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java @@ -2023,6 +2023,88 @@ public Map configMap() { } } + /** + * Configuration for service minio. + */ + @Builder + @Data + @RequiredArgsConstructor(access = AccessLevel.PRIVATE) + class Minio implements ServiceConfig { + /** + *

Access key ID.

+ *

Set this field together with secret_access_key.

+ */ + public final String accessKeyId; + /** + *

Bucket name.

+ *

This field is required.

+ */ + public final @NonNull String bucket; + /** + *

MinIO endpoint.

+ *

This field is required because MinIO deployments do not share a + * universal endpoint.

+ */ + public final @NonNull String endpoint; + /** + *

Signing region.

+ *

The default is auto. Set this field when the deployment requires a + * configured region.

+ */ + public final String region; + /** + *

Root within the bucket.

+ *

All operations happen under this root. The default is /.

+ */ + public final String root; + /** + *

Secret access key.

+ *

Set this field together with access_key_id.

+ */ + public final String secretAccessKey; + /** + *

Session token for temporary credentials.

+ *

This field requires access_key_id and secret_access_key.

+ */ + public final String sessionToken; + /** + *

Send requests without signing them.

+ *

This option cannot be combined with direct credentials.

+ */ + public final Boolean skipSignature; + + @Override + public String scheme() { + return "minio"; + } + + @Override + public Map configMap() { + final HashMap map = new HashMap<>(); + if (accessKeyId != null) { + map.put("access_key_id", accessKeyId); + } + map.put("bucket", bucket); + map.put("endpoint", endpoint); + if (region != null) { + map.put("region", region); + } + if (root != null) { + map.put("root", root); + } + if (secretAccessKey != null) { + map.put("secret_access_key", secretAccessKey); + } + if (sessionToken != null) { + map.put("session_token", sessionToken); + } + if (skipSignature != null) { + map.put("skip_signature", String.valueOf(skipSignature)); + } + return map; + } + } + /** * Configuration for service moka. */ @@ -2752,6 +2834,90 @@ public Map configMap() { } } + /** + * Configuration for service r2. + */ + @Builder + @Data + @RequiredArgsConstructor(access = AccessLevel.PRIVATE) + class R2 implements ServiceConfig { + /** + *

Access key ID.

+ *

Set this field together with secret_access_key.

+ */ + public final String accessKeyId; + /** + *

Cloudflare account ID used to derive the R2 endpoint.

+ *

Set exactly one of account_id and endpoint.

+ */ + public final String accountId; + /** + *

Bucket name.

+ *

This field is required.

+ */ + public final @NonNull String bucket; + /** + *

Explicit R2-compatible endpoint.

+ *

Use this field for a proxy, gateway, or test server. Set exactly one of + * endpoint and account_id.

+ */ + public final String endpoint; + /** + *

R2 jurisdiction.

+ *

Supported values are eu and fedramp. This field requires + * account_id and cannot be used with endpoint.

+ */ + public final String jurisdiction; + /** + *

Root within the bucket.

+ *

All operations happen under this root. The default is /.

+ */ + public final String root; + /** + *

Secret access key.

+ *

Set this field together with access_key_id.

+ */ + public final String secretAccessKey; + /** + *

Session token for temporary credentials.

+ *

This field requires access_key_id and secret_access_key.

+ */ + public final String sessionToken; + + @Override + public String scheme() { + return "r2"; + } + + @Override + public Map configMap() { + final HashMap map = new HashMap<>(); + if (accessKeyId != null) { + map.put("access_key_id", accessKeyId); + } + if (accountId != null) { + map.put("account_id", accountId); + } + map.put("bucket", bucket); + if (endpoint != null) { + map.put("endpoint", endpoint); + } + if (jurisdiction != null) { + map.put("jurisdiction", jurisdiction); + } + if (root != null) { + map.put("root", root); + } + if (secretAccessKey != null) { + map.put("secret_access_key", secretAccessKey); + } + if (sessionToken != null) { + map.put("session_token", sessionToken); + } + return map; + } + } + /** * Configuration for service redb. */ diff --git a/bindings/lua/Cargo.toml b/bindings/lua/Cargo.toml index 761445ae622f..5b3289c73a89 100644 --- a/bindings/lua/Cargo.toml +++ b/bindings/lua/Cargo.toml @@ -51,8 +51,10 @@ opendal = { version = ">=0", path = "../../core", features = [ "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-webdav", "services-webhdfs", diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml index e39be989b59d..a9847adfee68 100644 --- a/bindings/nodejs/Cargo.toml +++ b/bindings/nodejs/Cargo.toml @@ -37,8 +37,10 @@ default = [ "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-webdav", "services-webhdfs", @@ -102,8 +104,10 @@ services-ghac = ["opendal/services-ghac"] services-http = ["opendal/services-http"] services-ipmfs = ["opendal/services-ipmfs"] services-memory = ["opendal/services-memory"] +services-minio = ["opendal/services-minio"] services-obs = ["opendal/services-obs"] services-oss = ["opendal/services-oss"] +services-r2 = ["opendal/services-r2"] services-s3 = ["opendal/services-s3"] services-webdav = ["opendal/services-webdav"] services-webhdfs = ["opendal/services-webhdfs"] diff --git a/bindings/ocaml/Cargo.toml b/bindings/ocaml/Cargo.toml index 74e501446bd2..1c976a265e50 100644 --- a/bindings/ocaml/Cargo.toml +++ b/bindings/ocaml/Cargo.toml @@ -45,8 +45,10 @@ opendal = { version = ">=0", path = "../../core", features = [ "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-webdav", "services-webhdfs", diff --git a/bindings/php/Cargo.toml b/bindings/php/Cargo.toml index 8e6ce7ac2d5e..5d2cc404bc24 100644 --- a/bindings/php/Cargo.toml +++ b/bindings/php/Cargo.toml @@ -44,8 +44,10 @@ opendal = { version = ">=0", path = "../../core", features = [ "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-webdav", "services-webhdfs", diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index a30fc777d51c..9fc44e274e90 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -39,8 +39,10 @@ default = [ "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-webdav", "services-webhdfs", @@ -141,6 +143,7 @@ services-lakefs = [ services-memcached = ["opendal/services-memcached"] services-memory = ["opendal/services-memory"] services-mini-moka = ["opendal/services-mini-moka"] +services-minio = ["opendal/services-minio"] services-moka = ["opendal/services-moka"] services-mongodb = ["opendal/services-mongodb"] services-monoiofs = [ @@ -158,6 +161,7 @@ services-pcloud = [ ] # FIXME EXCLUDED: Needs tests/ maintenance services-persy = ["opendal/services-persy"] services-postgresql = ["opendal/services-postgresql"] +services-r2 = ["opendal/services-r2"] services-redb = ["opendal/services-redb"] services-redis = ["opendal/services-redis"] services-rocksdb = [ diff --git a/bindings/python/python/opendal/config.py b/bindings/python/python/opendal/config.py index 54a4b7a6315a..9835d7bf5f49 100644 --- a/bindings/python/python/opendal/config.py +++ b/bindings/python/python/opendal/config.py @@ -492,6 +492,29 @@ class MiniMokaConfig(TypedDict): """Sets the time to live of the cache. Refer to [`mini-moka::sync::CacheBuilder::time_to_live`](https://docs.rs/mini-moka/latest/mini_moka/sync/struct.CacheBuilder.html#method.time_to_live)""" +class MinioConfig(TypedDict): + """Configuration for the `minio` service.""" + + scheme: Required[Literal["minio"]] + """The service scheme; fixed to `"minio"`.""" + access_key_id: NotRequired[str] + """Access key ID. Set this field together with `secret_access_key`.""" + bucket: Required[str] + """Bucket name. This field is required.""" + endpoint: Required[str] + """MinIO endpoint. This field is required because MinIO deployments do not share a universal endpoint.""" + region: NotRequired[str] + """Signing region. The default is `auto`. Set this field when the deployment requires a configured region.""" + root: NotRequired[str | os.PathLike[str]] + """Root within the bucket. All operations happen under this root. The default is `/`.""" + secret_access_key: NotRequired[str] + """Secret access key. Set this field together with `access_key_id`.""" + session_token: NotRequired[str] + """Session token for temporary credentials. This field requires `access_key_id` and `secret_access_key`.""" + skip_signature: NotRequired[bool] + """Send requests without signing them. This option cannot be combined with direct credentials.""" + + class MokaConfig(TypedDict): """Configuration for the `moka` service.""" @@ -664,6 +687,29 @@ class PostgresqlConfig(TypedDict): """the value field of postgresql""" +class R2Config(TypedDict): + """Configuration for the `r2` service.""" + + scheme: Required[Literal["r2"]] + """The service scheme; fixed to `"r2"`.""" + access_key_id: NotRequired[str] + """Access key ID. Set this field together with `secret_access_key`.""" + account_id: NotRequired[str] + """Cloudflare account ID used to derive the R2 endpoint. Set exactly one of `account_id` and `endpoint`.""" + bucket: Required[str] + """Bucket name. This field is required.""" + endpoint: NotRequired[str] + """Explicit R2-compatible endpoint. Use this field for a proxy, gateway, or test server. Set exactly one of `endpoint` and `account_id`.""" + jurisdiction: NotRequired[str] + """R2 jurisdiction. Supported values are `eu` and `fedramp`. This field requires `account_id` and cannot be used with `endpoint`.""" + root: NotRequired[str | os.PathLike[str]] + """Root within the bucket. All operations happen under this root. The default is `/`.""" + secret_access_key: NotRequired[str] + """Secret access key. Set this field together with `access_key_id`.""" + session_token: NotRequired[str] + """Session token for temporary credentials. This field requires `access_key_id` and `secret_access_key`.""" + + class RedbConfig(TypedDict): """Configuration for the `redb` service.""" @@ -1010,6 +1056,7 @@ class YandexDiskConfig(TypedDict): | MemcachedConfig | MemoryConfig | MiniMokaConfig + | MinioConfig | MokaConfig | MongodbConfig | MysqlConfig @@ -1018,6 +1065,7 @@ class YandexDiskConfig(TypedDict): | OssConfig | PersyConfig | PostgresqlConfig + | R2Config | RedbConfig | RedisConfig | S3Config @@ -1064,6 +1112,7 @@ class YandexDiskConfig(TypedDict): "MemcachedConfig", "MemoryConfig", "MiniMokaConfig", + "MinioConfig", "MokaConfig", "MongodbConfig", "MysqlConfig", @@ -1072,6 +1121,7 @@ class YandexDiskConfig(TypedDict): "OssConfig", "PersyConfig", "PostgresqlConfig", + "R2Config", "RedbConfig", "RedisConfig", "S3Config", diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs index a26e498dd245..413106920600 100644 --- a/bindings/python/src/services.rs +++ b/bindings/python/src/services.rs @@ -81,6 +81,8 @@ pub enum Scheme { Memory, #[cfg(feature = "services-mini-moka")] MiniMoka, + #[cfg(feature = "services-minio")] + Minio, #[cfg(feature = "services-moka")] Moka, #[cfg(feature = "services-mongodb")] @@ -97,6 +99,8 @@ pub enum Scheme { Persy, #[cfg(feature = "services-postgresql")] Postgresql, + #[cfg(feature = "services-r2")] + R2, #[cfg(feature = "services-redb")] Redb, #[cfg(feature = "services-redis")] @@ -221,6 +225,8 @@ impl_enum_to_str!( Memory => "memory", #[cfg(feature = "services-mini-moka")] MiniMoka => "mini-moka", + #[cfg(feature = "services-minio")] + Minio => "minio", #[cfg(feature = "services-moka")] Moka => "moka", #[cfg(feature = "services-mongodb")] @@ -237,6 +243,8 @@ impl_enum_to_str!( Persy => "persy", #[cfg(feature = "services-postgresql")] Postgresql => "postgresql", + #[cfg(feature = "services-r2")] + R2 => "r2", #[cfg(feature = "services-redb")] Redb => "redb", #[cfg(feature = "services-redis")] diff --git a/bindings/ruby/Cargo.toml b/bindings/ruby/Cargo.toml index 21dc44f215f0..f8a48993e39b 100644 --- a/bindings/ruby/Cargo.toml +++ b/bindings/ruby/Cargo.toml @@ -52,8 +52,10 @@ opendal = { version = ">=0", path = "../../core", features = [ "services-http", "services-ipmfs", "services-memory", + "services-minio", "services-obs", "services-oss", + "services-r2", "services-s3", "services-webdav", "services-webhdfs", diff --git a/core/Cargo.toml b/core/Cargo.toml index d7d686e71ff3..5443c74503c5 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -186,6 +186,7 @@ services-memcached = ["dep:opendal-service-memcached"] # Deprecated: memory service is always enabled. services-memory = ["opendal-core/services-memory"] services-mini-moka = ["dep:opendal-service-mini-moka"] +services-minio = ["dep:opendal-service-s3"] services-moka = ["dep:opendal-service-moka"] services-mongodb = ["dep:opendal-service-mongodb"] services-monoiofs = ["dep:opendal-service-monoiofs"] @@ -197,6 +198,7 @@ services-oss = ["dep:opendal-service-oss"] services-pcloud = ["dep:opendal-service-pcloud"] services-persy = ["dep:opendal-service-persy"] services-postgresql = ["dep:opendal-service-postgresql"] +services-r2 = ["dep:opendal-service-s3"] services-redb = ["dep:opendal-service-redb"] services-redis = ["dep:opendal-service-redis", "opendal-service-redis?/rustls"] services-redis-native-tls = [ diff --git a/core/core/src/docs/rfcs/7948_s3_provider_presets.md b/core/core/src/docs/rfcs/7948_s3_provider_presets.md index e64f69291a21..b07f27e9070f 100644 --- a/core/core/src/docs/rfcs/7948_s3_provider_presets.md +++ b/core/core/src/docs/rfcs/7948_s3_provider_presets.md @@ -1,7 +1,7 @@ - Proposal Name: `s3_provider_presets` - Start Date: 2026-07-24 - RFC PR: [apache/opendal#7948](https://github.com/apache/opendal/pull/7948) -- Tracking Issue: [apache/opendal#0000](https://github.com/apache/opendal/issues/0000) +- Tracking Issue: [apache/opendal#8003](https://github.com/apache/opendal/issues/8003) # Summary @@ -9,6 +9,10 @@ Add first-class `r2` and `minio` services to `opendal-service-s3`. Each service has its own URI scheme, config type, and builder. Its config exposes only the connection, location, and authentication fields supported by that provider. +The `opendal` facade exposes `services-r2` and `services-minio` features while +all providers continue to share the `opendal-service-s3` implementation crate. +The existing `services-s3` feature continues to enable only generic S3. + The provider builder validates those fields, converts them into an internal `S3Config`, and delegates requests to the existing S3 implementation. Existing `s3` construction and behavior remain unchanged. @@ -87,18 +91,33 @@ the complete S3 configuration surface continue to use `s3`. The `opendal-service-s3` crate exports `R2`, `R2Config`, `R2_SCHEME`, `Minio`, `MinioConfig`, and `MINIO_SCHEME` alongside `S3`, `S3Config`, and `S3_SCHEME`. -Enabling `services-s3` registers all three schemes: +The facade features select the public builders and URI registrations: -```rust,ignore -registry.register::(S3_SCHEME); -registry.register::(R2_SCHEME); -registry.register::(MINIO_SCHEME); -``` +- `services-r2` exports `R2` and `R2Config` and registers `r2`. +- `services-minio` exports `Minio` and `MinioConfig` and registers `minio`. +- `services-s3` exports `S3` and `S3Config` and registers only `s3`. + +All three facade features activate the same optional `opendal-service-s3` +dependency. Provider presets do not create additional crates. A compatible +provider receives a facade feature only when OpenDAL gives it a dedicated +config type, builder, and URI scheme. Each provider implements the existing `Configurator` and `Builder` contracts. `Operator::from_uri`, `Operator::via_iter`, and `Operator::from_config` therefore work without changes to `OperatorRegistry`. +## Documentation ownership + +The provider config types and builder documentation are the source of truth for +R2 and MinIO setup. Generated service and binding references read those config +types, so their field lists stay aligned with the deserialization contract. + +`compatible_services.md` remains the long-tail guide for using the generic +`S3` service with providers that do not have a built-in preset, and for users +who intentionally need the S3 escape hatch. When OpenDAL adds a preset, that +guide replaces the provider's manual configuration recipe with a link to the +preset instead of maintaining both narratives. + ## Provider configuration Provider config types are independent structs. They do not embed or flatten @@ -204,8 +223,10 @@ Provider validation includes: # Compatibility and migration -This proposal adds schemes and types without changing `s3`. OpenDAL never -reinterprets an existing S3 endpoint as a provider preset. +This proposal adds schemes, types, and provider features without changing +`s3`. Existing `services-s3` users retain generic S3 without implicitly +enabling provider presets. OpenDAL never reinterprets an existing S3 endpoint +as a provider preset. An application can migrate by changing `s3` to `r2` or `minio`, removing AWS-only options, and supplying the provider's required fields. Applications diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml index ecf4cea1c059..820408caa190 100644 --- a/core/fuzz/Cargo.toml +++ b/core/fuzz/Cargo.toml @@ -57,6 +57,7 @@ services-ipmfs = ["opendal/services-ipmfs"] services-memcached = ["opendal/services-memcached"] services-memory = ["opendal/services-memory"] services-mini-moka = ["opendal/services-mini-moka"] +services-minio = ["opendal/services-minio"] services-moka = ["opendal/services-moka"] services-mongodb = ["opendal/services-mongodb"] services-mysql = ["opendal/services-mysql"] @@ -65,6 +66,7 @@ services-onedrive = ["opendal/services-onedrive"] services-oss = ["opendal/services-oss"] services-persy = ["opendal/services-persy"] services-postgresql = ["opendal/services-postgresql"] +services-r2 = ["opendal/services-r2"] services-redb = ["opendal/services-redb"] services-redis = ["opendal/services-redis"] services-rocksdb = ["opendal/services-rocksdb"] diff --git a/core/services/s3/README.md b/core/services/s3/README.md index 611e62a58ca4..449ef2cd217f 100644 --- a/core/services/s3/README.md +++ b/core/services/s3/README.md @@ -1,19 +1,26 @@ # Apache OpenDAL™ Amazon S3 Service `opendal-service-s3` provides access to Amazon S3 and S3-compatible object storage for -applications built with Apache OpenDAL™. +applications built with Apache OpenDAL™. It includes provider presets for Cloudflare R2 +and MinIO. ## Use through `opendal` -Applications should normally enable this service through the `opendal` facade with the -`services-s3` feature: +Applications should enable the service or provider they use through its +matching `opendal` facade feature: ```shell +# Generic S3 cargo add opendal --features services-s3 +# Cloudflare R2 +cargo add opendal --features services-r2 +# MinIO +cargo add opendal --features services-minio ``` -The service is available as `opendal::services::S3`. Configure the -service builder, then pass it to `opendal::Operator::new`. +The matching builders are available as `opendal::services::S3`, +`opendal::services::R2`, and `opendal::services::Minio`. Configure a builder, +then pass it to `opendal::Operator::new`. ## Use with `opendal-core` @@ -38,8 +45,10 @@ fn register_for_uri() { } ``` -`register_for_uri` is only needed for scheme-driven construction through -`Operator::from_uri` or `Operator::via_iter`. +`register_for_uri` registers only the `s3` scheme. Use +`register_r2_service` or `register_minio_service` for the corresponding +provider scheme. Registration is only needed for scheme-driven construction +through `Operator::from_uri` or `Operator::via_iter`. Services that send HTTP requests also require an HTTP transport in `OperationContext`. See the @@ -48,6 +57,8 @@ Services that send HTTP requests also require an HTTP transport in ## Documentation - [Service configuration and examples](https://opendal.apache.org/services/s3) +- [Cloudflare R2 preset](https://opendal.apache.org/services/r2) +- [MinIO preset](https://opendal.apache.org/services/minio) - [Rust API documentation](https://docs.rs/opendal-service-s3) - [Apache OpenDAL documentation](https://opendal.apache.org/docs/) diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs index 9d0869789371..e032d3ee19ae 100644 --- a/core/services/s3/src/backend.rs +++ b/core/services/s3/src/backend.rs @@ -725,6 +725,26 @@ impl Builder for S3Builder { type Config = S3Config; fn build(self) -> Result { + self.build_inner(S3_SCHEME) + } +} + +impl S3Builder { + pub(crate) fn from_provider_config( + config: S3Config, + credential_providers: ProvideCredentialChain, + ) -> Self { + Self { + config, + credential_providers: Some(credential_providers), + } + } + + pub(crate) fn build_with_scheme(self, scheme: &'static str) -> Result { + self.build_inner(scheme) + } + + fn build_inner(self, scheme: &'static str) -> Result { debug!("backend build started: {:?}", self); let S3Builder { @@ -746,7 +766,7 @@ impl Builder for S3Builder { } else { Err( Error::new(ErrorKind::ConfigInvalid, "The bucket is misconfigured") - .with_context("service", S3_SCHEME), + .with_context("service", scheme), ) }?; debug!("backend use bucket {}", bucket); @@ -822,7 +842,7 @@ impl Builder for S3Builder { "region is missing. Please find it by S3::detect_region() or set them in env.", ) .with_operation("Builder::build") - .with_context("service", S3_SCHEME) + .with_context("service", scheme) })? }; debug!("backend use region: {region}"); @@ -847,7 +867,9 @@ impl Builder for S3Builder { // operation. let ctx = Context::new().with_file_read(TokioFileRead).with_env(OsEnv); - let mut provider = { + let provider = if let Some(credential_providers) = credential_providers { + credential_providers + } else { let mut builder = DefaultCredentialProvider::builder(); if config.disable_config_load { @@ -858,53 +880,50 @@ impl Builder for S3Builder { builder = builder.no_imds(); } - ProvideCredentialChain::new().push(builder.build()) - }; - - // Insert static key if user provided. - if let (Some(ak), Some(sk)) = (&config.access_key_id, &config.secret_access_key) { - let static_provider = if let Some(token) = config.session_token.as_deref() { - StaticCredentialProvider::new(ak, sk).with_session_token(token) - } else { - StaticCredentialProvider::new(ak, sk) - }; - provider = provider.push_front(static_provider); - } + let mut provider = ProvideCredentialChain::new().push(builder.build()); - // Insert assume role provider if user provided. - if let Some(role_arn) = &config.role_arn { - // The assume-role provider owns its STS signer, so give it a - // concrete HTTP sender instead of relying on a future operation - // context. - let sts_ctx = ctx.clone().with_http_send(HttpTransporter::default()); - let sts_request_signer = AwsV4Signer::new("sts", ®ion); - let sts_signer = Signer::new(sts_ctx, provider, sts_request_signer); - let mut assume_role_provider = - AssumeRoleCredentialProvider::new(role_arn.clone(), sts_signer) - .with_region(region.clone()) - .with_regional_sts_endpoint(); - - if let Some(external_id) = &config.external_id { - assume_role_provider = assume_role_provider.with_external_id(external_id.clone()); - } - if let Some(role_session_name) = &config.role_session_name { - assume_role_provider = - assume_role_provider.with_role_session_name(role_session_name.clone()); - } - if let Some(duration_seconds) = config.assume_role_duration_seconds { - assume_role_provider = assume_role_provider.with_duration_seconds(duration_seconds); + // Insert static key if user provided. + if let (Some(ak), Some(sk)) = (&config.access_key_id, &config.secret_access_key) { + let static_provider = if let Some(token) = config.session_token.as_deref() { + StaticCredentialProvider::new(ak, sk).with_session_token(token) + } else { + StaticCredentialProvider::new(ak, sk) + }; + provider = provider.push_front(static_provider); } - if let Some(tags) = &config.assume_role_session_tags { - assume_role_provider = assume_role_provider - .with_tags(tags.iter().map(|(k, v)| (k.clone(), v.clone())).collect()); + + // Insert assume role provider if user provided. + if let Some(role_arn) = &config.role_arn { + // The assume-role provider owns its STS signer, so give it a + // concrete HTTP sender instead of relying on a future operation + // context. + let sts_ctx = ctx.clone().with_http_send(HttpTransporter::default()); + let sts_request_signer = AwsV4Signer::new("sts", ®ion); + let sts_signer = Signer::new(sts_ctx, provider, sts_request_signer); + let mut assume_role_provider = + AssumeRoleCredentialProvider::new(role_arn.clone(), sts_signer) + .with_region(region.clone()) + .with_regional_sts_endpoint(); + + if let Some(external_id) = &config.external_id { + assume_role_provider = + assume_role_provider.with_external_id(external_id.clone()); + } + if let Some(role_session_name) = &config.role_session_name { + assume_role_provider = + assume_role_provider.with_role_session_name(role_session_name.clone()); + } + if let Some(duration_seconds) = config.assume_role_duration_seconds { + assume_role_provider = + assume_role_provider.with_duration_seconds(duration_seconds); + } + if let Some(tags) = &config.assume_role_session_tags { + assume_role_provider = assume_role_provider + .with_tags(tags.iter().map(|(k, v)| (k.clone(), v.clone())).collect()); + } + provider = ProvideCredentialChain::new().push(assume_role_provider); } - provider = ProvideCredentialChain::new().push(assume_role_provider); - } - // Replace provider if user provide their own. - let provider = if let Some(credential_providers) = credential_providers { - credential_providers - } else { provider }; @@ -916,7 +935,7 @@ impl Builder for S3Builder { Ok(S3Backend { core: Arc::new(S3Core { - info: ServiceInfo::new(S3_SCHEME, &root, bucket), + info: ServiceInfo::new(scheme, &root, bucket), capability: Capability { stat: true, stat_with_if_match: true, diff --git a/core/services/s3/src/compatible_services.md b/core/services/s3/src/compatible_services.md index 328c13d572e2..9238372f6b86 100644 --- a/core/services/s3/src/compatible_services.md +++ b/core/services/s3/src/compatible_services.md @@ -1,6 +1,15 @@ - ## Compatible Services +OpenDAL provides provider presets for services with a maintained, narrow +configuration contract: + +- Use [`crate::R2`] or the `r2` scheme for Cloudflare R2. +- Use [`crate::Minio`] or the `minio` scheme for MinIO deployments. + +The presets own provider defaults and validation. Use this generic [`crate::S3`] +builder for AWS S3, for compatible services without a built-in preset, or when +an application requires S3 options outside a preset's contract. + ### AWS S3 [AWS S3](https://aws.amazon.com/s3/) is the default implementations of s3 services. Only `bucket` is required. @@ -41,22 +50,6 @@ builder.bucket(""); builder.enable_virtual_host_style(); ``` -### Minio - -[minio](https://min.io/) is an open-source s3 compatible services. - -To connect to minio, we need to set: - -- `endpoint`: The endpoint of minio, for example: `http://127.0.0.1:9000` -- `region`: The region of minio. If you don't care about it, just set it to "auto", it will be ignored. -- `bucket`: The bucket name of minio. - -```rust,ignore -builder.endpoint("http://127.0.0.1:9000"); -builder.region(""); -builder.bucket(""); -``` - ### QingStor Object Storage [QingStor Object Storage](https://www.qingcloud.com/products/qingstor) is a S3-compatible service provided by [QingCloud](https://www.qingcloud.com/). @@ -122,25 +115,6 @@ To connect to wasabi, we need to set: > Refer to [What are the service URLs for Wasabi's different storage regions?](https://wasabi-support.zendesk.com/hc/en-us/articles/360015106031) for more details. -### Cloudflare R2 - -[Cloudflare R2](https://developers.cloudflare.com/r2/) provides s3 compatible API. - -> Cloudflare R2 Storage allows developers to store large amounts of unstructured data without the costly egress bandwidth fees associated with typical cloud storage services. - - -To connect to r2, we need to set: - -- `endpoint`: The endpoint of r2, for example: `https://.r2.cloudflarestorage.com` -- `bucket`: The bucket name of r2. -- `region`: When you create a new bucket, the data location is set to Automatic by default. So please use `auto` for region. -- `enable_exact_buf_write`: R2 requires the non-tailing parts size to be exactly the same. Please enable this option to avoid the error `All non-trailing parts must have the same length`. - -R2 has the following capability differences from S3: - -- `delete_max_size`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please override `delete_max_size` to `700`. -- `stat_with_override_cache_control`, `stat_with_override_content_disposition`, `stat_with_override_content_type`: R2 doesn't support stat with response override queries. Please override them to `false`. - ### Google Cloud Storage XML API [Google Cloud Storage XML API](https://cloud.google.com/storage/docs/xml-api/overview) provides s3 compatible API. - `endpoint`: The endpoint of Google Cloud Storage XML API, for example: `https://storage.googleapis.com` diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md index 66dfe5ab00fd..feb4e98af44a 100644 --- a/core/services/s3/src/docs.md +++ b/core/services/s3/src/docs.md @@ -79,7 +79,7 @@ async fn main() -> Result<()> { .root("/path/to/dir") // Set the bucket name. This is required. .bucket("test") - // Set the region. This is required for some services, if you don't care about it, for example Minio service, just set it to "auto", it will be ignored. + // Set the signing region. Compatible services define their own region rules. .region("us-east-1") // Set the endpoint. // diff --git a/core/services/s3/src/lib.rs b/core/services/s3/src/lib.rs index 8e34b7dd12ff..684132a2aa7b 100644 --- a/core/services/s3/src/lib.rs +++ b/core/services/s3/src/lib.rs @@ -26,16 +26,29 @@ mod copier; mod core; mod deleter; mod lister; +mod minio; +mod minio_config; +mod preset; +mod r2; +mod r2_config; mod reader; mod writer; pub use backend::S3Builder as S3; pub use config::S3Config; +pub use minio::MinioBuilder as Minio; +pub use minio_config::MinioConfig; +pub use r2::R2Builder as R2; +pub use r2_config::R2Config; /// URI scheme used for service registration and scheme-driven construction. pub const S3_SCHEME: &str = "s3"; +/// URI scheme for the Cloudflare R2 preset. +pub const R2_SCHEME: &str = "r2"; +/// URI scheme for the MinIO preset. +pub const MINIO_SCHEME: &str = "minio"; -/// Register this service's URI scheme or schemes with an operator registry. +/// Register the Amazon S3 URI scheme with an operator registry. /// /// Registration enables scheme-driven construction through /// [`opendal_core::Operator::from_uri`] and @@ -44,3 +57,54 @@ pub const S3_SCHEME: &str = "s3"; pub fn register_s3_service(registry: &opendal_core::OperatorRegistry) { registry.register::(S3_SCHEME); } + +/// Register the Cloudflare R2 URI scheme with an operator registry. +/// +/// Registration enables scheme-driven construction through +/// [`opendal_core::Operator::from_uri`] and +/// [`opendal_core::Operator::via_iter`]. Direct construction through +/// [`opendal_core::Operator::new`] does not require registration. +pub fn register_r2_service(registry: &opendal_core::OperatorRegistry) { + registry.register::(R2_SCHEME); +} + +/// Register the MinIO URI scheme with an operator registry. +/// +/// Registration enables scheme-driven construction through +/// [`opendal_core::Operator::from_uri`] and +/// [`opendal_core::Operator::via_iter`]. Direct construction through +/// [`opendal_core::Operator::new`] does not require registration. +pub fn register_minio_service(registry: &opendal_core::OperatorRegistry) { + registry.register::(MINIO_SCHEME); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registers_s3_and_provider_schemes() { + let registry = opendal_core::OperatorRegistry::get(); + register_s3_service(registry); + register_r2_service(registry); + register_minio_service(registry); + + let schemes = registry.schemes(); + assert!(schemes.contains(S3_SCHEME)); + assert!(schemes.contains(R2_SCHEME)); + assert!(schemes.contains(MINIO_SCHEME)); + + let r2 = registry + .load(("r2://bucket/root", [("account_id", "example-account")])) + .unwrap(); + assert_eq!(r2.info().scheme(), R2_SCHEME); + + let minio = registry + .load(( + "minio://bucket/root", + [("endpoint", "http://127.0.0.1:9000")], + )) + .unwrap(); + assert_eq!(minio.info().scheme(), MINIO_SCHEME); + } +} diff --git a/core/services/s3/src/minio.md b/core/services/s3/src/minio.md new file mode 100644 index 000000000000..be0cb6e84c87 --- /dev/null +++ b/core/services/s3/src/minio.md @@ -0,0 +1,51 @@ +## Configuration + +Use [`crate::MinioConfig`] for serializable configuration or this builder for +direct construction. The MinIO preset accepts only deployment connection and +credential settings. Use [`crate::S3`] when an application needs the complete +S3 configuration surface. Applications that use the `opendal` facade enable +the `services-minio` feature. + +Every MinIO deployment must provide an `endpoint`. The signing `region` +defaults to `auto`; set it explicitly when the deployment requires a configured +region. + +The preset loads credentials from direct settings, standard AWS environment +variables, or static credentials in the shared AWS credential files. It does +not use AWS SSO, web identity, credential processes, ECS, EC2 metadata, or +AssumeRole. Call [`MinioBuilder::skip_signature`] for a deployment that accepts +anonymous requests. + +## Examples + +Build an operator for a local MinIO deployment: + +```rust +use opendal_core::{Operator, Result}; +use opendal_service_s3::Minio; + +fn build() -> Result { + Operator::new( + Minio::default() + .bucket("data") + .endpoint("http://127.0.0.1:9000") + .access_key_id("minioadmin") + .secret_access_key("minioadmin"), + ) +} +``` + +After registering the service, construct it from a `minio://` URI: + +```rust +use opendal_core::{Operator, OperatorRegistry, Result}; +use opendal_service_s3::register_minio_service; + +fn build() -> Result { + register_minio_service(OperatorRegistry::get()); + Operator::from_uri(( + "minio://data/root", + [("endpoint", "http://127.0.0.1:9000")], + )) +} +``` diff --git a/core/services/s3/src/minio.rs b/core/services/s3/src/minio.rs new file mode 100644 index 000000000000..2422fa6d8a7f --- /dev/null +++ b/core/services/s3/src/minio.rs @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; + +use opendal_core::Builder; +use opendal_core::Result; +use opendal_core::raw::Service; +use reqsign_aws_v4::Credential; +use reqsign_core::ProvideCredentialChain; + +use crate::MINIO_SCHEME; +use crate::backend::S3Builder; +use crate::config::S3Config; +use crate::minio_config::MinioConfig; +use crate::preset; + +/// Builds a MinIO service with provider-specific configuration. +#[doc = include_str!("minio.md")] +#[derive(Default)] +pub struct MinioBuilder { + config: MinioConfig, +} + +impl Debug for MinioBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MinioBuilder") + .field("config", &self.config) + .finish_non_exhaustive() + } +} + +impl MinioBuilder { + pub(crate) fn from_config(config: MinioConfig) -> Self { + Self { config } + } + + /// Set the root within the bucket. + pub fn root(mut self, root: &str) -> Self { + self.config.root = if root.is_empty() { + None + } else { + Some(root.to_string()) + }; + self + } + + /// Set the bucket name. + pub fn bucket(mut self, bucket: &str) -> Self { + self.config.bucket = bucket.to_string(); + self + } + + /// Set the MinIO endpoint. + pub fn endpoint(mut self, endpoint: &str) -> Self { + self.config.endpoint = Some(endpoint.to_string()); + self + } + + /// Set the signing region. + pub fn region(mut self, region: &str) -> Self { + self.config.region = Some(region.to_string()); + self + } + + /// Set the access key ID. + pub fn access_key_id(mut self, access_key_id: &str) -> Self { + self.config.access_key_id = Some(access_key_id.to_string()); + self + } + + /// Set the secret access key. + pub fn secret_access_key(mut self, secret_access_key: &str) -> Self { + self.config.secret_access_key = Some(secret_access_key.to_string()); + self + } + + /// Set the session token for temporary credentials. + pub fn session_token(mut self, session_token: &str) -> Self { + self.config.session_token = Some(session_token.to_string()); + self + } + + /// Send requests without signing them. + pub fn skip_signature(mut self) -> Self { + self.config.skip_signature = true; + self + } + + fn into_s3_config(self) -> Result<(S3Config, ProvideCredentialChain)> { + let MinioConfig { + root, + bucket, + endpoint, + region, + access_key_id, + secret_access_key, + session_token, + skip_signature, + } = self.config; + + preset::validate_required(&bucket, MINIO_SCHEME, "bucket must not be empty")?; + preset::validate_optional( + endpoint.as_deref(), + MINIO_SCHEME, + "endpoint must not be empty when set", + )?; + preset::validate_optional( + region.as_deref(), + MINIO_SCHEME, + "region must not be empty when set", + )?; + + let endpoint = endpoint + .ok_or_else(|| preset::config_error(MINIO_SCHEME, "endpoint is required for MinIO"))?; + + if skip_signature + && (access_key_id.is_some() || secret_access_key.is_some() || session_token.is_some()) + { + return Err(preset::config_error( + MINIO_SCHEME, + "skip_signature cannot be combined with direct credentials", + )); + } + + preset::validate_credentials( + access_key_id.as_deref(), + secret_access_key.as_deref(), + session_token.as_deref(), + MINIO_SCHEME, + )?; + + let credential_providers = preset::credential_chain( + access_key_id.as_deref(), + secret_access_key.as_deref(), + session_token.as_deref(), + ); + + let config = S3Config { + root, + bucket, + endpoint: Some(endpoint), + region: Some(region.unwrap_or_else(|| "auto".to_string())), + access_key_id, + secret_access_key, + session_token, + disable_ec2_metadata: true, + skip_signature, + ..Default::default() + }; + + Ok((config, credential_providers)) + } +} + +impl Builder for MinioBuilder { + type Config = MinioConfig; + + fn build(self) -> Result { + let (config, credential_providers) = self.into_s3_config()?; + S3Builder::from_provider_config(config, credential_providers) + .build_with_scheme(MINIO_SCHEME) + } +} + +#[cfg(test)] +mod tests { + use opendal_core::ErrorKind; + use opendal_core::Operator; + + use super::*; + + fn build_config(builder: MinioBuilder) -> S3Config { + builder.into_s3_config().unwrap().0 + } + + #[test] + fn defaults_region_to_auto() { + let config = build_config( + MinioBuilder::default() + .bucket("bucket") + .endpoint("http://127.0.0.1:9000"), + ); + assert_eq!(config.region.as_deref(), Some("auto")); + } + + #[test] + fn accepts_explicit_region() { + let config = build_config( + MinioBuilder::default() + .bucket("bucket") + .endpoint("http://127.0.0.1:9000") + .region("us-east-1"), + ); + assert_eq!(config.region.as_deref(), Some("us-east-1")); + } + + #[test] + fn requires_endpoint() { + let err = MinioBuilder::default() + .bucket("bucket") + .into_s3_config() + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn rejects_incomplete_credentials() { + let err = MinioBuilder::default() + .bucket("bucket") + .endpoint("http://127.0.0.1:9000") + .session_token("token") + .into_s3_config() + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn rejects_credentials_in_anonymous_mode() { + let err = MinioBuilder::default() + .bucket("bucket") + .endpoint("http://127.0.0.1:9000") + .access_key_id("access") + .secret_access_key("secret") + .skip_signature() + .into_s3_config() + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn reports_minio_scheme() { + let operator = Operator::new( + MinioBuilder::default() + .bucket("bucket") + .endpoint("http://127.0.0.1:9000") + .access_key_id("access") + .secret_access_key("secret"), + ) + .unwrap(); + assert_eq!(operator.info().scheme(), MINIO_SCHEME); + } +} diff --git a/core/services/s3/src/minio_config.rs b/core/services/s3/src/minio_config.rs new file mode 100644 index 000000000000..3053f8f5d0c8 --- /dev/null +++ b/core/services/s3/src/minio_config.rs @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; + +use opendal_core::Configurator; +use opendal_core::OperatorUri; +use opendal_core::Result; +use serde::Deserialize; +use serde::Serialize; + +use crate::minio::MinioBuilder; +use crate::preset; + +/// Configuration for a MinIO deployment. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +#[non_exhaustive] +pub struct MinioConfig { + /// Root within the bucket. + /// + /// All operations happen under this root. The default is `/`. + /// + /// + /// + pub root: Option, + /// Bucket name. + /// + /// This field is required. + /// + /// + /// + pub bucket: String, + /// MinIO endpoint. + /// + /// This field is required because MinIO deployments do not share a + /// universal endpoint. + /// + /// + /// + /// + /// + pub endpoint: Option, + /// Signing region. + /// + /// The default is `auto`. Set this field when the deployment requires a + /// configured region. + /// + /// + /// + pub region: Option, + /// Access key ID. + /// + /// Set this field together with `secret_access_key`. + /// + /// + pub access_key_id: Option, + /// Secret access key. + /// + /// Set this field together with `access_key_id`. + /// + /// + pub secret_access_key: Option, + /// Session token for temporary credentials. + /// + /// This field requires `access_key_id` and `secret_access_key`. + /// + /// + pub session_token: Option, + /// Send requests without signing them. + /// + /// This option cannot be combined with direct credentials. + /// + /// + /// + pub skip_signature: bool, +} + +impl Debug for MinioConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MinioConfig") + .field("root", &self.root) + .field("bucket", &self.bucket) + .field("endpoint", &self.endpoint) + .field("region", &self.region) + .field("skip_signature", &self.skip_signature) + .finish_non_exhaustive() + } +} + +impl Configurator for MinioConfig { + type Builder = MinioBuilder; + + fn from_uri(uri: &OperatorUri) -> Result { + preset::from_uri(uri) + } + + fn into_builder(self) -> Self::Builder { + MinioBuilder::from_config(self) + } +} + +#[cfg(test)] +mod tests { + use std::iter; + + use opendal_core::ErrorKind; + + use super::*; + + #[test] + fn from_uri_extracts_bucket_root_and_options() { + let uri = OperatorUri::new( + "minio://example-bucket/path/to/root?endpoint=http%3A%2F%2F127.0.0.1%3A9000", + iter::empty(), + ) + .unwrap(); + let config = MinioConfig::from_uri(&uri).unwrap(); + + assert_eq!(config.bucket, "example-bucket"); + assert_eq!(config.root.as_deref(), Some("path/to/root")); + assert_eq!(config.endpoint.as_deref(), Some("http://127.0.0.1:9000")); + } + + #[test] + fn rejects_unknown_fields() { + let err = + MinioConfig::from_iter([("role_arn".to_string(), "role".to_string())]).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn debug_redacts_credentials() { + let config = MinioConfig { + bucket: "bucket".to_string(), + access_key_id: Some("access-value".to_string()), + secret_access_key: Some("secret-value".to_string()), + session_token: Some("token-value".to_string()), + ..Default::default() + }; + + let output = format!("{config:?}"); + assert!(!output.contains("access-value")); + assert!(!output.contains("secret-value")); + assert!(!output.contains("token-value")); + } +} diff --git a/core/services/s3/src/preset.rs b/core/services/s3/src/preset.rs new file mode 100644 index 000000000000..39f73ebb1c7b --- /dev/null +++ b/core/services/s3/src/preset.rs @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use opendal_core::Configurator; +use opendal_core::Error; +use opendal_core::ErrorKind; +use opendal_core::OperatorUri; +use opendal_core::Result; +use reqsign_aws_v4::Credential; +use reqsign_aws_v4::EnvCredentialProvider; +use reqsign_aws_v4::ProfileCredentialProvider; +use reqsign_aws_v4::StaticCredentialProvider; +use reqsign_core::ProvideCredentialChain; + +pub(crate) fn from_uri(uri: &OperatorUri) -> Result { + let mut map = uri.options().clone(); + + if let Some(name) = uri.name() { + map.insert("bucket".to_string(), name.to_string()); + } + + if let Some(root) = uri.root() { + map.insert("root".to_string(), root.to_string()); + } + + C::from_iter(map) +} + +pub(crate) fn config_error(service: &'static str, message: &'static str) -> Error { + Error::new(ErrorKind::ConfigInvalid, message) + .with_operation("Builder::build") + .with_context("service", service) +} + +pub(crate) fn validate_required( + value: &str, + service: &'static str, + message: &'static str, +) -> Result<()> { + if value.trim().is_empty() { + return Err(config_error(service, message)); + } + Ok(()) +} + +pub(crate) fn validate_optional( + value: Option<&str>, + service: &'static str, + message: &'static str, +) -> Result<()> { + if value.is_some_and(|value| value.trim().is_empty()) { + return Err(config_error(service, message)); + } + Ok(()) +} + +pub(crate) fn validate_credentials( + access_key_id: Option<&str>, + secret_access_key: Option<&str>, + session_token: Option<&str>, + service: &'static str, +) -> Result<()> { + validate_optional( + access_key_id, + service, + "access_key_id must not be empty when set", + )?; + validate_optional( + secret_access_key, + service, + "secret_access_key must not be empty when set", + )?; + validate_optional( + session_token, + service, + "session_token must not be empty when set", + )?; + + match (access_key_id, secret_access_key, session_token) { + (None, None, None) | (Some(_), Some(_), None | Some(_)) => Ok(()), + _ => Err(config_error( + service, + "access_key_id and secret_access_key must be provided together; session_token requires both", + )), + } +} + +pub(crate) fn credential_chain( + access_key_id: Option<&str>, + secret_access_key: Option<&str>, + session_token: Option<&str>, +) -> ProvideCredentialChain { + let mut chain = ProvideCredentialChain::new() + .push(EnvCredentialProvider::new()) + .push(ProfileCredentialProvider::new()); + + if let (Some(access_key_id), Some(secret_access_key)) = (access_key_id, secret_access_key) { + let provider = if let Some(session_token) = session_token { + StaticCredentialProvider::new(access_key_id, secret_access_key) + .with_session_token(session_token) + } else { + StaticCredentialProvider::new(access_key_id, secret_access_key) + }; + chain = chain.push_front(provider); + } + + chain +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use reqsign_core::Context; + use reqsign_core::ProvideCredential; + use reqsign_core::StaticEnv; + + use super::*; + + #[test] + fn credential_chain_contains_only_declared_sources() { + assert_eq!(credential_chain(None, None, None).len(), 2); + assert_eq!( + credential_chain(Some("access"), Some("secret"), Some("token")).len(), + 3 + ); + } + + #[tokio::test] + async fn direct_credentials_take_priority_over_environment() { + let context = Context::new().with_env(StaticEnv { + home_dir: None, + envs: HashMap::from([ + ("AWS_ACCESS_KEY_ID".to_string(), "env-access".to_string()), + ( + "AWS_SECRET_ACCESS_KEY".to_string(), + "env-secret".to_string(), + ), + ]), + }); + let chain = credential_chain(Some("direct-access"), Some("direct-secret"), None); + + let credential = chain.provide_credential(&context).await.unwrap().unwrap(); + assert_eq!(credential.access_key_id, "direct-access"); + assert_eq!(credential.secret_access_key, "direct-secret"); + } + + #[tokio::test] + async fn loads_credentials_from_standard_environment() { + let context = Context::new().with_env(StaticEnv { + home_dir: None, + envs: HashMap::from([ + ("AWS_ACCESS_KEY_ID".to_string(), "env-access".to_string()), + ( + "AWS_SECRET_ACCESS_KEY".to_string(), + "env-secret".to_string(), + ), + ]), + }); + let chain = credential_chain(None, None, None); + + let credential = chain.provide_credential(&context).await.unwrap().unwrap(); + assert_eq!(credential.access_key_id, "env-access"); + assert_eq!(credential.secret_access_key, "env-secret"); + } +} diff --git a/core/services/s3/src/r2.md b/core/services/s3/src/r2.md new file mode 100644 index 000000000000..e099f0d2c4a6 --- /dev/null +++ b/core/services/s3/src/r2.md @@ -0,0 +1,53 @@ +## Configuration + +Use [`crate::R2Config`] for serializable configuration or this builder for +direct construction. The R2 preset accepts only R2 connection and credential +settings. Use [`crate::S3`] when an application needs the complete S3 +configuration surface. Applications that use the `opendal` facade enable the +`services-r2` feature. + +Set exactly one endpoint source: + +- Set `account_id` to derive + `https://.r2.cloudflarestorage.com`. +- Set `account_id` and `jurisdiction` to derive a jurisdictional endpoint. +- Set `endpoint` for a proxy, gateway, or test server. + +The preset uses `auto` as the signing region. It loads credentials from direct +settings, standard AWS environment variables, or static credentials in the +shared AWS credential files. It does not use AWS SSO, web identity, credential +processes, ECS, EC2 metadata, or AssumeRole. + +## Examples + +Build an operator from an account ID: + +```rust +use opendal_core::{Operator, Result}; +use opendal_service_s3::R2; + +fn build() -> Result { + Operator::new( + R2::default() + .bucket("data") + .account_id("example-account") + .access_key_id("example-access-key") + .secret_access_key("example-secret-key"), + ) +} +``` + +After registering the service, construct it from an `r2://` URI: + +```rust +use opendal_core::{Operator, OperatorRegistry, Result}; +use opendal_service_s3::register_r2_service; + +fn build() -> Result { + register_r2_service(OperatorRegistry::get()); + Operator::from_uri(( + "r2://data/root", + [("account_id", "example-account")], + )) +} +``` diff --git a/core/services/s3/src/r2.rs b/core/services/s3/src/r2.rs new file mode 100644 index 000000000000..a77ab3d44234 --- /dev/null +++ b/core/services/s3/src/r2.rs @@ -0,0 +1,298 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; + +use opendal_core::Builder; +use opendal_core::Result; +use opendal_core::raw::Service; +use reqsign_aws_v4::Credential; +use reqsign_core::ProvideCredentialChain; + +use crate::R2_SCHEME; +use crate::backend::S3Builder; +use crate::config::S3Config; +use crate::preset; +use crate::r2_config::R2Config; + +/// Builds a Cloudflare R2 service with provider-specific configuration. +#[doc = include_str!("r2.md")] +#[derive(Default)] +pub struct R2Builder { + config: R2Config, +} + +impl Debug for R2Builder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("R2Builder") + .field("config", &self.config) + .finish_non_exhaustive() + } +} + +impl R2Builder { + pub(crate) fn from_config(config: R2Config) -> Self { + Self { config } + } + + /// Set the root within the bucket. + pub fn root(mut self, root: &str) -> Self { + self.config.root = if root.is_empty() { + None + } else { + Some(root.to_string()) + }; + self + } + + /// Set the bucket name. + pub fn bucket(mut self, bucket: &str) -> Self { + self.config.bucket = bucket.to_string(); + self + } + + /// Set the Cloudflare account ID used to derive the endpoint. + pub fn account_id(mut self, account_id: &str) -> Self { + self.config.account_id = Some(account_id.to_string()); + self + } + + /// Set the R2 jurisdiction to `eu` or `fedramp`. + pub fn jurisdiction(mut self, jurisdiction: &str) -> Self { + self.config.jurisdiction = Some(jurisdiction.to_string()); + self + } + + /// Set an explicit R2-compatible endpoint. + pub fn endpoint(mut self, endpoint: &str) -> Self { + self.config.endpoint = Some(endpoint.to_string()); + self + } + + /// Set the access key ID. + pub fn access_key_id(mut self, access_key_id: &str) -> Self { + self.config.access_key_id = Some(access_key_id.to_string()); + self + } + + /// Set the secret access key. + pub fn secret_access_key(mut self, secret_access_key: &str) -> Self { + self.config.secret_access_key = Some(secret_access_key.to_string()); + self + } + + /// Set the session token for temporary credentials. + pub fn session_token(mut self, session_token: &str) -> Self { + self.config.session_token = Some(session_token.to_string()); + self + } + + fn into_s3_config(self) -> Result<(S3Config, ProvideCredentialChain)> { + let R2Config { + root, + bucket, + account_id, + jurisdiction, + endpoint, + access_key_id, + secret_access_key, + session_token, + } = self.config; + + preset::validate_required(&bucket, R2_SCHEME, "bucket must not be empty")?; + preset::validate_optional( + account_id.as_deref(), + R2_SCHEME, + "account_id must not be empty when set", + )?; + preset::validate_optional( + jurisdiction.as_deref(), + R2_SCHEME, + "jurisdiction must not be empty when set", + )?; + preset::validate_optional( + endpoint.as_deref(), + R2_SCHEME, + "endpoint must not be empty when set", + )?; + preset::validate_credentials( + access_key_id.as_deref(), + secret_access_key.as_deref(), + session_token.as_deref(), + R2_SCHEME, + )?; + + let endpoint = match (account_id.as_deref(), endpoint.as_deref()) { + (Some(account_id), None) => match jurisdiction.as_deref() { + None => format!("https://{account_id}.r2.cloudflarestorage.com"), + Some(jurisdiction @ ("eu" | "fedramp")) => { + format!("https://{account_id}.{jurisdiction}.r2.cloudflarestorage.com") + } + Some(_) => { + return Err(preset::config_error( + R2_SCHEME, + "jurisdiction must be either eu or fedramp", + )); + } + }, + (None, Some(endpoint)) if jurisdiction.is_none() => endpoint.to_string(), + (None, Some(_)) => { + return Err(preset::config_error( + R2_SCHEME, + "jurisdiction requires account_id and cannot be used with endpoint", + )); + } + (Some(_), Some(_)) => { + return Err(preset::config_error( + R2_SCHEME, + "account_id and endpoint are mutually exclusive", + )); + } + (None, None) => { + return Err(preset::config_error( + R2_SCHEME, + "exactly one of account_id and endpoint is required", + )); + } + }; + + let credential_providers = preset::credential_chain( + access_key_id.as_deref(), + secret_access_key.as_deref(), + session_token.as_deref(), + ); + + let config = S3Config { + root, + bucket, + endpoint: Some(endpoint), + region: Some("auto".to_string()), + access_key_id, + secret_access_key, + session_token, + disable_ec2_metadata: true, + ..Default::default() + }; + + Ok((config, credential_providers)) + } +} + +impl Builder for R2Builder { + type Config = R2Config; + + fn build(self) -> Result { + let (config, credential_providers) = self.into_s3_config()?; + S3Builder::from_provider_config(config, credential_providers).build_with_scheme(R2_SCHEME) + } +} + +#[cfg(test)] +mod tests { + use opendal_core::ErrorKind; + use opendal_core::Operator; + + use super::*; + + fn build_config(builder: R2Builder) -> S3Config { + builder.into_s3_config().unwrap().0 + } + + #[test] + fn derives_default_endpoint() { + let config = build_config(R2Builder::default().bucket("bucket").account_id("account")); + assert_eq!( + config.endpoint.as_deref(), + Some("https://account.r2.cloudflarestorage.com") + ); + assert_eq!(config.region.as_deref(), Some("auto")); + } + + #[test] + fn derives_jurisdiction_endpoints() { + for jurisdiction in ["eu", "fedramp"] { + let config = build_config( + R2Builder::default() + .bucket("bucket") + .account_id("account") + .jurisdiction(jurisdiction), + ); + let expected = format!("https://account.{jurisdiction}.r2.cloudflarestorage.com"); + assert_eq!(config.endpoint.as_deref(), Some(expected.as_str())); + } + } + + #[test] + fn accepts_explicit_endpoint() { + let config = build_config( + R2Builder::default() + .bucket("bucket") + .endpoint("http://127.0.0.1:9000"), + ); + assert_eq!(config.endpoint.as_deref(), Some("http://127.0.0.1:9000")); + } + + #[test] + fn rejects_invalid_endpoint_and_jurisdiction_combinations() { + let cases = [ + R2Builder::default().bucket("bucket"), + R2Builder::default() + .bucket("bucket") + .account_id("account") + .endpoint("https://example.com"), + R2Builder::default() + .bucket("bucket") + .endpoint("https://example.com") + .jurisdiction("eu"), + R2Builder::default() + .bucket("bucket") + .account_id("account") + .jurisdiction("invalid"), + ]; + + for builder in cases { + assert_eq!( + builder.into_s3_config().unwrap_err().kind(), + ErrorKind::ConfigInvalid + ); + } + } + + #[test] + fn rejects_incomplete_credentials() { + let err = R2Builder::default() + .bucket("bucket") + .account_id("account") + .session_token("token") + .into_s3_config() + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn reports_r2_scheme() { + let operator = Operator::new( + R2Builder::default() + .bucket("bucket") + .account_id("account") + .access_key_id("access") + .secret_access_key("secret"), + ) + .unwrap(); + assert_eq!(operator.info().scheme(), R2_SCHEME); + } +} diff --git a/core/services/s3/src/r2_config.rs b/core/services/s3/src/r2_config.rs new file mode 100644 index 000000000000..95b924a7a57d --- /dev/null +++ b/core/services/s3/src/r2_config.rs @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::Debug; + +use opendal_core::Configurator; +use opendal_core::OperatorUri; +use opendal_core::Result; +use serde::Deserialize; +use serde::Serialize; + +use crate::preset; +use crate::r2::R2Builder; + +/// Configuration for Cloudflare R2. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default, deny_unknown_fields)] +#[non_exhaustive] +pub struct R2Config { + /// Root within the bucket. + /// + /// All operations happen under this root. The default is `/`. + /// + /// + /// + pub root: Option, + /// Bucket name. + /// + /// This field is required. + /// + /// + /// + pub bucket: String, + /// Cloudflare account ID used to derive the R2 endpoint. + /// + /// Set exactly one of `account_id` and `endpoint`. + /// + /// + /// + /// + pub account_id: Option, + /// R2 jurisdiction. + /// + /// Supported values are `eu` and `fedramp`. This field requires + /// `account_id` and cannot be used with `endpoint`. + /// + /// + pub jurisdiction: Option, + /// Explicit R2-compatible endpoint. + /// + /// Use this field for a proxy, gateway, or test server. Set exactly one of + /// `endpoint` and `account_id`. + /// + /// + /// + pub endpoint: Option, + /// Access key ID. + /// + /// Set this field together with `secret_access_key`. + /// + /// + pub access_key_id: Option, + /// Secret access key. + /// + /// Set this field together with `access_key_id`. + /// + /// + pub secret_access_key: Option, + /// Session token for temporary credentials. + /// + /// This field requires `access_key_id` and `secret_access_key`. + /// + /// + pub session_token: Option, +} + +impl Debug for R2Config { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("R2Config") + .field("root", &self.root) + .field("bucket", &self.bucket) + .field("account_id", &self.account_id) + .field("jurisdiction", &self.jurisdiction) + .field("endpoint", &self.endpoint) + .finish_non_exhaustive() + } +} + +impl Configurator for R2Config { + type Builder = R2Builder; + + fn from_uri(uri: &OperatorUri) -> Result { + preset::from_uri(uri) + } + + fn into_builder(self) -> Self::Builder { + R2Builder::from_config(self) + } +} + +#[cfg(test)] +mod tests { + use std::iter; + + use opendal_core::ErrorKind; + + use super::*; + + #[test] + fn from_uri_extracts_bucket_root_and_options() { + let uri = OperatorUri::new( + "r2://example-bucket/path/to/root?account_id=example-account", + iter::empty(), + ) + .unwrap(); + let config = R2Config::from_uri(&uri).unwrap(); + + assert_eq!(config.bucket, "example-bucket"); + assert_eq!(config.root.as_deref(), Some("path/to/root")); + assert_eq!(config.account_id.as_deref(), Some("example-account")); + } + + #[test] + fn rejects_unknown_fields() { + let err = R2Config::from_iter([("role_arn".to_string(), "role".to_string())]).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + } + + #[test] + fn debug_redacts_credentials() { + let config = R2Config { + bucket: "bucket".to_string(), + access_key_id: Some("access-value".to_string()), + secret_access_key: Some("secret-value".to_string()), + session_token: Some("token-value".to_string()), + ..Default::default() + }; + + let output = format!("{config:?}"); + assert!(!output.contains("access-value")); + assert!(!output.contains("secret-value")); + assert!(!output.contains("token-value")); + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index 197bf54f5328..21c6ed82776a 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -215,6 +215,12 @@ fn init_default_registry_inner(registry: &OperatorRegistry) { #[cfg(feature = "services-s3")] opendal_service_s3::register_s3_service(registry); + #[cfg(feature = "services-r2")] + opendal_service_s3::register_r2_service(registry); + + #[cfg(feature = "services-minio")] + opendal_service_s3::register_minio_service(registry); + #[cfg(feature = "services-seafile")] opendal_service_seafile::register_seafile_service(registry); @@ -274,7 +280,7 @@ fn register_default_operator_registry() { /// /// [`Memory`](services::Memory) is always available. Other builders are /// re-exported when their matching facade feature is enabled; for example, -/// `services-s3` enables [`S3`](services::S3). +/// `services-r2` enables the `R2` builder. /// /// Pass a configured builder to [`Operator::new`]. The facade automatically /// registers enabled services for [`Operator::from_uri`] and @@ -379,8 +385,12 @@ pub mod services { pub use opendal_service_redis::*; #[cfg(feature = "services-rocksdb")] pub use opendal_service_rocksdb::*; + #[cfg(feature = "services-minio")] + pub use opendal_service_s3::{MINIO_SCHEME, Minio, MinioConfig, register_minio_service}; + #[cfg(feature = "services-r2")] + pub use opendal_service_s3::{R2, R2_SCHEME, R2Config, register_r2_service}; #[cfg(feature = "services-s3")] - pub use opendal_service_s3::*; + pub use opendal_service_s3::{S3, S3_SCHEME, S3Config, register_s3_service}; #[cfg(feature = "services-seafile")] pub use opendal_service_seafile::*; #[cfg(feature = "services-sftp")] @@ -411,6 +421,43 @@ pub mod services { pub use opendal_service_yandex_disk::*; } +#[cfg(test)] +mod service_feature_tests { + use super::*; + + #[test] + fn registers_enabled_s3_provider_schemes() { + init_default_registry(); + let schemes = OperatorRegistry::get().schemes(); + + assert!(schemes.contains("memory")); + + #[cfg(feature = "services-s3")] + { + let _ = services::S3::default(); + assert!(schemes.contains("s3")); + } + #[cfg(not(feature = "services-s3"))] + assert!(!schemes.contains("s3")); + + #[cfg(feature = "services-r2")] + { + let _ = services::R2::default(); + assert!(schemes.contains("r2")); + } + #[cfg(not(feature = "services-r2"))] + assert!(!schemes.contains("r2")); + + #[cfg(feature = "services-minio")] + { + let _ = services::Minio::default(); + assert!(schemes.contains("minio")); + } + #[cfg(not(feature = "services-minio"))] + assert!(!schemes.contains("minio")); + } +} + /// Layers enabled through `layers-*` Cargo features. /// /// A layer adds cross-service behavior to an [`Operator`]. Enable the matching diff --git a/dev/src/generate/docs.rs b/dev/src/generate/docs.rs index 9bcd49ae04e1..0df031607124 100644 --- a/dev/src/generate/docs.rs +++ b/dev/src/generate/docs.rs @@ -26,7 +26,7 @@ //! all expect. Adding a new binding is a matter of adding an entry to //! [`BINDINGS`] plus a snippet renderer in [`render_examples`]. -use crate::generate::parser::{AttrDeprecated, Config, ConfigType, Services}; +use crate::generate::parser::{AttrDeprecated, Config, ConfigType, S3_PROVIDER_PRESETS, Services}; use anyhow::{Context, Result}; use regex::Regex; use serde::Serialize; @@ -198,14 +198,19 @@ pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> { .map(Field::from) .collect(); let grouped = group_fields(&fields, &groups); - let required: Vec<&Field> = fields.iter().filter(|f| f.required).collect(); + let required: Vec<&Field> = fields.iter().filter(|f| f.required || f.minimal).collect(); let mut examples = Vec::new(); for (binding, support) in BINDINGS.iter().zip(support.iter()) { - let enabled = match support { - None => true, - Some(set) => set.contains(&scheme), - }; + // Go loads one dynamic package per scheme, while these presets share + // the Rust S3 crate and don't have standalone Go service packages. + let enabled = + !matches!(binding.id, "go") || !S3_PROVIDER_PRESETS.contains(&scheme.as_str()); + let enabled = enabled + && match support { + None => true, + Some(set) => set.contains(&scheme), + }; if !enabled { continue; } @@ -286,6 +291,7 @@ struct Field<'a> { name: &'a str, value: ConfigType, required: bool, + minimal: bool, comments: &'a str, group: &'a str, default: Option<&'a str>, @@ -298,6 +304,7 @@ impl<'a> From<&'a Config> for Field<'a> { name: &c.name, value: c.value, required: !c.optional, + minimal: c.minimal, comments: &c.comments, group: c.group.as_deref().unwrap_or(DEFAULT_GROUP), default: c.default_value.as_deref(), @@ -388,8 +395,9 @@ struct Syntax { /// /// `minimal` lists only the required fields. `full` walks every group in order, /// emitting a group header (when there is more than one group), each field's -/// doc comment, and the assignment — uncommented when required, commented out -/// otherwise so the block is a copy-and-trim reference. +/// doc comment, and the assignment. Required fields and optional fields chosen +/// for the minimal example stay uncommented so both snippets are valid; +/// everything else is commented out as a copy-and-trim reference. fn build(required: &[&Field], grouped: &[(&str, Vec<&Field>)], s: &Syntax) -> (String, String) { let doc_prefix = format!("{}{}", s.indent, s.comment); @@ -423,7 +431,7 @@ fn build(required: &[&Field], grouped: &[(&str, Vec<&Field>)], s: &Syntax) -> (S comment_block(field.comments, &mut field_comment); let assign = (s.assign)(field); // field have block indententation - if field.required { + if field.required || field.minimal { config_options.push(format!("{field_comment}{}{assign}", s.indent)); } else { config_options.push(format!("{field_comment}{}{}{assign}", s.indent, s.comment)); @@ -528,3 +536,31 @@ fn render_examples( build(required, grouped, &syntax) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn minimal_optional_fields_stay_active_in_full_examples() { + let field = Field { + name: "account_id", + value: ConfigType::String, + required: false, + minimal: true, + comments: "Cloudflare account ID.", + group: DEFAULT_GROUP, + default: None, + example: Some("example-account"), + }; + let required = vec![&field]; + let grouped = vec![(DEFAULT_GROUP, vec![&field])]; + + let (minimal, full) = render_examples("rust", "r2", &required, &grouped); + + let assignment = "(\"account_id\".to_string(), \"example-account\".to_string()),"; + assert!(minimal.contains(assignment)); + assert!(full.contains(&format!(" {assignment}"))); + assert!(!full.contains(&format!("// {assignment}"))); + } +} diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index f46eaf697bd3..954976b4474b 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -21,6 +21,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fs; use std::fs::read_dir; +use std::path::Path; use std::str::FromStr; use syn::{ Expr, ExprLit, Field, GenericArgument, Item, Lit, LitStr, Meta, PathArguments, Type, TypePath, @@ -29,6 +30,12 @@ use syn::{ pub type Services = HashMap; +pub const S3_PROVIDER_PRESETS: &[&str] = &["minio", "r2"]; + +pub fn service_feature(service: &str) -> String { + format!("services-{}", service.replace('_', "-")) +} + pub fn sorted_services(services: Services, test: fn(&str) -> bool) -> Services { let mut srvs = Services::new(); for (k, srv) in services.into_iter() { @@ -59,7 +66,8 @@ pub struct Config { pub name: String, /// The value type this config. pub value: ConfigType, - /// If given config is optional or not. + /// Whether this config is optional in public configuration. An + /// `@required true` marker can make an `Option` field required. pub optional: bool, /// if this field is deprecated, a deprecated message will be provided. pub deprecated: Option, @@ -75,6 +83,9 @@ pub struct Config { pub default_value: Option, /// An example value, parsed from a `@example ` doc marker. pub example: Option, + /// Whether generated minimal examples should include this optional field, + /// parsed from a `@minimal true` doc marker. + pub minimal: bool, } #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] @@ -159,13 +170,15 @@ pub struct AttrDeprecated { pub note: String, } -/// Structured markers extracted from a field's doc comment, e.g. `@group`, -/// `@default` and `@example`. +/// Structured markers extracted from a field's doc comment, such as `@group`, +/// `@default`, `@example`, `@minimal`, and `@required`. #[derive(Debug, Default, Clone, Eq, PartialEq)] struct DocMarkers { group: Option, default_value: Option, example: Option, + minimal: bool, + required: bool, } /// List and parse given path to a `Services` struct. @@ -200,29 +213,67 @@ pub fn parse(path: &str) -> Result { if dir.file_type()?.is_file() { continue; } - let path = dir.path().join("config.rs"); + let service_dir = dir.path(); + let path = service_dir.join("config.rs"); // Try the old layout (config.rs directly) first, then the new layout (src/config.rs) let path = if path.exists() { path } else { - dir.path().join("src/config.rs") + service_dir.join("src/config.rs") }; - if !path.exists() { + if path.exists() { + insert_service( + &mut map, + dir.file_name().to_string_lossy().to_string(), + &path, + )?; + } + + let src_dir = service_dir.join("src"); + if !src_dir.exists() { continue; } - let content = fs::read_to_string(&path)?; - let parser = ServiceParser { - service: dir.file_name().to_string_lossy().to_string(), - path: path.to_string_lossy().to_string(), - content, - }; - let service = parser.parse().context(format!("path: {path:?}"))?; - map.insert(parser.service, service); + // A service crate can expose additional schemes through + // `_config.rs` files next to its primary `config.rs`. + for entry in read_dir(src_dir)? { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let Some(service) = supplemental_service_name(&entry.path()) else { + continue; + }; + insert_service(&mut map, service, &entry.path())?; + } } Ok(map) } +fn supplemental_service_name(path: &Path) -> Option { + path.file_name()? + .to_str()? + .strip_suffix("_config.rs") + .filter(|name| !name.is_empty()) + .map(ToString::to_string) +} + +fn insert_service(map: &mut Services, service: String, path: &Path) -> Result<()> { + if map.contains_key(&service) { + bail!("duplicate config for service {service}"); + } + + let content = fs::read_to_string(path)?; + let parser = ServiceParser { + service: service.clone(), + path: path.to_string_lossy().to_string(), + content, + }; + let parsed = parser.parse().context(format!("path: {path:?}"))?; + map.insert(service, parsed); + Ok(()) +} + /// ServiceParser is used to parse a service config file. pub struct ServiceParser { service: String, @@ -334,6 +385,7 @@ impl ServiceParser { } v => return Err(anyhow!("unsupported config type {v:?}")), }; + let optional = optional && !markers.required; Ok(Config { name: name.to_string(), @@ -344,6 +396,7 @@ impl ServiceParser { group: markers.group, default_value: markers.default_value, example: markers.example, + minimal: markers.minimal, }) } @@ -388,6 +441,8 @@ impl ServiceParser { "group" => Some(("group", value)), "default" => Some(("default", value)), "example" => Some(("example", value)), + "minimal" => Some(("minimal", value)), + "required" => Some(("required", value)), _ => None, } } @@ -419,6 +474,8 @@ impl ServiceParser { "group" => markers.group = Some(value), "default" => markers.default_value = Some(value), "example" => markers.example = Some(value), + "minimal" => markers.minimal = value == "true", + "required" => markers.required = value == "true", _ => {} } } @@ -499,6 +556,7 @@ mod tests { group: None, default_value: None, example: None, + minimal: false, }, ), ( @@ -512,6 +570,7 @@ mod tests { group: None, default_value: None, example: None, + minimal: false, }, ), ]; @@ -793,6 +852,7 @@ Please tune this value based on services' document." group: None, default_value: None, example: None, + minimal: false, }, ); assert_eq!( @@ -808,6 +868,7 @@ For example, Ceph RADOS S3 doesn't support write with if match.".to_string(), group: None, default_value: None, example: None, + minimal: false, }, ); } @@ -823,7 +884,9 @@ For example, Ceph RADOS S3 doesn't support write with if match.".to_string(), /// /// /// @example example-bucket - pub bucket: String + /// + /// + pub bucket: Option } "#; let x: ItemStruct = syn::parse_str(input).unwrap(); @@ -835,9 +898,19 @@ For example, Ceph RADOS S3 doesn't support write with if match.".to_string(), assert_eq!(actual.group.as_deref(), Some("General")); assert_eq!(actual.default_value.as_deref(), Some("my-bucket")); assert_eq!(actual.example.as_deref(), Some("example-bucket")); + assert!(actual.minimal); + assert!(!actual.optional); assert_eq!(actual.comments, "bucket name of this backend.\n\nrequired."); } + #[test] + fn test_service_feature() { + assert_eq!(service_feature("s3"), "services-s3"); + assert_eq!(service_feature("r2"), "services-r2"); + assert_eq!(service_feature("minio"), "services-minio"); + assert_eq!(service_feature("mini_moka"), "services-mini-moka"); + } + #[test] fn test_parse() { let path = workspace_dir() @@ -847,5 +920,14 @@ For example, Ceph RADOS S3 doesn't support write with if match.".to_string(), // Parse should just pass. let _ = parse(&path.to_string_lossy()).unwrap(); + + let path = workspace_dir() + .join("core/services") + .canonicalize() + .unwrap(); + let services = parse(&path.to_string_lossy()).unwrap(); + assert!(services.contains_key("s3")); + assert!(services.contains_key("r2")); + assert!(services.contains_key("minio")); } } diff --git a/dev/src/generate/python.rs b/dev/src/generate/python.rs index 0763c066a8bd..a47a6f3d225a 100644 --- a/dev/src/generate/python.rs +++ b/dev/src/generate/python.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::generate::parser::{Config, ConfigType, Services, sorted_services}; +use crate::generate::parser::{Config, ConfigType, Services, service_feature, sorted_services}; use anyhow::Result; use minijinja::value::ViaDeserialize; use minijinja::{Environment, context}; @@ -38,7 +38,7 @@ pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> { env.add_template("python", include_str!("python.j2"))?; env.add_template("python_config", include_str!("python_config.j2"))?; env.add_function("snake_to_kebab_case", snake_to_kebab_case); - env.add_function("service_to_feature", service_to_feature); + env.add_function("service_to_feature", service_feature); env.add_function("service_to_pascal", service_to_pascal); env.add_function("make_python_type", make_python_type); env.add_function("make_pydoc_param_header", make_pydoc_param_header); @@ -72,10 +72,6 @@ fn snake_to_kebab_case(str: &str) -> String { str.replace('_', "-") } -fn service_to_feature(service: &str) -> String { - format!("services-{}", snake_to_kebab_case(service)) -} - fn service_to_pascal(service: &str) -> String { let mut result = String::with_capacity(service.len()); let mut capitalize = true; diff --git a/justfile b/justfile index 956577df4476..91c8b9c9aae8 100644 --- a/justfile +++ b/justfile @@ -17,7 +17,7 @@ # Generate code for language # -# Available languages: python, java +# Available languages: python, java, docs generate language: cargo run --quiet --manifest-path=dev/Cargo.toml -- generate -l {{language}} diff --git a/website/data/services.json b/website/data/services.json index f9600f47118f..c0f9c29c4a8f 100644 --- a/website/data/services.json +++ b/website/data/services.json @@ -2769,6 +2769,109 @@ } ] }, + { + "name": "minio", + "scheme": "minio", + "groups": [ + "General", + "Credentials" + ], + "configs": [ + { + "name": "root", + "type": "string", + "required": false, + "group": "General", + "default": "/", + "comments": "Root within the bucket.\n\nAll operations happen under this root. The default is `/`." + }, + { + "name": "bucket", + "type": "string", + "required": true, + "group": "General", + "example": "my-bucket", + "comments": "Bucket name.\n\nThis field is required." + }, + { + "name": "endpoint", + "type": "string", + "required": true, + "group": "General", + "example": "http://127.0.0.1:9000", + "comments": "MinIO endpoint.\n\nThis field is required because MinIO deployments do not share a\nuniversal endpoint." + }, + { + "name": "region", + "type": "string", + "required": false, + "group": "General", + "default": "auto", + "comments": "Signing region.\n\nThe default is `auto`. Set this field when the deployment requires a\nconfigured region." + }, + { + "name": "access_key_id", + "type": "string", + "required": false, + "group": "Credentials", + "comments": "Access key ID.\n\nSet this field together with `secret_access_key`." + }, + { + "name": "secret_access_key", + "type": "string", + "required": false, + "group": "Credentials", + "comments": "Secret access key.\n\nSet this field together with `access_key_id`." + }, + { + "name": "session_token", + "type": "string", + "required": false, + "group": "Credentials", + "comments": "Session token for temporary credentials.\n\nThis field requires `access_key_id` and `secret_access_key`." + }, + { + "name": "skip_signature", + "type": "bool", + "required": false, + "group": "Credentials", + "default": "false", + "comments": "Send requests without signing them.\n\nThis option cannot be combined with direct credentials." + } + ], + "examples": [ + { + "binding": "rust", + "language": "rust", + "minimal": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"minio\", [\n (\"bucket\".to_string(), \"my-bucket\".to_string()),\n (\"endpoint\".to_string(), \"http://127.0.0.1:9000\".to_string()),\n])?;", + "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"minio\", [\n // --- General ---\n // Root within the bucket.\n //\n // All operations happen under this root. The default is `/`.\n // (\"root\".to_string(), \"/\".to_string()),\n\n // Bucket name.\n //\n // This field is required.\n (\"bucket\".to_string(), \"my-bucket\".to_string()),\n\n // MinIO endpoint.\n //\n // This field is required because MinIO deployments do not share a\n // universal endpoint.\n (\"endpoint\".to_string(), \"http://127.0.0.1:9000\".to_string()),\n\n // Signing region.\n //\n // The default is `auto`. Set this field when the deployment requires a\n // configured region.\n // (\"region\".to_string(), \"auto\".to_string()),\n // --- Credentials ---\n // Access key ID.\n //\n // Set this field together with `secret_access_key`.\n // (\"access_key_id\".to_string(), \"...\".to_string()),\n\n // Secret access key.\n //\n // Set this field together with `access_key_id`.\n // (\"secret_access_key\".to_string(), \"...\".to_string()),\n\n // Session token for temporary credentials.\n //\n // This field requires `access_key_id` and `secret_access_key`.\n // (\"session_token\".to_string(), \"...\".to_string()),\n\n // Send requests without signing them.\n //\n // This option cannot be combined with direct credentials.\n // (\"skip_signature\".to_string(), \"false\".to_string()),\n])?;" + }, + { + "binding": "java", + "language": "java", + "minimal": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\nconfig.put(\"bucket\", \"my-bucket\");\nconfig.put(\"endpoint\", \"http://127.0.0.1:9000\");\nOperator operator = Operator.of(\"minio\", config);", + "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\n// --- General ---\n// Root within the bucket.\n//\n// All operations happen under this root. The default is `/`.\n// config.put(\"root\", \"/\");\n\n// Bucket name.\n//\n// This field is required.\nconfig.put(\"bucket\", \"my-bucket\");\n\n// MinIO endpoint.\n//\n// This field is required because MinIO deployments do not share a\n// universal endpoint.\nconfig.put(\"endpoint\", \"http://127.0.0.1:9000\");\n\n// Signing region.\n//\n// The default is `auto`. Set this field when the deployment requires a\n// configured region.\n// config.put(\"region\", \"auto\");\n// --- Credentials ---\n// Access key ID.\n//\n// Set this field together with `secret_access_key`.\n// config.put(\"access_key_id\", \"...\");\n\n// Secret access key.\n//\n// Set this field together with `access_key_id`.\n// config.put(\"secret_access_key\", \"...\");\n\n// Session token for temporary credentials.\n//\n// This field requires `access_key_id` and `secret_access_key`.\n// config.put(\"session_token\", \"...\");\n\n// Send requests without signing them.\n//\n// This option cannot be combined with direct credentials.\n// config.put(\"skip_signature\", \"false\");\nOperator operator = Operator.of(\"minio\", config);" + }, + { + "binding": "python", + "language": "python", + "minimal": "import opendal\n\noperator = opendal.Operator(\n \"minio\",\n bucket=\"my-bucket\",\n endpoint=\"http://127.0.0.1:9000\",\n)", + "full": "import opendal\n\noperator = opendal.Operator(\n \"minio\",\n # --- General ---\n # Root within the bucket.\n #\n # All operations happen under this root. The default is `/`.\n # root=\"/\",\n\n # Bucket name.\n #\n # This field is required.\n bucket=\"my-bucket\",\n\n # MinIO endpoint.\n #\n # This field is required because MinIO deployments do not share a\n # universal endpoint.\n endpoint=\"http://127.0.0.1:9000\",\n\n # Signing region.\n #\n # The default is `auto`. Set this field when the deployment requires a\n # configured region.\n # region=\"auto\",\n # --- Credentials ---\n # Access key ID.\n #\n # Set this field together with `secret_access_key`.\n # access_key_id=\"...\",\n\n # Secret access key.\n #\n # Set this field together with `access_key_id`.\n # secret_access_key=\"...\",\n\n # Session token for temporary credentials.\n #\n # This field requires `access_key_id` and `secret_access_key`.\n # session_token=\"...\",\n\n # Send requests without signing them.\n #\n # This option cannot be combined with direct credentials.\n # skip_signature=\"false\",\n)" + }, + { + "binding": "nodejs", + "language": "javascript", + "minimal": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"minio\", {\n bucket: \"my-bucket\",\n endpoint: \"http://127.0.0.1:9000\",\n});", + "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"minio\", {\n // --- General ---\n // Root within the bucket.\n //\n // All operations happen under this root. The default is `/`.\n // root: \"/\",\n\n // Bucket name.\n //\n // This field is required.\n bucket: \"my-bucket\",\n\n // MinIO endpoint.\n //\n // This field is required because MinIO deployments do not share a\n // universal endpoint.\n endpoint: \"http://127.0.0.1:9000\",\n\n // Signing region.\n //\n // The default is `auto`. Set this field when the deployment requires a\n // configured region.\n // region: \"auto\",\n // --- Credentials ---\n // Access key ID.\n //\n // Set this field together with `secret_access_key`.\n // access_key_id: \"...\",\n\n // Secret access key.\n //\n // Set this field together with `access_key_id`.\n // secret_access_key: \"...\",\n\n // Session token for temporary credentials.\n //\n // This field requires `access_key_id` and `secret_access_key`.\n // session_token: \"...\",\n\n // Send requests without signing them.\n //\n // This option cannot be combined with direct credentials.\n // skip_signature: \"false\",\n});" + }, + { + "binding": "ruby", + "language": "ruby", + "minimal": "require \"opendal\"\n\noperator = OpenDal::Operator.new(\"minio\", {\n \"bucket\" => \"my-bucket\",\n \"endpoint\" => \"http://127.0.0.1:9000\",\n})", + "full": "require \"opendal\"\n\noperator = OpenDal::Operator.new(\"minio\", {\n # --- General ---\n # Root within the bucket.\n #\n # All operations happen under this root. The default is `/`.\n # \"root\" => \"/\",\n\n # Bucket name.\n #\n # This field is required.\n \"bucket\" => \"my-bucket\",\n\n # MinIO endpoint.\n #\n # This field is required because MinIO deployments do not share a\n # universal endpoint.\n \"endpoint\" => \"http://127.0.0.1:9000\",\n\n # Signing region.\n #\n # The default is `auto`. Set this field when the deployment requires a\n # configured region.\n # \"region\" => \"auto\",\n # --- Credentials ---\n # Access key ID.\n #\n # Set this field together with `secret_access_key`.\n # \"access_key_id\" => \"...\",\n\n # Secret access key.\n #\n # Set this field together with `access_key_id`.\n # \"secret_access_key\" => \"...\",\n\n # Session token for temporary credentials.\n #\n # This field requires `access_key_id` and `secret_access_key`.\n # \"session_token\" => \"...\",\n\n # Send requests without signing them.\n #\n # This option cannot be combined with direct credentials.\n # \"skip_signature\" => \"false\",\n})" + } + ] + }, { "name": "moka", "scheme": "moka", @@ -3674,6 +3777,108 @@ } ] }, + { + "name": "r2", + "scheme": "r2", + "groups": [ + "General", + "Credentials" + ], + "configs": [ + { + "name": "root", + "type": "string", + "required": false, + "group": "General", + "default": "/", + "comments": "Root within the bucket.\n\nAll operations happen under this root. The default is `/`." + }, + { + "name": "bucket", + "type": "string", + "required": true, + "group": "General", + "example": "my-bucket", + "comments": "Bucket name.\n\nThis field is required." + }, + { + "name": "account_id", + "type": "string", + "required": false, + "group": "General", + "example": "example-account", + "comments": "Cloudflare account ID used to derive the R2 endpoint.\n\nSet exactly one of `account_id` and `endpoint`." + }, + { + "name": "jurisdiction", + "type": "string", + "required": false, + "group": "General", + "comments": "R2 jurisdiction.\n\nSupported values are `eu` and `fedramp`. This field requires\n`account_id` and cannot be used with `endpoint`." + }, + { + "name": "endpoint", + "type": "string", + "required": false, + "group": "General", + "example": "https://example.r2.cloudflarestorage.com", + "comments": "Explicit R2-compatible endpoint.\n\nUse this field for a proxy, gateway, or test server. Set exactly one of\n`endpoint` and `account_id`." + }, + { + "name": "access_key_id", + "type": "string", + "required": false, + "group": "Credentials", + "comments": "Access key ID.\n\nSet this field together with `secret_access_key`." + }, + { + "name": "secret_access_key", + "type": "string", + "required": false, + "group": "Credentials", + "comments": "Secret access key.\n\nSet this field together with `access_key_id`." + }, + { + "name": "session_token", + "type": "string", + "required": false, + "group": "Credentials", + "comments": "Session token for temporary credentials.\n\nThis field requires `access_key_id` and `secret_access_key`." + } + ], + "examples": [ + { + "binding": "rust", + "language": "rust", + "minimal": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"r2\", [\n (\"bucket\".to_string(), \"my-bucket\".to_string()),\n (\"account_id\".to_string(), \"example-account\".to_string()),\n])?;", + "full": "use opendal::Operator;\n\nlet operator = Operator::via_iter(\"r2\", [\n // --- General ---\n // Root within the bucket.\n //\n // All operations happen under this root. The default is `/`.\n // (\"root\".to_string(), \"/\".to_string()),\n\n // Bucket name.\n //\n // This field is required.\n (\"bucket\".to_string(), \"my-bucket\".to_string()),\n\n // Cloudflare account ID used to derive the R2 endpoint.\n //\n // Set exactly one of `account_id` and `endpoint`.\n (\"account_id\".to_string(), \"example-account\".to_string()),\n\n // R2 jurisdiction.\n //\n // Supported values are `eu` and `fedramp`. This field requires\n // `account_id` and cannot be used with `endpoint`.\n // (\"jurisdiction\".to_string(), \"...\".to_string()),\n\n // Explicit R2-compatible endpoint.\n //\n // Use this field for a proxy, gateway, or test server. Set exactly one of\n // `endpoint` and `account_id`.\n // (\"endpoint\".to_string(), \"https://example.r2.cloudflarestorage.com\".to_string()),\n // --- Credentials ---\n // Access key ID.\n //\n // Set this field together with `secret_access_key`.\n // (\"access_key_id\".to_string(), \"...\".to_string()),\n\n // Secret access key.\n //\n // Set this field together with `access_key_id`.\n // (\"secret_access_key\".to_string(), \"...\".to_string()),\n\n // Session token for temporary credentials.\n //\n // This field requires `access_key_id` and `secret_access_key`.\n // (\"session_token\".to_string(), \"...\".to_string()),\n])?;" + }, + { + "binding": "java", + "language": "java", + "minimal": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\nconfig.put(\"bucket\", \"my-bucket\");\nconfig.put(\"account_id\", \"example-account\");\nOperator operator = Operator.of(\"r2\", config);", + "full": "import java.util.HashMap;\nimport java.util.Map;\nimport org.apache.opendal.Operator;\n\nMap config = new HashMap<>();\n// --- General ---\n// Root within the bucket.\n//\n// All operations happen under this root. The default is `/`.\n// config.put(\"root\", \"/\");\n\n// Bucket name.\n//\n// This field is required.\nconfig.put(\"bucket\", \"my-bucket\");\n\n// Cloudflare account ID used to derive the R2 endpoint.\n//\n// Set exactly one of `account_id` and `endpoint`.\nconfig.put(\"account_id\", \"example-account\");\n\n// R2 jurisdiction.\n//\n// Supported values are `eu` and `fedramp`. This field requires\n// `account_id` and cannot be used with `endpoint`.\n// config.put(\"jurisdiction\", \"...\");\n\n// Explicit R2-compatible endpoint.\n//\n// Use this field for a proxy, gateway, or test server. Set exactly one of\n// `endpoint` and `account_id`.\n// config.put(\"endpoint\", \"https://example.r2.cloudflarestorage.com\");\n// --- Credentials ---\n// Access key ID.\n//\n// Set this field together with `secret_access_key`.\n// config.put(\"access_key_id\", \"...\");\n\n// Secret access key.\n//\n// Set this field together with `access_key_id`.\n// config.put(\"secret_access_key\", \"...\");\n\n// Session token for temporary credentials.\n//\n// This field requires `access_key_id` and `secret_access_key`.\n// config.put(\"session_token\", \"...\");\nOperator operator = Operator.of(\"r2\", config);" + }, + { + "binding": "python", + "language": "python", + "minimal": "import opendal\n\noperator = opendal.Operator(\n \"r2\",\n bucket=\"my-bucket\",\n account_id=\"example-account\",\n)", + "full": "import opendal\n\noperator = opendal.Operator(\n \"r2\",\n # --- General ---\n # Root within the bucket.\n #\n # All operations happen under this root. The default is `/`.\n # root=\"/\",\n\n # Bucket name.\n #\n # This field is required.\n bucket=\"my-bucket\",\n\n # Cloudflare account ID used to derive the R2 endpoint.\n #\n # Set exactly one of `account_id` and `endpoint`.\n account_id=\"example-account\",\n\n # R2 jurisdiction.\n #\n # Supported values are `eu` and `fedramp`. This field requires\n # `account_id` and cannot be used with `endpoint`.\n # jurisdiction=\"...\",\n\n # Explicit R2-compatible endpoint.\n #\n # Use this field for a proxy, gateway, or test server. Set exactly one of\n # `endpoint` and `account_id`.\n # endpoint=\"https://example.r2.cloudflarestorage.com\",\n # --- Credentials ---\n # Access key ID.\n #\n # Set this field together with `secret_access_key`.\n # access_key_id=\"...\",\n\n # Secret access key.\n #\n # Set this field together with `access_key_id`.\n # secret_access_key=\"...\",\n\n # Session token for temporary credentials.\n #\n # This field requires `access_key_id` and `secret_access_key`.\n # session_token=\"...\",\n)" + }, + { + "binding": "nodejs", + "language": "javascript", + "minimal": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"r2\", {\n bucket: \"my-bucket\",\n account_id: \"example-account\",\n});", + "full": "import { Operator } from \"opendal\";\n\nconst operator = new Operator(\"r2\", {\n // --- General ---\n // Root within the bucket.\n //\n // All operations happen under this root. The default is `/`.\n // root: \"/\",\n\n // Bucket name.\n //\n // This field is required.\n bucket: \"my-bucket\",\n\n // Cloudflare account ID used to derive the R2 endpoint.\n //\n // Set exactly one of `account_id` and `endpoint`.\n account_id: \"example-account\",\n\n // R2 jurisdiction.\n //\n // Supported values are `eu` and `fedramp`. This field requires\n // `account_id` and cannot be used with `endpoint`.\n // jurisdiction: \"...\",\n\n // Explicit R2-compatible endpoint.\n //\n // Use this field for a proxy, gateway, or test server. Set exactly one of\n // `endpoint` and `account_id`.\n // endpoint: \"https://example.r2.cloudflarestorage.com\",\n // --- Credentials ---\n // Access key ID.\n //\n // Set this field together with `secret_access_key`.\n // access_key_id: \"...\",\n\n // Secret access key.\n //\n // Set this field together with `access_key_id`.\n // secret_access_key: \"...\",\n\n // Session token for temporary credentials.\n //\n // This field requires `access_key_id` and `secret_access_key`.\n // session_token: \"...\",\n});" + }, + { + "binding": "ruby", + "language": "ruby", + "minimal": "require \"opendal\"\n\noperator = OpenDal::Operator.new(\"r2\", {\n \"bucket\" => \"my-bucket\",\n \"account_id\" => \"example-account\",\n})", + "full": "require \"opendal\"\n\noperator = OpenDal::Operator.new(\"r2\", {\n # --- General ---\n # Root within the bucket.\n #\n # All operations happen under this root. The default is `/`.\n # \"root\" => \"/\",\n\n # Bucket name.\n #\n # This field is required.\n \"bucket\" => \"my-bucket\",\n\n # Cloudflare account ID used to derive the R2 endpoint.\n #\n # Set exactly one of `account_id` and `endpoint`.\n \"account_id\" => \"example-account\",\n\n # R2 jurisdiction.\n #\n # Supported values are `eu` and `fedramp`. This field requires\n # `account_id` and cannot be used with `endpoint`.\n # \"jurisdiction\" => \"...\",\n\n # Explicit R2-compatible endpoint.\n #\n # Use this field for a proxy, gateway, or test server. Set exactly one of\n # `endpoint` and `account_id`.\n # \"endpoint\" => \"https://example.r2.cloudflarestorage.com\",\n # --- Credentials ---\n # Access key ID.\n #\n # Set this field together with `secret_access_key`.\n # \"access_key_id\" => \"...\",\n\n # Secret access key.\n #\n # Set this field together with `access_key_id`.\n # \"secret_access_key\" => \"...\",\n\n # Session token for temporary credentials.\n #\n # This field requires `access_key_id` and `secret_access_key`.\n # \"session_token\" => \"...\",\n})" + } + ] + }, { "name": "redb", "scheme": "redb",