From 7c00831e659efbfac754e2ddf464be7416446592 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 15:38:17 +0800 Subject: [PATCH 01/12] [rest] Add database branch and tag APIs --- docs/static/rest-catalog-open-api.yaml | 293 ++++++++++++++++++ .../apache/paimon/rest/DatabaseReference.java | 89 ++++++ .../paimon/rest/DatabaseReferenceType.java | 35 +++ .../org/apache/paimon/rest/HttpClient.java | 53 +++- .../apache/paimon/rest/HttpClientUtils.java | 5 + .../java/org/apache/paimon/rest/RESTApi.java | 114 +++++++ .../paimon/rest/RESTCatalogOptions.java | 8 + .../org/apache/paimon/rest/ResourcePaths.java | 13 + .../ListDatabaseReferencesResponse.java | 72 +++++ .../SingleDatabaseReferenceResponse.java | 51 +++ .../rest/RESTApiDatabaseReferenceTest.java | 282 +++++++++++++++++ 11 files changed, 1009 insertions(+), 6 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java create mode 100644 paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 02ee6df595be..24964106bc00 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -203,6 +203,256 @@ paths: $ref: '#/components/responses/DatabaseNotExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/trees: + get: + tags: + - database-reference + summary: List database references + operationId: listDatabaseReferences + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: type + in: query + required: false + schema: + type: string + enum: [ "branch", "tag" ] + - name: maxResults + in: query + required: false + schema: + type: integer + format: int32 + - name: pageToken + in: query + required: false + schema: + type: string + responses: + "200": + description: Database branches and immutable tags. + content: + application/json: + schema: + $ref: '#/components/schemas/ListDatabaseReferencesResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/DatabaseNotExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + post: + tags: + - database-reference + summary: Create database reference + operationId: createDatabaseReference + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: query + required: true + schema: + type: string + - name: type + in: query + required: true + schema: + type: string + enum: [ "branch", "tag" ] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DatabaseReference' + responses: + "200": + description: Created branch or immutable tag. + content: + application/json: + schema: + $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/DatabaseNotExistErrorResponse' + "409": + description: Reference already exists. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/trees/{name}: + get: + tags: + - database-reference + summary: Get database reference + operationId: getDatabaseReference + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: path + required: true + schema: + type: string + responses: + "200": + description: Named branch or immutable tag. + content: + application/json: + schema: + $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + description: Database or reference does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + put: + tags: + - database-reference + summary: Fast-forward database branch from immutable tag + operationId: fastForwardDatabaseBranch + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: path + required: true + schema: + type: string + - name: mode + in: query + required: true + schema: + type: string + const: "FAST_FORWARD" + - name: type + in: query + required: false + schema: + type: string + const: "branch" + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DatabaseReference' + responses: + "200": + description: Updated target branch. + content: + application/json: + schema: + $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + description: Database, target branch, or source tag does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "409": + description: Target is not an ancestor of the source tag. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + delete: + tags: + - database-reference + summary: Delete database reference + operationId: deleteDatabaseReference + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: path + required: true + schema: + type: string + - name: type + in: query + required: false + schema: + type: string + enum: [ "branch", "tag" ] + responses: + "200": + description: Deleted branch or immutable tag. + content: + application/json: + schema: + $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + description: Database or reference does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "409": + description: Reference type does not match or the default branch is protected. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' /v1/{prefix}/databases/{database}/register: post: tags: @@ -269,6 +519,7 @@ paths: in: query schema: type: string + - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -461,6 +712,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -595,6 +847,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/DatabaseReferenceHeader' requestBody: content: application/json: @@ -810,6 +1063,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -2406,6 +2660,15 @@ paths: $ref: '#/components/responses/SemanticViewNotImplementedErrorResponse' components: + parameters: + DatabaseReferenceHeader: + name: Paimon-Reference + in: header + required: false + description: Database-level branch or immutable tag. Required for a versioned database. + schema: + type: string + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" ############################# # Reusable Response Objects # ############################# @@ -3753,6 +4016,36 @@ components: $ref: '#/components/schemas/Identifier' nextPageToken: type: string + DatabaseReference: + type: object + required: + - type + - name + properties: + type: + type: string + enum: [ "BRANCH", "TAG" ] + name: + type: string + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + SingleDatabaseReferenceResponse: + type: object + required: + - reference + properties: + reference: + $ref: '#/components/schemas/DatabaseReference' + ListDatabaseReferencesResponse: + type: object + required: + - references + properties: + references: + type: array + items: + $ref: '#/components/schemas/DatabaseReference' + nextPageToken: + type: string ConfigResponse: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java new file mode 100644 index 000000000000..416dff1749ce --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; +import java.util.Objects; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** A named database-level branch or immutable tag. */ +@Experimental +public class DatabaseReference implements RESTRequest { + + private static final String FIELD_TYPE = "type"; + private static final String FIELD_NAME = "name"; + private static final String NAME_PATTERN = "[A-Za-z0-9][A-Za-z0-9._-]{0,127}"; + + @JsonProperty(FIELD_TYPE) + private final DatabaseReferenceType type; + + @JsonProperty(FIELD_NAME) + private final String name; + + @JsonCreator + @ConstructorProperties({FIELD_TYPE, FIELD_NAME}) + public DatabaseReference( + @JsonProperty(FIELD_TYPE) DatabaseReferenceType type, + @JsonProperty(FIELD_NAME) String name) { + checkArgument(type != null, "Reference type must not be null"); + checkArgument( + name != null && name.matches(NAME_PATTERN), "Invalid reference name: %s", name); + this.type = type; + this.name = name; + } + + @JsonGetter(FIELD_TYPE) + public DatabaseReferenceType getType() { + return type; + } + + @JsonGetter(FIELD_NAME) + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DatabaseReference)) { + return false; + } + DatabaseReference that = (DatabaseReference) o; + return type == that.type && name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(type, name); + } + + @Override + public String toString() { + return type + ":" + name; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java new file mode 100644 index 000000000000..55b3e7813a62 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.annotation.Experimental; + +import java.util.Locale; + +/** Types of database-level references supported by the REST catalog. */ +@Experimental +public enum DatabaseReferenceType { + BRANCH, + TAG; + + /** Lowercase form used by the trees query parameters. */ + public String queryValue() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java index 8205dfe21295..e30ecab35f3f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -32,6 +32,7 @@ import org.apache.hc.client5.http.classic.methods.HttpDelete; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpPut; import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; import org.apache.hc.core5.http.ClassicHttpResponse; import org.apache.hc.core5.http.ContentType; @@ -97,12 +98,21 @@ public T post( RESTRequest body, Class responseType, RESTAuthFunction restAuthFunction) { - HttpPost httpPost = HttpClientUtils.newHttpPost(getRequestUrl(path, null)); + return post(path, Collections.emptyMap(), body, responseType, restAuthFunction); + } + + public T post( + String path, + Map queryParams, + RESTRequest body, + Class responseType, + RESTAuthFunction restAuthFunction) { + HttpPost httpPost = HttpClientUtils.newHttpPost(getRequestUrl(path, queryParams)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpPost.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = getHeaders(path, "POST", encodedBody, restAuthFunction); + Header[] authHeaders = getHeaders(path, queryParams, "POST", encodedBody, restAuthFunction); httpPost.setHeaders(authHeaders); // A POST the server cannot absorb twice is sent exactly once, whatever the status says. return exec( @@ -113,22 +123,53 @@ public T post( : null); } + public T put( + String path, + Map queryParams, + RESTRequest body, + Class responseType, + RESTAuthFunction restAuthFunction) { + HttpPut httpPut = HttpClientUtils.newHttpPut(getRequestUrl(path, queryParams)); + String encodedBody = RESTUtil.encodedBody(body); + if (encodedBody != null) { + httpPut.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); + } + Header[] authHeaders = getHeaders(path, queryParams, "PUT", encodedBody, restAuthFunction); + httpPut.setHeaders(authHeaders); + return exec( + httpPut, + responseType, + body != null && !body.isRetrySafe() + ? ExponentialHttpRequestRetryStrategy.retryUnsafeContext() + : null); + } + @Override public T delete(String path, RESTAuthFunction restAuthFunction) { - return delete(path, null, restAuthFunction); + return delete(path, Collections.emptyMap(), null, null, restAuthFunction); } @Override public T delete( String path, RESTRequest body, RESTAuthFunction restAuthFunction) { - HttpDelete httpDelete = HttpClientUtils.newHttpDelete(getRequestUrl(path, null)); + return delete(path, Collections.emptyMap(), body, null, restAuthFunction); + } + + public T delete( + String path, + Map queryParams, + RESTRequest body, + Class responseType, + RESTAuthFunction restAuthFunction) { + HttpDelete httpDelete = HttpClientUtils.newHttpDelete(getRequestUrl(path, queryParams)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpDelete.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = getHeaders(path, "DELETE", encodedBody, restAuthFunction); + Header[] authHeaders = + getHeaders(path, queryParams, "DELETE", encodedBody, restAuthFunction); httpDelete.setHeaders(authHeaders); - return exec(httpDelete, null); + return exec(httpDelete, responseType); } @VisibleForTesting diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java index 1444c8118d14..94d0ba80f723 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java @@ -26,6 +26,7 @@ import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpHead; import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpPut; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.entity.DecompressingEntity; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -238,6 +239,10 @@ public static HttpPost newHttpPost(String uri) { return newRequest(uri, HttpPost::new); } + public static HttpPut newHttpPut(String uri) { + return newRequest(uri, HttpPut::new); + } + public static HttpDelete newHttpDelete(String uri) { return newRequest(uri, HttpDelete::new); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 8246cb153335..64b5aeab8873 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -90,6 +90,7 @@ import org.apache.paimon.rest.responses.GetViewResponse; import org.apache.paimon.rest.responses.ListBranchesResponse; import org.apache.paimon.rest.responses.ListConsumersResponse; +import org.apache.paimon.rest.responses.ListDatabaseReferencesResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListFunctionDetailsResponse; import org.apache.paimon.rest.responses.ListFunctionsGloballyResponse; @@ -109,6 +110,7 @@ import org.apache.paimon.rest.responses.ListViewsGloballyResponse; import org.apache.paimon.rest.responses.ListViewsResponse; import org.apache.paimon.rest.responses.PagedResponse; +import org.apache.paimon.rest.responses.SingleDatabaseReferenceResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; @@ -194,6 +196,11 @@ public class RESTApi { public static final String PARTITION_NAME_PATTERN = "partitionNamePattern"; public static final String TAG_NAME_PREFIX = "tagNamePrefix"; + private static final String REFERENCE_NAME = "name"; + private static final String REFERENCE_TYPE = "type"; + private static final String REFERENCE_UPDATE_MODE = "mode"; + private static final String FAST_FORWARD = "FAST_FORWARD"; + public static final long TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000L; public static final ObjectMapper OBJECT_MAPPER = JsonSerdeUtil.OBJECT_MAPPER_INSTANCE; @@ -360,6 +367,113 @@ public void alterDatabase(String name, List removals, Map listDatabaseReferences( + String databaseName, @Nullable DatabaseReferenceType type) { + return listDataFromPageApi( + queryParams -> { + if (type != null) { + queryParams.put(REFERENCE_TYPE, type.queryValue()); + } + return client.get( + resourcePaths.databaseTrees(databaseName), + queryParams, + ListDatabaseReferencesResponse.class, + restAuthFunction); + }); + } + + /** List one page of database-level branches and immutable tags. */ + @Experimental + public PagedList listDatabaseReferencesPaged( + String databaseName, + @Nullable DatabaseReferenceType type, + @Nullable Integer maxResults, + @Nullable String pageToken) { + Map queryParams = buildPagedQueryParams(maxResults, pageToken); + if (type != null) { + queryParams.put(REFERENCE_TYPE, type.queryValue()); + } + ListDatabaseReferencesResponse response = + client.get( + resourcePaths.databaseTrees(databaseName), + queryParams, + ListDatabaseReferencesResponse.class, + restAuthFunction); + List references = response.getReferences(); + return new PagedList<>( + references == null ? emptyList() : references, response.getNextPageToken()); + } + + /** Get one database-level branch or immutable tag. */ + @Experimental + public DatabaseReference getDatabaseReference(String databaseName, String referenceName) { + SingleDatabaseReferenceResponse response = + client.get( + resourcePaths.databaseTree(databaseName, referenceName), + SingleDatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + + /** Create a database-level branch or immutable tag from an existing reference. */ + @Experimental + public DatabaseReference createDatabaseReference( + String databaseName, + String referenceName, + DatabaseReferenceType type, + DatabaseReference source) { + Map queryParams = Maps.newHashMap(); + queryParams.put(REFERENCE_NAME, referenceName); + queryParams.put(REFERENCE_TYPE, type.queryValue()); + SingleDatabaseReferenceResponse response = + client.post( + resourcePaths.databaseTrees(databaseName), + queryParams, + source, + SingleDatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + + /** Fast-forward a database-level branch to an immutable tag. */ + @Experimental + public DatabaseReference fastForwardDatabaseBranch( + String databaseName, String targetBranch, String sourceTag) { + Map queryParams = Maps.newHashMap(); + queryParams.put(REFERENCE_UPDATE_MODE, FAST_FORWARD); + queryParams.put(REFERENCE_TYPE, DatabaseReferenceType.BRANCH.queryValue()); + SingleDatabaseReferenceResponse response = + client.put( + resourcePaths.databaseTree(databaseName, targetBranch), + queryParams, + new DatabaseReference(DatabaseReferenceType.TAG, sourceTag), + SingleDatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + + /** Delete one database-level branch or immutable tag. */ + @Experimental + public DatabaseReference deleteDatabaseReference( + String databaseName, + String referenceName, + @Nullable DatabaseReferenceType expectedType) { + Map queryParams = Maps.newHashMap(); + if (expectedType != null) { + queryParams.put(REFERENCE_TYPE, expectedType.queryValue()); + } + SingleDatabaseReferenceResponse response = + client.delete( + resourcePaths.databaseTree(databaseName, referenceName), + queryParams, + null, + SingleDatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + /** * List tables for a database. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index e7da1f6827dc..3629907dc65f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -98,6 +98,14 @@ public class RESTCatalogOptions { .withDescription( "The user agent of http client connecting to REST Catalog server."); + /** Database-level branch or immutable tag carried by table API requests. */ + public static final ConfigOption DATABASE_REFERENCE = + ConfigOptions.key("header.Paimon-Reference") + .stringType() + .noDefaultValue() + .withDescription( + "Database-level branch or immutable tag used by REST table requests."); + public static final ConfigOption DLF_OSS_ENDPOINT = ConfigOptions.key("dlf.oss-endpoint") .stringType() diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index f3065708da9c..862f78860200 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -36,6 +36,7 @@ public class ResourcePaths { protected static final String PARTITIONS = "partitions"; protected static final String BRANCHES = "branches"; protected static final String TAGS = "tags"; + protected static final String TREES = "trees"; protected static final String SNAPSHOTS = "snapshots"; protected static final String CONSUMERS = "consumers"; protected static final String SCHEMAS = "schemas"; @@ -144,6 +145,18 @@ public String database(String databaseName) { return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName)); } + /** Database-level branches and immutable tags. */ + @Experimental + public String databaseTrees(String databaseName) { + return SLASH.join(database(databaseName), TREES); + } + + /** One named database-level branch or immutable tag. */ + @Experimental + public String databaseTree(String databaseName, String referenceName) { + return SLASH.join(databaseTrees(databaseName), encodeString(referenceName)); + } + public String tables(String databaseName) { return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLES); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java new file mode 100644 index 000000000000..3fbbc18e7887 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; +import java.util.List; + +/** Paged response for database-level branches and tags. */ +@Experimental +public class ListDatabaseReferencesResponse implements PagedResponse { + + private static final String FIELD_REFERENCES = "references"; + private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken"; + + @JsonProperty(FIELD_REFERENCES) + private final List references; + + @Nullable + @JsonProperty(FIELD_NEXT_PAGE_TOKEN) + private final String nextPageToken; + + @JsonCreator + @ConstructorProperties({FIELD_REFERENCES, FIELD_NEXT_PAGE_TOKEN}) + public ListDatabaseReferencesResponse( + @JsonProperty(FIELD_REFERENCES) List references, + @Nullable @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) { + this.references = references; + this.nextPageToken = nextPageToken; + } + + @JsonGetter(FIELD_REFERENCES) + public List getReferences() { + return references; + } + + @Nullable + @JsonGetter(FIELD_NEXT_PAGE_TOKEN) + @Override + public String getNextPageToken() { + return nextPageToken; + } + + @Override + public List data() { + return references; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java new file mode 100644 index 000000000000..1bbda053890b --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Response containing one database-level reference. */ +@Experimental +public class SingleDatabaseReferenceResponse implements RESTResponse { + + private static final String FIELD_REFERENCE = "reference"; + + @JsonProperty(FIELD_REFERENCE) + private final DatabaseReference reference; + + @JsonCreator + @ConstructorProperties({FIELD_REFERENCE}) + public SingleDatabaseReferenceResponse( + @JsonProperty(FIELD_REFERENCE) DatabaseReference reference) { + this.reference = reference; + } + + @JsonGetter(FIELD_REFERENCE) + public DatabaseReference getReference() { + return reference; + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java new file mode 100644 index 000000000000..2ccae1011bce --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.options.Options; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.apache.paimon.rest.RESTCatalogInternalOptions.PREFIX; +import static org.apache.paimon.rest.RESTCatalogOptions.DATABASE_REFERENCE; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.URI; +import static org.assertj.core.api.Assertions.assertThat; + +/** HTTP contract tests for database-level branches and immutable tags. */ +class RESTApiDatabaseReferenceTest { + + private static final String TREES_PATH = "/v1/catalog%2Fid/databases/training+db/trees"; + + private final Queue replies = new ConcurrentLinkedQueue<>(); + private final List requests = new CopyOnWriteArrayList<>(); + + private HttpServer server; + private RESTApi api; + + @BeforeEach + void setUp() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/v1/", + exchange -> { + requests.add(new Request(exchange)); + Reply reply = replies.poll(); + if (reply == null) { + reply = new Reply(500, "{\"code\":500,\"message\":\"unexpected request\"}"); + } + byte[] data = reply.body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(reply.code, data.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(data); + } finally { + exchange.close(); + } + }); + server.start(); + + Options options = new Options(); + options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); + options.set(PREFIX, "catalog/id"); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "test-token"); + api = new RESTApi(options, false); + } + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(0); + } + } + + @Test + void testBranchAndImmutableTagHappyPath() throws Exception { + enqueue( + 200, + "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}," + + "{\"type\":\"TAG\",\"name\":\"train-v1\"}]," + + "\"nextPageToken\":\"next\"}"); + PagedList page = + api.listDatabaseReferencesPaged( + "training db", DatabaseReferenceType.TAG, 100, "start token"); + assertThat(page.getElements()) + .containsExactly( + new DatabaseReference(DatabaseReferenceType.BRANCH, "main"), + new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); + assertThat(page.getNextPageToken()).isEqualTo("next"); + assertRequest(0, "GET", TREES_PATH); + assertThat(queryParameters(requests.get(0).query)) + .containsEntry("type", "tag") + .containsEntry("maxResults", "100") + .containsEntry("pageToken", "start token"); + + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); + assertThat(api.getDatabaseReference("training db", "main")) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + assertRequest(1, "GET", TREES_PATH + "/main"); + + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); + DatabaseReference branch = + api.createDatabaseReference( + "training db", + "exp-1", + DatabaseReferenceType.BRANCH, + new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + assertThat(branch).isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); + assertRequest(2, "POST", TREES_PATH); + assertThat(queryParameters(requests.get(2).query)) + .containsEntry("name", "exp-1") + .containsEntry("type", "branch"); + assertReferenceBody(requests.get(2), "BRANCH", "main"); + + enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); + DatabaseReference tag = + api.createDatabaseReference( + "training db", + "train-v1", + DatabaseReferenceType.TAG, + new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); + assertThat(tag).isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); + assertRequest(3, "POST", TREES_PATH); + assertThat(queryParameters(requests.get(3).query)) + .containsEntry("name", "train-v1") + .containsEntry("type", "tag"); + assertReferenceBody(requests.get(3), "BRANCH", "exp-1"); + + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); + assertThat(api.fastForwardDatabaseBranch("training db", "main", "train-v1")) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + assertRequest(4, "PUT", TREES_PATH + "/main"); + assertThat(queryParameters(requests.get(4).query)) + .containsEntry("mode", "FAST_FORWARD") + .containsEntry("type", "branch"); + assertReferenceBody(requests.get(4), "TAG", "train-v1"); + + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); + assertThat( + api.deleteDatabaseReference( + "training db", "exp-1", DatabaseReferenceType.BRANCH)) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); + assertRequest(5, "DELETE", TREES_PATH + "/exp-1"); + assertThat(queryParameters(requests.get(5).query)).containsEntry("type", "branch"); + assertThat(requests.get(5).body).isEmpty(); + } + + @Test + void testListAllReferencesFollowsPages() { + enqueue( + 200, + "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}]," + + "\"nextPageToken\":\"p2\"}"); + enqueue(200, "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"exp-1\"}]}"); + + assertThat(api.listDatabaseReferences("training db", DatabaseReferenceType.BRANCH)) + .extracting(DatabaseReference::getName) + .containsExactly("main", "exp-1"); + assertThat(requests).hasSize(2); + assertThat(queryParameters(requests.get(0).query)).containsEntry("type", "branch"); + assertThat(queryParameters(requests.get(1).query)) + .containsEntry("type", "branch") + .containsEntry("pageToken", "p2"); + } + + @Test + void testReferenceOptionIsSentToTableApis() { + enqueue(200, "{\"tables\":[]}"); + Options options = new Options(); + options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); + options.set(PREFIX, "catalog/id"); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "test-token"); + options.set(DATABASE_REFERENCE, "train-v1"); + + new RESTApi(options, false).listTables("training db"); + + assertRequest(0, "GET", "/v1/catalog%2Fid/databases/training+db/tables"); + assertThat(requests.get(0).reference).isEqualTo("train-v1"); + } + + private void enqueue(int code, String body) { + replies.add(new Reply(code, body)); + } + + private void assertRequest(int index, String method, String path) { + Request request = requests.get(index); + assertThat(request.method).isEqualTo(method); + assertThat(request.path).isEqualTo(path); + assertThat(request.authorization).isEqualTo("Bearer test-token"); + } + + private static void assertReferenceBody(Request request, String type, String name) + throws Exception { + assertThat(RESTApi.fromJson(request.body, Map.class)) + .containsEntry("type", type) + .containsEntry("name", name) + .hasSize(2); + } + + private static Map queryParameters(String query) { + Map values = new LinkedHashMap<>(); + if (query == null || query.isEmpty()) { + return values; + } + for (String parameter : query.split("&")) { + String[] pair = parameter.split("=", 2); + values.put(decode(pair[0]), decode(pair[1])); + } + return values; + } + + private static String decode(String value) { + try { + return URLDecoder.decode(value, "UTF-8"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static class Reply { + private final int code; + private final String body; + + private Reply(int code, String body) { + this.code = code; + this.body = body; + } + } + + private static class Request { + private final String method; + private final String path; + private final String query; + private final String body; + private final String authorization; + private final String reference; + + private Request(HttpExchange exchange) throws IOException { + method = exchange.getRequestMethod(); + path = exchange.getRequestURI().getRawPath(); + query = exchange.getRequestURI().getRawQuery(); + body = read(exchange.getRequestBody()); + authorization = exchange.getRequestHeaders().getFirst("Authorization"); + reference = exchange.getRequestHeaders().getFirst("Paimon-Reference"); + } + + private static String read(InputStream input) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = input.read(buffer)) >= 0) { + output.write(buffer, 0, length); + } + return new String(output.toByteArray(), StandardCharsets.UTF_8); + } + } +} From 8532152fd282f420be66e91619ab75b25ece528b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 16:35:34 +0800 Subject: [PATCH 02/12] [rest] Expose tree management through RESTCatalog --- .../paimon/management/TreeManagement.java | 76 ++++++ .../paimon/rest/RESTTreeManagement.java | 73 ++++++ .../org/apache/paimon/rest/RESTCatalog.java | 6 + .../rest/RESTCatalogTreeManagementTest.java | 231 ++++++++++++++++++ 4 files changed, 386 insertions(+) create mode 100644 paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java new file mode 100644 index 000000000000..29d7c3fe776b --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.PagedList; +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.DatabaseReferenceType; + +import javax.annotation.Nullable; + +import java.util.List; + +/** Control-plane contract for database-level writable branches and immutable tags. */ +@Experimental +public interface TreeManagement { + + /** Lists all references, following pagination. A null type includes branches and tags. */ + default List listReferences( + String databaseName, @Nullable DatabaseReferenceType type) { + return PagedList.listAllFromPagedApi( + pageToken -> listReferencesPaged(databaseName, type, null, pageToken)); + } + + /** + * Lists one page of references. + * + * @param type reference type to include; null includes branches and tags + * @param maxResults maximum page size; null or zero uses the server default + * @param pageToken opaque continuation token; null for the first page + */ + PagedList listReferencesPaged( + String databaseName, + @Nullable DatabaseReferenceType type, + @Nullable Integer maxResults, + @Nullable String pageToken); + + /** Gets a named branch or tag. A missing reference is an error. */ + DatabaseReference getReference(String databaseName, String referenceName); + + /** Creates a branch or immutable tag from an existing reference in the same database. */ + DatabaseReference createReference( + String databaseName, + String referenceName, + DatabaseReferenceType type, + DatabaseReference source); + + /** Fast-forwards a branch to an immutable tag in the same database. */ + DatabaseReference fastForwardBranch(String databaseName, String targetBranch, String sourceTag); + + /** + * Deletes and returns a named reference. A missing reference is an error. + * + * @param expectedType required type of the reference to delete; null omits the type check + */ + DatabaseReference deleteReference( + String databaseName, + String referenceName, + @Nullable DatabaseReferenceType expectedType); +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java new file mode 100644 index 000000000000..2be049fb8e02 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.TreeManagement; + +import javax.annotation.Nullable; + +/** REST implementation of tree management, bound to the configured REST catalog prefix. */ +@Experimental +public class RESTTreeManagement implements TreeManagement { + + private final RESTApi api; + + public RESTTreeManagement(RESTApi api) { + this.api = api; + } + + @Override + public PagedList listReferencesPaged( + String databaseName, + @Nullable DatabaseReferenceType type, + @Nullable Integer maxResults, + @Nullable String pageToken) { + return api.listDatabaseReferencesPaged(databaseName, type, maxResults, pageToken); + } + + @Override + public DatabaseReference getReference(String databaseName, String referenceName) { + return api.getDatabaseReference(databaseName, referenceName); + } + + @Override + public DatabaseReference createReference( + String databaseName, + String referenceName, + DatabaseReferenceType type, + DatabaseReference source) { + return api.createDatabaseReference(databaseName, referenceName, type, source); + } + + @Override + public DatabaseReference fastForwardBranch( + String databaseName, String targetBranch, String sourceTag) { + return api.fastForwardDatabaseBranch(databaseName, targetBranch, sourceTag); + } + + @Override + public DatabaseReference deleteReference( + String databaseName, + String referenceName, + @Nullable DatabaseReferenceType expectedType) { + return api.deleteDatabaseReference(databaseName, referenceName, expectedType); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index f691fc06f5b7..58ffc5edb518 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -44,6 +44,7 @@ import org.apache.paimon.management.PermissionManagement; import org.apache.paimon.management.PolicyManagement; import org.apache.paimon.management.SemanticViewManagement; +import org.apache.paimon.management.TreeManagement; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionStatistics; @@ -166,6 +167,11 @@ public SemanticViewManagement semanticViewManagement() { return new RESTSemanticViewManagement(api); } + @Experimental + public TreeManagement treeManagement() { + return new RESTTreeManagement(api); + } + @Override public List listDatabases() { return api.listDatabases(); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java new file mode 100644 index 000000000000..adb307836e76 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.management.TreeManagement; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; +import org.apache.paimon.rest.exceptions.NotImplementedException; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.paimon.options.CatalogOptions.WAREHOUSE; +import static org.apache.paimon.rest.DatabaseReferenceType.BRANCH; +import static org.apache.paimon.rest.DatabaseReferenceType.TAG; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.URI; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Exercises database tree management through a configured REST catalog and its HTTP client. */ +class RESTCatalogTreeManagementTest { + + private static final String DATABASE = "training db"; + private static final String TREES_PATH = "/v1/catalog%2Fid/databases/training+db/trees"; + private static final String MAIN_JSON = "{\"type\":\"BRANCH\",\"name\":\"main\"}"; + private static final String BRANCH_JSON = "{\"type\":\"BRANCH\",\"name\":\"exp-1\"}"; + private static final String TAG_JSON = "{\"type\":\"TAG\",\"name\":\"train-v1\"}"; + + private MockWebServer server; + private RESTCatalog catalog; + private TreeManagement trees; + + @BeforeEach + void setUp() throws Exception { + server = new MockWebServer(); + server.start(); + enqueue( + 200, + "{\"defaults\":{},\"overrides\":{\"prefix\":\"catalog/id\"," + + "\"header.X-Catalog-Context\":\"configured\"}}"); + + Options options = new Options(); + options.set(URI, server.url("/").toString()); + options.set(WAREHOUSE, "warehouse-id"); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "test-token"); + catalog = new RESTCatalog(CatalogContext.create(options)); + trees = catalog.treeManagement(); + + RecordedRequest config = server.takeRequest(10, TimeUnit.SECONDS); + assertThat(config).isNotNull(); + assertThat(config.getRequestUrl().encodedPath()).isEqualTo("/v1/config"); + assertThat(config.getRequestUrl().queryParameter("warehouse")).isEqualTo("warehouse-id"); + assertThat(server.getRequestCount()).isEqualTo(1); + } + + @AfterEach + void tearDown() throws Exception { + if (catalog != null) { + catalog.close(); + } + if (server != null) { + server.shutdown(); + } + } + + @Test + void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { + DatabaseReference main = new DatabaseReference(BRANCH, "main"); + DatabaseReference branch = new DatabaseReference(BRANCH, "exp-1"); + DatabaseReference tag = new DatabaseReference(TAG, "train-v1"); + + enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); + assertThat(trees.getReference(DATABASE, "main")).isEqualTo(main); + takeRequest("GET", TREES_PATH + "/main"); + + enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); + assertThat(trees.createReference(DATABASE, "exp-1", BRANCH, main)).isEqualTo(branch); + RecordedRequest createBranch = takeRequest("POST", TREES_PATH); + assertThat(createBranch.getRequestUrl().queryParameter("name")).isEqualTo("exp-1"); + assertThat(createBranch.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertReferenceBody(createBranch, "BRANCH", "main"); + + enqueue(200, "{\"reference\":" + TAG_JSON + "}"); + assertThat(trees.createReference(DATABASE, "train-v1", TAG, branch)).isEqualTo(tag); + RecordedRequest createTag = takeRequest("POST", TREES_PATH); + assertThat(createTag.getRequestUrl().queryParameter("name")).isEqualTo("train-v1"); + assertThat(createTag.getRequestUrl().queryParameter("type")).isEqualTo("tag"); + assertReferenceBody(createTag, "BRANCH", "exp-1"); + + enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); + assertThat(trees.fastForwardBranch(DATABASE, "main", "train-v1")).isEqualTo(main); + RecordedRequest fastForward = takeRequest("PUT", TREES_PATH + "/main"); + assertThat(fastForward.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertThat(fastForward.getRequestUrl().queryParameter("mode")).isEqualTo("FAST_FORWARD"); + assertReferenceBody(fastForward, "TAG", "train-v1"); + + enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); + assertThat(trees.deleteReference(DATABASE, "exp-1", BRANCH)).isEqualTo(branch); + RecordedRequest deleteBranch = takeRequest("DELETE", TREES_PATH + "/exp-1"); + assertThat(deleteBranch.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertThat(deleteBranch.getBodySize()).isZero(); + + enqueue(200, "{\"reference\":" + TAG_JSON + "}"); + assertThat(trees.deleteReference(DATABASE, "train-v1", null)).isEqualTo(tag); + assertThat(takeRequest("DELETE", TREES_PATH + "/train-v1").getRequestUrl().query()) + .isNull(); + assertThat(server.getRequestCount()).isEqualTo(7); + } + + @Test + void testListPageAndListAllPreserveFilterAndTokens() throws Exception { + enqueue(200, "{\"references\":[" + TAG_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); + PagedList page = + trees.listReferencesPaged(DATABASE, TAG, 10, "start +/%"); + assertThat(page.getElements()).containsExactly(new DatabaseReference(TAG, "train-v1")); + assertThat(page.getNextPageToken()).isEqualTo("next +/%?&"); + RecordedRequest paged = takeRequest("GET", TREES_PATH); + assertThat(paged.getRequestUrl().queryParameter("type")).isEqualTo("tag"); + assertThat(paged.getRequestUrl().queryParameter("maxResults")).isEqualTo("10"); + assertThat(paged.getRequestUrl().queryParameter("pageToken")).isEqualTo("start +/%"); + + enqueue(200, "{\"references\":[" + MAIN_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); + enqueue(200, "{\"references\":[" + BRANCH_JSON + "]}"); + assertThat(trees.listReferences(DATABASE, BRANCH)) + .containsExactly( + new DatabaseReference(BRANCH, "main"), + new DatabaseReference(BRANCH, "exp-1")); + RecordedRequest first = takeRequest("GET", TREES_PATH); + assertThat(first.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertThat(first.getRequestUrl().queryParameter("pageToken")).isNull(); + RecordedRequest second = takeRequest("GET", TREES_PATH); + assertThat(second.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertThat(second.getRequestUrl().queryParameter("pageToken")).isEqualTo("next +/%?&"); + assertThat(second.getRequestUrl().queryParameter("maxResults")).isNull(); + } + + @Test + void testListAllTypesAndEmptyReferences() throws Exception { + enqueue(200, "{\"references\":[" + MAIN_JSON + "," + TAG_JSON + "]}"); + assertThat(trees.listReferences(DATABASE, null)) + .containsExactly( + new DatabaseReference(BRANCH, "main"), + new DatabaseReference(TAG, "train-v1")); + assertThat(takeRequest("GET", TREES_PATH).getRequestUrl().query()).isNull(); + + enqueue(200, "{\"references\":[]}"); + assertThat(trees.listReferences(DATABASE, null)).isEmpty(); + takeRequest("GET", TREES_PATH); + assertThat(server.getRequestCount()).isEqualTo(3); + } + + @Test + void testErrorsPropagate() { + enqueue(404, "{\"code\":404,\"message\":\"reference missing\"}"); + assertThatThrownBy(() -> trees.getReference(DATABASE, "missing")) + .isInstanceOf(NoSuchResourceException.class) + .hasMessageContaining("reference missing"); + + enqueue(409, "{\"code\":409,\"message\":\"reference already exists\"}"); + assertThatThrownBy( + () -> + trees.createReference( + DATABASE, + "exp-1", + BRANCH, + new DatabaseReference(BRANCH, "main"))) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageContaining("reference already exists"); + + enqueue(501, "{\"code\":501,\"message\":\"trees unsupported\"}"); + assertThatThrownBy(() -> trees.listReferences(DATABASE, null)) + .isInstanceOf(NotImplementedException.class) + .hasMessageContaining("trees unsupported"); + assertThat(server.getRequestCount()).isEqualTo(4); + } + + private void enqueue(int status, String body) { + server.enqueue( + new MockResponse() + .setResponseCode(status) + .setHeader("Content-Type", "application/json") + .setBody(body)); + } + + private RecordedRequest takeRequest(String method, String path) throws Exception { + RecordedRequest request = server.takeRequest(10, TimeUnit.SECONDS); + assertThat(request).isNotNull(); + assertThat(request.getMethod()).isEqualTo(method); + assertThat(request.getRequestUrl().encodedPath()).isEqualTo(path); + assertThat(request.getHeader("Authorization")).isEqualTo("Bearer test-token"); + assertThat(request.getHeader("X-Catalog-Context")).isEqualTo("configured"); + return request; + } + + private static void assertReferenceBody(RecordedRequest request, String type, String name) + throws Exception { + assertThat(RESTApi.fromJson(request.getBody().readUtf8(), Map.class)) + .containsEntry("type", type) + .containsEntry("name", name) + .hasSize(2); + } +} From 87de0554252c742d7fbf34046424f84a8658f05e Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 16:45:26 +0800 Subject: [PATCH 03/12] [rest] Simplify database reference requests and pagination --- docs/static/rest-catalog-open-api.yaml | 95 +++++++++---------- .../paimon/management/TreeManagement.java | 9 -- .../apache/paimon/rest/DatabaseReference.java | 2 +- .../org/apache/paimon/rest/HttpClient.java | 28 ++---- .../java/org/apache/paimon/rest/RESTApi.java | 60 +++--------- .../paimon/rest/RESTCatalogOptions.java | 8 -- .../CreateDatabaseReferenceRequest.java | 71 ++++++++++++++ .../DeleteDatabaseReferenceRequest.java | 57 +++++++++++ .../FastForwardDatabaseBranchRequest.java | 50 ++++++++++ ...se.java => DatabaseReferenceResponse.java} | 5 +- .../rest/RESTApiDatabaseReferenceTest.java | 73 +++++++------- .../RequestJacksonCompatibilityTest.java | 42 ++++++++ .../rest/RESTCatalogTreeManagementTest.java | 54 ++++++----- 13 files changed, 353 insertions(+), 201 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java rename paimon-api/src/main/java/org/apache/paimon/rest/responses/{SingleDatabaseReferenceResponse.java => DatabaseReferenceResponse.java} (90%) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 24964106bc00..98c1350d0b52 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -208,7 +208,7 @@ paths: tags: - database-reference summary: List database references - operationId: listDatabaseReferences + operationId: listDatabaseReferencesPaged parameters: - name: prefix in: path @@ -266,30 +266,19 @@ paths: required: true schema: type: string - - name: name - in: query - required: true - schema: - type: string - - name: type - in: query - required: true - schema: - type: string - enum: [ "branch", "tag" ] requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/DatabaseReference' + $ref: '#/components/schemas/CreateDatabaseReferenceRequest' responses: "200": description: Created branch or immutable tag. content: application/json: schema: - $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + $ref: '#/components/schemas/DatabaseReferenceResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -330,7 +319,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + $ref: '#/components/schemas/DatabaseReferenceResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -362,31 +351,19 @@ paths: required: true schema: type: string - - name: mode - in: query - required: true - schema: - type: string - const: "FAST_FORWARD" - - name: type - in: query - required: false - schema: - type: string - const: "branch" requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/DatabaseReference' + $ref: '#/components/schemas/FastForwardDatabaseBranchRequest' responses: "200": description: Updated target branch. content: application/json: schema: - $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + $ref: '#/components/schemas/DatabaseReferenceResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -424,19 +401,19 @@ paths: required: true schema: type: string - - name: type - in: query - required: false - schema: - type: string - enum: [ "branch", "tag" ] + requestBody: + required: false + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteDatabaseReferenceRequest' responses: "200": description: Deleted branch or immutable tag. content: application/json: schema: - $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + $ref: '#/components/schemas/DatabaseReferenceResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -519,7 +496,6 @@ paths: in: query schema: type: string - - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -712,7 +688,6 @@ paths: required: true schema: type: string - - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -847,7 +822,6 @@ paths: required: true schema: type: string - - $ref: '#/components/parameters/DatabaseReferenceHeader' requestBody: content: application/json: @@ -1063,7 +1037,6 @@ paths: required: true schema: type: string - - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -2660,15 +2633,6 @@ paths: $ref: '#/components/responses/SemanticViewNotImplementedErrorResponse' components: - parameters: - DatabaseReferenceHeader: - name: Paimon-Reference - in: header - required: false - description: Database-level branch or immutable tag. Required for a versioned database. - schema: - type: string - pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" ############################# # Reusable Response Objects # ############################# @@ -4028,7 +3992,38 @@ components: name: type: string pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" - SingleDatabaseReferenceResponse: + CreateDatabaseReferenceRequest: + type: object + required: + - name + - type + - source + properties: + name: + type: string + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + type: + type: string + enum: [ "BRANCH", "TAG" ] + source: + $ref: '#/components/schemas/DatabaseReference' + FastForwardDatabaseBranchRequest: + type: object + required: + - sourceTag + properties: + sourceTag: + type: string + description: Immutable tag in the same database to fast-forward the target branch to. + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + DeleteDatabaseReferenceRequest: + type: object + properties: + type: + type: string + description: Expected reference type. Omit to delete without checking the type. + enum: [ "BRANCH", "TAG" ] + DatabaseReferenceResponse: type: object required: - reference diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java index 29d7c3fe776b..4ca5dd9a26e7 100644 --- a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -25,19 +25,10 @@ import javax.annotation.Nullable; -import java.util.List; - /** Control-plane contract for database-level writable branches and immutable tags. */ @Experimental public interface TreeManagement { - /** Lists all references, following pagination. A null type includes branches and tags. */ - default List listReferences( - String databaseName, @Nullable DatabaseReferenceType type) { - return PagedList.listAllFromPagedApi( - pageToken -> listReferencesPaged(databaseName, type, null, pageToken)); - } - /** * Lists one page of references. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java index 416dff1749ce..845a68deb003 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java @@ -31,7 +31,7 @@ /** A named database-level branch or immutable tag. */ @Experimental -public class DatabaseReference implements RESTRequest { +public class DatabaseReference { private static final String FIELD_TYPE = "type"; private static final String FIELD_NAME = "name"; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java index e30ecab35f3f..57f44a93fa03 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -98,21 +98,12 @@ public T post( RESTRequest body, Class responseType, RESTAuthFunction restAuthFunction) { - return post(path, Collections.emptyMap(), body, responseType, restAuthFunction); - } - - public T post( - String path, - Map queryParams, - RESTRequest body, - Class responseType, - RESTAuthFunction restAuthFunction) { - HttpPost httpPost = HttpClientUtils.newHttpPost(getRequestUrl(path, queryParams)); + HttpPost httpPost = HttpClientUtils.newHttpPost(getRequestUrl(path, null)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpPost.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = getHeaders(path, queryParams, "POST", encodedBody, restAuthFunction); + Header[] authHeaders = getHeaders(path, "POST", encodedBody, restAuthFunction); httpPost.setHeaders(authHeaders); // A POST the server cannot absorb twice is sent exactly once, whatever the status says. return exec( @@ -125,16 +116,15 @@ public T post( public T put( String path, - Map queryParams, RESTRequest body, Class responseType, RESTAuthFunction restAuthFunction) { - HttpPut httpPut = HttpClientUtils.newHttpPut(getRequestUrl(path, queryParams)); + HttpPut httpPut = HttpClientUtils.newHttpPut(getRequestUrl(path, null)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpPut.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = getHeaders(path, queryParams, "PUT", encodedBody, restAuthFunction); + Header[] authHeaders = getHeaders(path, "PUT", encodedBody, restAuthFunction); httpPut.setHeaders(authHeaders); return exec( httpPut, @@ -146,28 +136,26 @@ public T put( @Override public T delete(String path, RESTAuthFunction restAuthFunction) { - return delete(path, Collections.emptyMap(), null, null, restAuthFunction); + return delete(path, null, null, restAuthFunction); } @Override public T delete( String path, RESTRequest body, RESTAuthFunction restAuthFunction) { - return delete(path, Collections.emptyMap(), body, null, restAuthFunction); + return delete(path, body, null, restAuthFunction); } public T delete( String path, - Map queryParams, RESTRequest body, Class responseType, RESTAuthFunction restAuthFunction) { - HttpDelete httpDelete = HttpClientUtils.newHttpDelete(getRequestUrl(path, queryParams)); + HttpDelete httpDelete = HttpClientUtils.newHttpDelete(getRequestUrl(path, null)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpDelete.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = - getHeaders(path, queryParams, "DELETE", encodedBody, restAuthFunction); + Header[] authHeaders = getHeaders(path, "DELETE", encodedBody, restAuthFunction); httpDelete.setHeaders(authHeaders); return exec(httpDelete, responseType); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 64b5aeab8873..44018af6c895 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -47,14 +47,17 @@ import org.apache.paimon.rest.requests.AuthTableQueryRequest; import org.apache.paimon.rest.requests.CommitTableRequest; import org.apache.paimon.rest.requests.CreateBranchRequest; +import org.apache.paimon.rest.requests.CreateDatabaseReferenceRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreateFunctionRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; +import org.apache.paimon.rest.requests.DeleteDatabaseReferenceRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.DropPolicyRequest; +import org.apache.paimon.rest.requests.FastForwardDatabaseBranchRequest; import org.apache.paimon.rest.requests.ForwardBranchRequest; import org.apache.paimon.rest.requests.GrantPermissionRequest; import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; @@ -75,6 +78,7 @@ import org.apache.paimon.rest.responses.CommitTableResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DatabaseReferenceResponse; import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; @@ -110,7 +114,6 @@ import org.apache.paimon.rest.responses.ListViewsGloballyResponse; import org.apache.paimon.rest.responses.ListViewsResponse; import org.apache.paimon.rest.responses.PagedResponse; -import org.apache.paimon.rest.responses.SingleDatabaseReferenceResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; @@ -196,10 +199,7 @@ public class RESTApi { public static final String PARTITION_NAME_PATTERN = "partitionNamePattern"; public static final String TAG_NAME_PREFIX = "tagNamePrefix"; - private static final String REFERENCE_NAME = "name"; private static final String REFERENCE_TYPE = "type"; - private static final String REFERENCE_UPDATE_MODE = "mode"; - private static final String FAST_FORWARD = "FAST_FORWARD"; public static final long TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000L; @@ -367,23 +367,6 @@ public void alterDatabase(String name, List removals, Map listDatabaseReferences( - String databaseName, @Nullable DatabaseReferenceType type) { - return listDataFromPageApi( - queryParams -> { - if (type != null) { - queryParams.put(REFERENCE_TYPE, type.queryValue()); - } - return client.get( - resourcePaths.databaseTrees(databaseName), - queryParams, - ListDatabaseReferencesResponse.class, - restAuthFunction); - }); - } - /** List one page of database-level branches and immutable tags. */ @Experimental public PagedList listDatabaseReferencesPaged( @@ -409,10 +392,10 @@ public PagedList listDatabaseReferencesPaged( /** Get one database-level branch or immutable tag. */ @Experimental public DatabaseReference getDatabaseReference(String databaseName, String referenceName) { - SingleDatabaseReferenceResponse response = + DatabaseReferenceResponse response = client.get( resourcePaths.databaseTree(databaseName, referenceName), - SingleDatabaseReferenceResponse.class, + DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); } @@ -424,15 +407,11 @@ public DatabaseReference createDatabaseReference( String referenceName, DatabaseReferenceType type, DatabaseReference source) { - Map queryParams = Maps.newHashMap(); - queryParams.put(REFERENCE_NAME, referenceName); - queryParams.put(REFERENCE_TYPE, type.queryValue()); - SingleDatabaseReferenceResponse response = + DatabaseReferenceResponse response = client.post( resourcePaths.databaseTrees(databaseName), - queryParams, - source, - SingleDatabaseReferenceResponse.class, + new CreateDatabaseReferenceRequest(referenceName, type, source), + DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); } @@ -441,15 +420,11 @@ public DatabaseReference createDatabaseReference( @Experimental public DatabaseReference fastForwardDatabaseBranch( String databaseName, String targetBranch, String sourceTag) { - Map queryParams = Maps.newHashMap(); - queryParams.put(REFERENCE_UPDATE_MODE, FAST_FORWARD); - queryParams.put(REFERENCE_TYPE, DatabaseReferenceType.BRANCH.queryValue()); - SingleDatabaseReferenceResponse response = + DatabaseReferenceResponse response = client.put( resourcePaths.databaseTree(databaseName, targetBranch), - queryParams, - new DatabaseReference(DatabaseReferenceType.TAG, sourceTag), - SingleDatabaseReferenceResponse.class, + new FastForwardDatabaseBranchRequest(sourceTag), + DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); } @@ -460,16 +435,11 @@ public DatabaseReference deleteDatabaseReference( String databaseName, String referenceName, @Nullable DatabaseReferenceType expectedType) { - Map queryParams = Maps.newHashMap(); - if (expectedType != null) { - queryParams.put(REFERENCE_TYPE, expectedType.queryValue()); - } - SingleDatabaseReferenceResponse response = + DatabaseReferenceResponse response = client.delete( resourcePaths.databaseTree(databaseName, referenceName), - queryParams, - null, - SingleDatabaseReferenceResponse.class, + new DeleteDatabaseReferenceRequest(expectedType), + DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index 3629907dc65f..e7da1f6827dc 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -98,14 +98,6 @@ public class RESTCatalogOptions { .withDescription( "The user agent of http client connecting to REST Catalog server."); - /** Database-level branch or immutable tag carried by table API requests. */ - public static final ConfigOption DATABASE_REFERENCE = - ConfigOptions.key("header.Paimon-Reference") - .stringType() - .noDefaultValue() - .withDescription( - "Database-level branch or immutable tag used by REST table requests."); - public static final ConfigOption DLF_OSS_ENDPOINT = ConfigOptions.key("dlf.oss-endpoint") .stringType() diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java new file mode 100644 index 000000000000..cb70e45dc471 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Request for creating a database branch or immutable tag from an existing reference. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class CreateDatabaseReferenceRequest implements RESTRequest { + + private static final String FIELD_NAME = "name"; + private static final String FIELD_TYPE = "type"; + private static final String FIELD_SOURCE = "source"; + + private final String name; + private final DatabaseReferenceType type; + private final DatabaseReference source; + + @JsonCreator + @ConstructorProperties({FIELD_NAME, FIELD_TYPE, FIELD_SOURCE}) + public CreateDatabaseReferenceRequest( + @JsonProperty(FIELD_NAME) String name, + @JsonProperty(FIELD_TYPE) DatabaseReferenceType type, + @JsonProperty(FIELD_SOURCE) DatabaseReference source) { + this.name = name; + this.type = type; + this.source = source; + } + + @JsonGetter(FIELD_NAME) + public String getName() { + return name; + } + + @JsonGetter(FIELD_TYPE) + public DatabaseReferenceType getType() { + return type; + } + + @JsonGetter(FIELD_SOURCE) + public DatabaseReference getSource() { + return source; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java new file mode 100644 index 000000000000..a7193b9b608e --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; + +/** Request for deleting a database reference, optionally checking its type. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class DeleteDatabaseReferenceRequest implements RESTRequest { + + private static final String FIELD_TYPE = "type"; + + @Nullable private final DatabaseReferenceType type; + + @JsonCreator + @ConstructorProperties({FIELD_TYPE}) + public DeleteDatabaseReferenceRequest( + @Nullable @JsonProperty(FIELD_TYPE) DatabaseReferenceType type) { + this.type = type; + } + + @Nullable + @JsonGetter(FIELD_TYPE) + @JsonInclude(JsonInclude.Include.NON_NULL) + public DatabaseReferenceType getType() { + return type; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java new file mode 100644 index 000000000000..f05b56d35458 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Request for fast-forwarding a database branch to an immutable tag. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class FastForwardDatabaseBranchRequest implements RESTRequest { + + private static final String FIELD_SOURCE_TAG = "sourceTag"; + + private final String sourceTag; + + @JsonCreator + @ConstructorProperties({FIELD_SOURCE_TAG}) + public FastForwardDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE_TAG) String sourceTag) { + this.sourceTag = sourceTag; + } + + @JsonGetter(FIELD_SOURCE_TAG) + public String getSourceTag() { + return sourceTag; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java similarity index 90% rename from paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java rename to paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java index 1bbda053890b..d8570559fdd7 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java @@ -30,7 +30,7 @@ /** Response containing one database-level reference. */ @Experimental -public class SingleDatabaseReferenceResponse implements RESTResponse { +public class DatabaseReferenceResponse implements RESTResponse { private static final String FIELD_REFERENCE = "reference"; @@ -39,8 +39,7 @@ public class SingleDatabaseReferenceResponse implements RESTResponse { @JsonCreator @ConstructorProperties({FIELD_REFERENCE}) - public SingleDatabaseReferenceResponse( - @JsonProperty(FIELD_REFERENCE) DatabaseReference reference) { + public DatabaseReferenceResponse(@JsonProperty(FIELD_REFERENCE) DatabaseReference reference) { this.reference = reference; } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index 2ccae1011bce..260d1140f4e0 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -42,7 +42,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import static org.apache.paimon.rest.RESTCatalogInternalOptions.PREFIX; -import static org.apache.paimon.rest.RESTCatalogOptions.DATABASE_REFERENCE; import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; import static org.apache.paimon.rest.RESTCatalogOptions.URI; @@ -131,10 +130,10 @@ void testBranchAndImmutableTagHappyPath() throws Exception { new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); assertThat(branch).isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); assertRequest(2, "POST", TREES_PATH); - assertThat(queryParameters(requests.get(2).query)) - .containsEntry("name", "exp-1") - .containsEntry("type", "branch"); - assertReferenceBody(requests.get(2), "BRANCH", "main"); + assertBody( + requests.get(2), + "{\"name\":\"exp-1\",\"type\":\"BRANCH\"," + + "\"source\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); DatabaseReference tag = @@ -145,19 +144,16 @@ void testBranchAndImmutableTagHappyPath() throws Exception { new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); assertThat(tag).isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); assertRequest(3, "POST", TREES_PATH); - assertThat(queryParameters(requests.get(3).query)) - .containsEntry("name", "train-v1") - .containsEntry("type", "tag"); - assertReferenceBody(requests.get(3), "BRANCH", "exp-1"); + assertBody( + requests.get(3), + "{\"name\":\"train-v1\",\"type\":\"TAG\"," + + "\"source\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); assertThat(api.fastForwardDatabaseBranch("training db", "main", "train-v1")) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); assertRequest(4, "PUT", TREES_PATH + "/main"); - assertThat(queryParameters(requests.get(4).query)) - .containsEntry("mode", "FAST_FORWARD") - .containsEntry("type", "branch"); - assertReferenceBody(requests.get(4), "TAG", "train-v1"); + assertBody(requests.get(4), "{\"sourceTag\":\"train-v1\"}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); assertThat( @@ -165,21 +161,31 @@ void testBranchAndImmutableTagHappyPath() throws Exception { "training db", "exp-1", DatabaseReferenceType.BRANCH)) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); assertRequest(5, "DELETE", TREES_PATH + "/exp-1"); - assertThat(queryParameters(requests.get(5).query)).containsEntry("type", "branch"); - assertThat(requests.get(5).body).isEmpty(); + assertBody(requests.get(5), "{\"type\":\"BRANCH\"}"); } @Test - void testListAllReferencesFollowsPages() { + void testListReferencesPaged() { enqueue( 200, "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}]," + "\"nextPageToken\":\"p2\"}"); enqueue(200, "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"exp-1\"}]}"); - assertThat(api.listDatabaseReferences("training db", DatabaseReferenceType.BRANCH)) - .extracting(DatabaseReference::getName) - .containsExactly("main", "exp-1"); + PagedList first = + api.listDatabaseReferencesPaged( + "training db", DatabaseReferenceType.BRANCH, 1, null); + assertThat(first.getElements()) + .containsExactly(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + assertThat(first.getNextPageToken()).isEqualTo("p2"); + assertThat(requests).hasSize(1); + + PagedList second = + api.listDatabaseReferencesPaged( + "training db", DatabaseReferenceType.BRANCH, 1, first.getNextPageToken()); + assertThat(second.getElements()) + .containsExactly(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); + assertThat(second.getNextPageToken()).isNull(); assertThat(requests).hasSize(2); assertThat(queryParameters(requests.get(0).query)).containsEntry("type", "branch"); assertThat(queryParameters(requests.get(1).query)) @@ -188,19 +194,12 @@ void testListAllReferencesFollowsPages() { } @Test - void testReferenceOptionIsSentToTableApis() { - enqueue(200, "{\"tables\":[]}"); - Options options = new Options(); - options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); - options.set(PREFIX, "catalog/id"); - options.set(TOKEN_PROVIDER, "bear"); - options.set(TOKEN, "test-token"); - options.set(DATABASE_REFERENCE, "train-v1"); - - new RESTApi(options, false).listTables("training db"); - - assertRequest(0, "GET", "/v1/catalog%2Fid/databases/training+db/tables"); - assertThat(requests.get(0).reference).isEqualTo("train-v1"); + void testDeleteReferenceWithoutType() throws Exception { + enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); + assertThat(api.deleteDatabaseReference("training db", "train-v1", null)) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); + assertRequest(0, "DELETE", TREES_PATH + "/train-v1"); + assertBody(requests.get(0), "{}"); } private void enqueue(int code, String body) { @@ -214,12 +213,10 @@ private void assertRequest(int index, String method, String path) { assertThat(request.authorization).isEqualTo("Bearer test-token"); } - private static void assertReferenceBody(Request request, String type, String name) - throws Exception { + private static void assertBody(Request request, String expectedJson) throws Exception { + assertThat(request.query).isNull(); assertThat(RESTApi.fromJson(request.body, Map.class)) - .containsEntry("type", type) - .containsEntry("name", name) - .hasSize(2); + .isEqualTo(RESTApi.fromJson(expectedJson, Map.class)); } private static Map queryParameters(String query) { @@ -258,7 +255,6 @@ private static class Request { private final String query; private final String body; private final String authorization; - private final String reference; private Request(HttpExchange exchange) throws IOException { method = exchange.getRequestMethod(); @@ -266,7 +262,6 @@ private Request(HttpExchange exchange) throws IOException { query = exchange.getRequestURI().getRawQuery(); body = read(exchange.getRequestBody()); authorization = exchange.getRequestHeaders().getFirst("Authorization"); - reference = exchange.getRequestHeaders().getFirst("Paimon-Reference"); } private static String read(InputStream input) throws IOException { diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index e7e8a5517e94..7dec699b4367 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -18,6 +18,8 @@ package org.apache.paimon.rest.requests; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.DatabaseReferenceType; import org.apache.paimon.rest.RESTApi; import org.apache.paimon.rest.RESTRequest; @@ -113,6 +115,11 @@ public class RequestJacksonCompatibilityTest { }, "partitionSpecs", "ignoreIfNotExists"), + requestCase( + FastForwardDatabaseBranchRequest.class, + "{\"sourceTag\":\"train-v1\"}", + request -> assertThat(request.getSourceTag()).isEqualTo("train-v1"), + "sourceTag"), requestCase( ListPartitionsByFilterRequest.class, "{\"filter\":\"dt = '2026-08-24'\"," @@ -171,10 +178,12 @@ public class RequestJacksonCompatibilityTest { AlterTableRequest.class, AlterViewRequest.class, CommitTableRequest.class, + CreateDatabaseReferenceRequest.class, CreateFunctionRequest.class, CreatePartitionsRequest.class, CreateTableRequest.class, CreateViewRequest.class, + DeleteDatabaseReferenceRequest.class, DropPolicyRequest.class, GrantPermissionRequest.class, PolicyRequest.class, @@ -211,6 +220,39 @@ void testConstructorPropertyNamesAndOrder(RequestCase requestCase) { .isEqualTo(requestCase.propertyNames); } + @Test + void testCreateDatabaseReferenceRequestRoundTrips() throws Exception { + String json = + "{\"name\":\"exp-1\",\"type\":\"BRANCH\"," + + "\"source\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"; + CreateDatabaseReferenceRequest request = + EXTERNAL_MAPPER.readValue(json, CreateDatabaseReferenceRequest.class); + CreateDatabaseReferenceRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), CreateDatabaseReferenceRequest.class); + assertThat(roundTrip.getName()).isEqualTo("exp-1"); + assertThat(roundTrip.getType()).isEqualTo(DatabaseReferenceType.BRANCH); + assertThat(roundTrip.getSource()) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); + } + + @Test + void testDeleteDatabaseReferenceRequestRoundTrips() throws Exception { + DeleteDatabaseReferenceRequest request = + EXTERNAL_MAPPER.readValue( + "{\"type\":\"TAG\"}", DeleteDatabaseReferenceRequest.class); + assertThat( + RESTApi.fromJson( + RESTApi.toJson(request), + DeleteDatabaseReferenceRequest.class) + .getType()) + .isEqualTo(DatabaseReferenceType.TAG); + + DeleteDatabaseReferenceRequest withoutType = + EXTERNAL_MAPPER.readValue("{}", DeleteDatabaseReferenceRequest.class); + assertThat(RESTApi.toJson(withoutType)).isEqualTo("{}"); + assertThat(RESTApi.fromJson("{}", DeleteDatabaseReferenceRequest.class).getType()).isNull(); + } + @Test void testRequestCreatorAllowlistsAreComplete() throws Exception { Set> simpleRequests = diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index adb307836e76..b3464f9961e4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -105,39 +105,35 @@ void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); assertThat(trees.createReference(DATABASE, "exp-1", BRANCH, main)).isEqualTo(branch); RecordedRequest createBranch = takeRequest("POST", TREES_PATH); - assertThat(createBranch.getRequestUrl().queryParameter("name")).isEqualTo("exp-1"); - assertThat(createBranch.getRequestUrl().queryParameter("type")).isEqualTo("branch"); - assertReferenceBody(createBranch, "BRANCH", "main"); + assertBody( + createBranch, + "{\"name\":\"exp-1\",\"type\":\"BRANCH\",\"source\":" + MAIN_JSON + "}"); enqueue(200, "{\"reference\":" + TAG_JSON + "}"); assertThat(trees.createReference(DATABASE, "train-v1", TAG, branch)).isEqualTo(tag); RecordedRequest createTag = takeRequest("POST", TREES_PATH); - assertThat(createTag.getRequestUrl().queryParameter("name")).isEqualTo("train-v1"); - assertThat(createTag.getRequestUrl().queryParameter("type")).isEqualTo("tag"); - assertReferenceBody(createTag, "BRANCH", "exp-1"); + assertBody( + createTag, + "{\"name\":\"train-v1\",\"type\":\"TAG\",\"source\":" + BRANCH_JSON + "}"); enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); assertThat(trees.fastForwardBranch(DATABASE, "main", "train-v1")).isEqualTo(main); RecordedRequest fastForward = takeRequest("PUT", TREES_PATH + "/main"); - assertThat(fastForward.getRequestUrl().queryParameter("type")).isEqualTo("branch"); - assertThat(fastForward.getRequestUrl().queryParameter("mode")).isEqualTo("FAST_FORWARD"); - assertReferenceBody(fastForward, "TAG", "train-v1"); + assertBody(fastForward, "{\"sourceTag\":\"train-v1\"}"); enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); assertThat(trees.deleteReference(DATABASE, "exp-1", BRANCH)).isEqualTo(branch); RecordedRequest deleteBranch = takeRequest("DELETE", TREES_PATH + "/exp-1"); - assertThat(deleteBranch.getRequestUrl().queryParameter("type")).isEqualTo("branch"); - assertThat(deleteBranch.getBodySize()).isZero(); + assertBody(deleteBranch, "{\"type\":\"BRANCH\"}"); enqueue(200, "{\"reference\":" + TAG_JSON + "}"); assertThat(trees.deleteReference(DATABASE, "train-v1", null)).isEqualTo(tag); - assertThat(takeRequest("DELETE", TREES_PATH + "/train-v1").getRequestUrl().query()) - .isNull(); + assertBody(takeRequest("DELETE", TREES_PATH + "/train-v1"), "{}"); assertThat(server.getRequestCount()).isEqualTo(7); } @Test - void testListPageAndListAllPreserveFilterAndTokens() throws Exception { + void testListPagesPreserveFilterAndTokens() throws Exception { enqueue(200, "{\"references\":[" + TAG_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); PagedList page = trees.listReferencesPaged(DATABASE, TAG, 10, "start +/%"); @@ -150,13 +146,18 @@ void testListPageAndListAllPreserveFilterAndTokens() throws Exception { enqueue(200, "{\"references\":[" + MAIN_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); enqueue(200, "{\"references\":[" + BRANCH_JSON + "]}"); - assertThat(trees.listReferences(DATABASE, BRANCH)) - .containsExactly( - new DatabaseReference(BRANCH, "main"), - new DatabaseReference(BRANCH, "exp-1")); + PagedList firstPage = + trees.listReferencesPaged(DATABASE, BRANCH, null, null); + assertThat(firstPage.getElements()).containsExactly(new DatabaseReference(BRANCH, "main")); + assertThat(firstPage.getNextPageToken()).isEqualTo("next +/%?&"); RecordedRequest first = takeRequest("GET", TREES_PATH); assertThat(first.getRequestUrl().queryParameter("type")).isEqualTo("branch"); assertThat(first.getRequestUrl().queryParameter("pageToken")).isNull(); + PagedList secondPage = + trees.listReferencesPaged(DATABASE, BRANCH, null, firstPage.getNextPageToken()); + assertThat(secondPage.getElements()) + .containsExactly(new DatabaseReference(BRANCH, "exp-1")); + assertThat(secondPage.getNextPageToken()).isNull(); RecordedRequest second = takeRequest("GET", TREES_PATH); assertThat(second.getRequestUrl().queryParameter("type")).isEqualTo("branch"); assertThat(second.getRequestUrl().queryParameter("pageToken")).isEqualTo("next +/%?&"); @@ -166,14 +167,17 @@ void testListPageAndListAllPreserveFilterAndTokens() throws Exception { @Test void testListAllTypesAndEmptyReferences() throws Exception { enqueue(200, "{\"references\":[" + MAIN_JSON + "," + TAG_JSON + "]}"); - assertThat(trees.listReferences(DATABASE, null)) + assertThat(trees.listReferencesPaged(DATABASE, null, null, null).getElements()) .containsExactly( new DatabaseReference(BRANCH, "main"), new DatabaseReference(TAG, "train-v1")); assertThat(takeRequest("GET", TREES_PATH).getRequestUrl().query()).isNull(); enqueue(200, "{\"references\":[]}"); - assertThat(trees.listReferences(DATABASE, null)).isEmpty(); + PagedList emptyPage = + trees.listReferencesPaged(DATABASE, null, null, null); + assertThat(emptyPage.getElements()).isEmpty(); + assertThat(emptyPage.getNextPageToken()).isNull(); takeRequest("GET", TREES_PATH); assertThat(server.getRequestCount()).isEqualTo(3); } @@ -197,7 +201,7 @@ void testErrorsPropagate() { .hasMessageContaining("reference already exists"); enqueue(501, "{\"code\":501,\"message\":\"trees unsupported\"}"); - assertThatThrownBy(() -> trees.listReferences(DATABASE, null)) + assertThatThrownBy(() -> trees.listReferencesPaged(DATABASE, null, null, null)) .isInstanceOf(NotImplementedException.class) .hasMessageContaining("trees unsupported"); assertThat(server.getRequestCount()).isEqualTo(4); @@ -221,11 +225,9 @@ private RecordedRequest takeRequest(String method, String path) throws Exception return request; } - private static void assertReferenceBody(RecordedRequest request, String type, String name) - throws Exception { + private static void assertBody(RecordedRequest request, String expectedJson) throws Exception { + assertThat(request.getRequestUrl().query()).isNull(); assertThat(RESTApi.fromJson(request.getBody().readUtf8(), Map.class)) - .containsEntry("type", type) - .containsEntry("name", name) - .hasSize(2); + .isEqualTo(RESTApi.fromJson(expectedJson, Map.class)); } } From de4bf3b049aeb36f743c528f1d0c890bff656c2d Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 16:59:35 +0800 Subject: [PATCH 04/12] [rest] Use a dedicated forward path for database branches --- docs/static/rest-catalog-open-api.yaml | 33 ++++++++++--------- .../org/apache/paimon/rest/HttpClient.java | 21 ------------ .../apache/paimon/rest/HttpClientUtils.java | 5 --- .../java/org/apache/paimon/rest/RESTApi.java | 4 +-- .../org/apache/paimon/rest/ResourcePaths.java | 6 ++++ .../rest/RESTApiDatabaseReferenceTest.java | 2 +- .../rest/RESTCatalogTreeManagementTest.java | 2 +- 7 files changed, 27 insertions(+), 46 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 98c1350d0b52..ff452c9147f1 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -330,11 +330,11 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - put: + delete: tags: - database-reference - summary: Fast-forward database branch from immutable tag - operationId: fastForwardDatabaseBranch + summary: Delete database reference + operationId: deleteDatabaseReference parameters: - name: prefix in: path @@ -352,14 +352,14 @@ paths: schema: type: string requestBody: - required: true + required: false content: application/json: schema: - $ref: '#/components/schemas/FastForwardDatabaseBranchRequest' + $ref: '#/components/schemas/DeleteDatabaseReferenceRequest' responses: "200": - description: Updated target branch. + description: Deleted branch or immutable tag. content: application/json: schema: @@ -367,24 +367,25 @@ paths: "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - description: Database, target branch, or source tag does not exist. + description: Database or reference does not exist. content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' "409": - description: Target is not an ancestor of the source tag. + description: Reference type does not match or the default branch is protected. content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - delete: + /v1/{prefix}/databases/{database}/trees/{name}/forward: + post: tags: - database-reference - summary: Delete database reference - operationId: deleteDatabaseReference + summary: Fast-forward database branch from immutable tag + operationId: fastForwardDatabaseBranch parameters: - name: prefix in: path @@ -402,14 +403,14 @@ paths: schema: type: string requestBody: - required: false + required: true content: application/json: schema: - $ref: '#/components/schemas/DeleteDatabaseReferenceRequest' + $ref: '#/components/schemas/FastForwardDatabaseBranchRequest' responses: "200": - description: Deleted branch or immutable tag. + description: Updated target branch. content: application/json: schema: @@ -417,13 +418,13 @@ paths: "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - description: Database or reference does not exist. + description: Database, target branch, or source tag does not exist. content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' "409": - description: Reference type does not match or the default branch is protected. + description: Target is not an ancestor of the source tag. content: application/json: schema: diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java index 57f44a93fa03..925809504a56 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -32,7 +32,6 @@ import org.apache.hc.client5.http.classic.methods.HttpDelete; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpPost; -import org.apache.hc.client5.http.classic.methods.HttpPut; import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; import org.apache.hc.core5.http.ClassicHttpResponse; import org.apache.hc.core5.http.ContentType; @@ -114,26 +113,6 @@ public T post( : null); } - public T put( - String path, - RESTRequest body, - Class responseType, - RESTAuthFunction restAuthFunction) { - HttpPut httpPut = HttpClientUtils.newHttpPut(getRequestUrl(path, null)); - String encodedBody = RESTUtil.encodedBody(body); - if (encodedBody != null) { - httpPut.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); - } - Header[] authHeaders = getHeaders(path, "PUT", encodedBody, restAuthFunction); - httpPut.setHeaders(authHeaders); - return exec( - httpPut, - responseType, - body != null && !body.isRetrySafe() - ? ExponentialHttpRequestRetryStrategy.retryUnsafeContext() - : null); - } - @Override public T delete(String path, RESTAuthFunction restAuthFunction) { return delete(path, null, null, restAuthFunction); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java index 94d0ba80f723..1444c8118d14 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java @@ -26,7 +26,6 @@ import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpHead; import org.apache.hc.client5.http.classic.methods.HttpPost; -import org.apache.hc.client5.http.classic.methods.HttpPut; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.entity.DecompressingEntity; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -239,10 +238,6 @@ public static HttpPost newHttpPost(String uri) { return newRequest(uri, HttpPost::new); } - public static HttpPut newHttpPut(String uri) { - return newRequest(uri, HttpPut::new); - } - public static HttpDelete newHttpDelete(String uri) { return newRequest(uri, HttpDelete::new); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 44018af6c895..435e70adcb9a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -421,8 +421,8 @@ public DatabaseReference createDatabaseReference( public DatabaseReference fastForwardDatabaseBranch( String databaseName, String targetBranch, String sourceTag) { DatabaseReferenceResponse response = - client.put( - resourcePaths.databaseTree(databaseName, targetBranch), + client.post( + resourcePaths.forwardDatabaseBranch(databaseName, targetBranch), new FastForwardDatabaseBranchRequest(sourceTag), DatabaseReferenceResponse.class, restAuthFunction); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 862f78860200..636ccea026d8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -157,6 +157,12 @@ public String databaseTree(String databaseName, String referenceName) { return SLASH.join(databaseTrees(databaseName), encodeString(referenceName)); } + /** Action endpoint for fast-forwarding a database-level branch. */ + @Experimental + public String forwardDatabaseBranch(String databaseName, String branch) { + return SLASH.join(databaseTree(databaseName, branch), "forward"); + } + public String tables(String databaseName) { return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLES); } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index 260d1140f4e0..a5ffa5102ca1 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -152,7 +152,7 @@ void testBranchAndImmutableTagHappyPath() throws Exception { enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); assertThat(api.fastForwardDatabaseBranch("training db", "main", "train-v1")) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - assertRequest(4, "PUT", TREES_PATH + "/main"); + assertRequest(4, "POST", TREES_PATH + "/main/forward"); assertBody(requests.get(4), "{\"sourceTag\":\"train-v1\"}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index b3464f9961e4..0d23747be2df 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -118,7 +118,7 @@ void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); assertThat(trees.fastForwardBranch(DATABASE, "main", "train-v1")).isEqualTo(main); - RecordedRequest fastForward = takeRequest("PUT", TREES_PATH + "/main"); + RecordedRequest fastForward = takeRequest("POST", TREES_PATH + "/main/forward"); assertBody(fastForward, "{\"sourceTag\":\"train-v1\"}"); enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); From cad64b8a31877d440a6a0e84151565c69f445f76 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 17:10:58 +0800 Subject: [PATCH 05/12] [rest] Allow database forward from branches and tags --- docs/static/rest-catalog-open-api.yaml | 18 ++++++++++-------- .../paimon/management/TreeManagement.java | 5 +++-- .../java/org/apache/paimon/rest/RESTApi.java | 6 +++--- .../paimon/rest/RESTTreeManagement.java | 4 ++-- .../FastForwardDatabaseBranchRequest.java | 19 ++++++++++--------- .../rest/RESTApiDatabaseReferenceTest.java | 16 ++++++++++++---- .../RequestJacksonCompatibilityTest.java | 19 ++++++++++++++----- .../rest/RESTCatalogTreeManagementTest.java | 15 +++++++++++---- 8 files changed, 65 insertions(+), 37 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index ff452c9147f1..4f443f7aeae4 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -384,7 +384,11 @@ paths: post: tags: - database-reference - summary: Fast-forward database branch from immutable tag + summary: Fast-forward database branch from a branch or tag + description: >- + The target must be a branch. The source branch or tag is resolved in the same database + when the request is processed. Fast-forward succeeds only if the target is an ancestor + of the source; divergent histories return a conflict. operationId: fastForwardDatabaseBranch parameters: - name: prefix @@ -418,13 +422,13 @@ paths: "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - description: Database, target branch, or source tag does not exist. + description: Database, target branch, or source reference does not exist. content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' "409": - description: Target is not an ancestor of the source tag. + description: Target is not a branch or is not an ancestor of the source reference. content: application/json: schema: @@ -4011,12 +4015,10 @@ components: FastForwardDatabaseBranchRequest: type: object required: - - sourceTag + - source properties: - sourceTag: - type: string - description: Immutable tag in the same database to fast-forward the target branch to. - pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + source: + $ref: '#/components/schemas/DatabaseReference' DeleteDatabaseReferenceRequest: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java index 4ca5dd9a26e7..cbecace2a1cd 100644 --- a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -52,8 +52,9 @@ DatabaseReference createReference( DatabaseReferenceType type, DatabaseReference source); - /** Fast-forwards a branch to an immutable tag in the same database. */ - DatabaseReference fastForwardBranch(String databaseName, String targetBranch, String sourceTag); + /** Fast-forwards a branch to another branch or immutable tag in the same database. */ + DatabaseReference fastForwardBranch( + String databaseName, String targetBranch, DatabaseReference source); /** * Deletes and returns a named reference. A missing reference is an error. diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 435e70adcb9a..f96e777ce985 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -416,14 +416,14 @@ public DatabaseReference createDatabaseReference( return checkNotNull(response.getReference(), "Reference response must contain reference"); } - /** Fast-forward a database-level branch to an immutable tag. */ + /** Fast-forward a database-level branch to another branch or immutable tag. */ @Experimental public DatabaseReference fastForwardDatabaseBranch( - String databaseName, String targetBranch, String sourceTag) { + String databaseName, String targetBranch, DatabaseReference source) { DatabaseReferenceResponse response = client.post( resourcePaths.forwardDatabaseBranch(databaseName, targetBranch), - new FastForwardDatabaseBranchRequest(sourceTag), + new FastForwardDatabaseBranchRequest(source), DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java index 2be049fb8e02..c1ed8c34dc2e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java @@ -59,8 +59,8 @@ public DatabaseReference createReference( @Override public DatabaseReference fastForwardBranch( - String databaseName, String targetBranch, String sourceTag) { - return api.fastForwardDatabaseBranch(databaseName, targetBranch, sourceTag); + String databaseName, String targetBranch, DatabaseReference source) { + return api.fastForwardDatabaseBranch(databaseName, targetBranch, source); } @Override diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java index f05b56d35458..d35f98afc16c 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest.requests; import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; import org.apache.paimon.rest.RESTRequest; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; @@ -28,23 +29,23 @@ import java.beans.ConstructorProperties; -/** Request for fast-forwarding a database branch to an immutable tag. */ +/** Request for fast-forwarding a database branch to another branch or immutable tag. */ @Experimental @JsonIgnoreProperties(ignoreUnknown = true) public class FastForwardDatabaseBranchRequest implements RESTRequest { - private static final String FIELD_SOURCE_TAG = "sourceTag"; + private static final String FIELD_SOURCE = "source"; - private final String sourceTag; + private final DatabaseReference source; @JsonCreator - @ConstructorProperties({FIELD_SOURCE_TAG}) - public FastForwardDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE_TAG) String sourceTag) { - this.sourceTag = sourceTag; + @ConstructorProperties({FIELD_SOURCE}) + public FastForwardDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE) DatabaseReference source) { + this.source = source; } - @JsonGetter(FIELD_SOURCE_TAG) - public String getSourceTag() { - return sourceTag; + @JsonGetter(FIELD_SOURCE) + public DatabaseReference getSource() { + return source; } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index a5ffa5102ca1..0535965eac0d 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -26,6 +26,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -95,8 +97,9 @@ void tearDown() { } } - @Test - void testBranchAndImmutableTagHappyPath() throws Exception { + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testBranchAndImmutableTagHappyPath(DatabaseReferenceType sourceType) throws Exception { enqueue( 200, "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}," @@ -150,10 +153,15 @@ void testBranchAndImmutableTagHappyPath() throws Exception { + "\"source\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); - assertThat(api.fastForwardDatabaseBranch("training db", "main", "train-v1")) + DatabaseReference source = sourceType == DatabaseReferenceType.BRANCH ? branch : tag; + assertThat(api.fastForwardDatabaseBranch("training db", "main", source)) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); assertRequest(4, "POST", TREES_PATH + "/main/forward"); - assertBody(requests.get(4), "{\"sourceTag\":\"train-v1\"}"); + assertBody( + requests.get(4), + sourceType == DatabaseReferenceType.BRANCH + ? "{\"source\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}" + : "{\"source\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); assertThat( diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 7dec699b4367..576cce333bd8 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import java.beans.ConstructorProperties; @@ -115,11 +116,6 @@ public class RequestJacksonCompatibilityTest { }, "partitionSpecs", "ignoreIfNotExists"), - requestCase( - FastForwardDatabaseBranchRequest.class, - "{\"sourceTag\":\"train-v1\"}", - request -> assertThat(request.getSourceTag()).isEqualTo("train-v1"), - "sourceTag"), requestCase( ListPartitionsByFilterRequest.class, "{\"filter\":\"dt = '2026-08-24'\"," @@ -185,6 +181,7 @@ public class RequestJacksonCompatibilityTest { CreateViewRequest.class, DeleteDatabaseReferenceRequest.class, DropPolicyRequest.class, + FastForwardDatabaseBranchRequest.class, GrantPermissionRequest.class, PolicyRequest.class, RegisterTableRequest.class, @@ -253,6 +250,18 @@ void testDeleteDatabaseReferenceRequestRoundTrips() throws Exception { assertThat(RESTApi.fromJson("{}", DeleteDatabaseReferenceRequest.class).getType()).isNull(); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testFastForwardDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) + throws Exception { + String json = "{\"source\":{\"type\":\"" + sourceType.name() + "\",\"name\":\"training\"}}"; + FastForwardDatabaseBranchRequest request = + EXTERNAL_MAPPER.readValue(json, FastForwardDatabaseBranchRequest.class); + FastForwardDatabaseBranchRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), FastForwardDatabaseBranchRequest.class); + assertThat(roundTrip.getSource()).isEqualTo(new DatabaseReference(sourceType, "training")); + } + @Test void testRequestCreatorAllowlistsAreComplete() throws Exception { Set> simpleRequests = diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index 0d23747be2df..4425e1352713 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -32,6 +32,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -92,8 +94,10 @@ void tearDown() throws Exception { } } - @Test - void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testBranchAndTagOperationsUseCatalogConfiguration(DatabaseReferenceType sourceType) + throws Exception { DatabaseReference main = new DatabaseReference(BRANCH, "main"); DatabaseReference branch = new DatabaseReference(BRANCH, "exp-1"); DatabaseReference tag = new DatabaseReference(TAG, "train-v1"); @@ -117,9 +121,12 @@ void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { "{\"name\":\"train-v1\",\"type\":\"TAG\",\"source\":" + BRANCH_JSON + "}"); enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); - assertThat(trees.fastForwardBranch(DATABASE, "main", "train-v1")).isEqualTo(main); + DatabaseReference source = sourceType == BRANCH ? branch : tag; + assertThat(trees.fastForwardBranch(DATABASE, "main", source)).isEqualTo(main); RecordedRequest fastForward = takeRequest("POST", TREES_PATH + "/main/forward"); - assertBody(fastForward, "{\"sourceTag\":\"train-v1\"}"); + assertBody( + fastForward, + "{\"source\":" + (sourceType == BRANCH ? BRANCH_JSON : TAG_JSON) + "}"); enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); assertThat(trees.deleteReference(DATABASE, "exp-1", BRANCH)).isEqualTo(branch); From cb0b0c244708a9886c28e7a7e771a5f0c7543ee6 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 17:21:11 +0800 Subject: [PATCH 06/12] [rest] Add database branch merge API --- docs/static/rest-catalog-open-api.yaml | 67 +++++++++++++++++++ .../paimon/management/TreeManagement.java | 10 +++ .../java/org/apache/paimon/rest/RESTApi.java | 14 ++++ .../paimon/rest/RESTTreeManagement.java | 6 ++ .../org/apache/paimon/rest/ResourcePaths.java | 6 ++ .../requests/MergeDatabaseBranchRequest.java | 51 ++++++++++++++ .../rest/RESTApiDatabaseReferenceTest.java | 18 +++++ .../RequestJacksonCompatibilityTest.java | 15 +++++ .../rest/RESTCatalogTreeManagementTest.java | 49 ++++++++++++++ 9 files changed, 236 insertions(+) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 4f443f7aeae4..d2086de62596 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -435,6 +435,66 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/trees/{name}/merge: + post: + tags: + - database-reference + summary: Merge a branch or tag into a database branch + description: >- + The target must be a branch. The source branch or tag is resolved in the same database + when the request is processed. Merge compares table entries with their common ancestor, + including table creation and deletion. Changes made only on one side are preserved; + identical changes on both sides are accepted. Different changes to the same table entry + conflict; table row data is not merged. The server checks for conflicts before publishing + the result. A conflict leaves the target unchanged, and the source reference is never + modified. Fast-forward and no-op merges succeed. The server must retain ancestry and + merge relationships to resolve subsequent merges. + operationId: mergeDatabaseBranch + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/MergeDatabaseBranchRequest' + responses: + "200": + description: Target branch after the merge, including when no changes were needed. + content: + application/json: + schema: + $ref: '#/components/schemas/DatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + description: Database, target branch, or source reference does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "409": + description: Target is not a branch, no common ancestor is available, or table entries conflict. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' /v1/{prefix}/databases/{database}/register: post: tags: @@ -4019,6 +4079,13 @@ components: properties: source: $ref: '#/components/schemas/DatabaseReference' + MergeDatabaseBranchRequest: + type: object + required: + - source + properties: + source: + $ref: '#/components/schemas/DatabaseReference' DeleteDatabaseReferenceRequest: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java index cbecace2a1cd..920dbec16245 100644 --- a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -56,6 +56,16 @@ DatabaseReference createReference( DatabaseReference fastForwardBranch( String databaseName, String targetBranch, DatabaseReference source); + /** + * Merges a branch or immutable tag into a target branch in the same database. + * + *

Table entries are merged relative to a common ancestor. Conflicting changes fail the merge + * without modifying the target; the source reference is never modified. A merge with no changes + * succeeds. + */ + DatabaseReference mergeBranch( + String databaseName, String targetBranch, DatabaseReference source); + /** * Deletes and returns a named reference. A missing reference is an error. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index f96e777ce985..ea4ffb2f65f0 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -63,6 +63,7 @@ import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; +import org.apache.paimon.rest.requests.MergeDatabaseBranchRequest; import org.apache.paimon.rest.requests.PolicyRequest; import org.apache.paimon.rest.requests.RegisterTableRequest; import org.apache.paimon.rest.requests.RenameTableRequest; @@ -429,6 +430,19 @@ public DatabaseReference fastForwardDatabaseBranch( return checkNotNull(response.getReference(), "Reference response must contain reference"); } + /** Merge a branch or immutable tag into a database-level branch, failing on conflicts. */ + @Experimental + public DatabaseReference mergeDatabaseBranch( + String databaseName, String targetBranch, DatabaseReference source) { + DatabaseReferenceResponse response = + client.post( + resourcePaths.mergeDatabaseBranch(databaseName, targetBranch), + new MergeDatabaseBranchRequest(source), + DatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + /** Delete one database-level branch or immutable tag. */ @Experimental public DatabaseReference deleteDatabaseReference( diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java index c1ed8c34dc2e..87e84b223775 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java @@ -63,6 +63,12 @@ public DatabaseReference fastForwardBranch( return api.fastForwardDatabaseBranch(databaseName, targetBranch, source); } + @Override + public DatabaseReference mergeBranch( + String databaseName, String targetBranch, DatabaseReference source) { + return api.mergeDatabaseBranch(databaseName, targetBranch, source); + } + @Override public DatabaseReference deleteReference( String databaseName, diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 636ccea026d8..01422fdb7e1f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -163,6 +163,12 @@ public String forwardDatabaseBranch(String databaseName, String branch) { return SLASH.join(databaseTree(databaseName, branch), "forward"); } + /** Action endpoint for merging a branch or tag into a database-level branch. */ + @Experimental + public String mergeDatabaseBranch(String databaseName, String branch) { + return SLASH.join(databaseTree(databaseName, branch), "merge"); + } + public String tables(String databaseName) { return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLES); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java new file mode 100644 index 000000000000..34c11f261b9f --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Request for merging a branch or immutable tag into a database branch. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class MergeDatabaseBranchRequest implements RESTRequest { + + private static final String FIELD_SOURCE = "source"; + + private final DatabaseReference source; + + @JsonCreator + @ConstructorProperties({FIELD_SOURCE}) + public MergeDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE) DatabaseReference source) { + this.source = source; + } + + @JsonGetter(FIELD_SOURCE) + public DatabaseReference getSource() { + return source; + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index 0535965eac0d..91b838320857 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -201,6 +201,24 @@ void testListReferencesPaged() { .containsEntry("pageToken", "p2"); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testMergeBranchOrTag(DatabaseReferenceType sourceType) throws Exception { + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); + DatabaseReference source = new DatabaseReference(sourceType, "experiment"); + + assertThat(api.mergeDatabaseBranch("training db", "main", source)) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + + assertRequest(0, "POST", TREES_PATH + "/main/merge"); + assertBody( + requests.get(0), + sourceType == DatabaseReferenceType.BRANCH + ? "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}}" + : "{\"source\":{\"type\":\"TAG\",\"name\":\"experiment\"}}"); + assertThat(requests).hasSize(1); + } + @Test void testDeleteReferenceWithoutType() throws Exception { enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 576cce333bd8..952ef5ac388b 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -183,6 +183,7 @@ public class RequestJacksonCompatibilityTest { DropPolicyRequest.class, FastForwardDatabaseBranchRequest.class, GrantPermissionRequest.class, + MergeDatabaseBranchRequest.class, PolicyRequest.class, RegisterTableRequest.class, RenameTableRequest.class, @@ -262,6 +263,20 @@ void testFastForwardDatabaseBranchRequestRoundTrips(DatabaseReferenceType source assertThat(roundTrip.getSource()).isEqualTo(new DatabaseReference(sourceType, "training")); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testMergeDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) + throws Exception { + String json = + "{\"source\":{\"type\":\"" + sourceType.name() + "\",\"name\":\"experiment\"}}"; + MergeDatabaseBranchRequest request = + EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class); + MergeDatabaseBranchRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); + assertThat(roundTrip.getSource()) + .isEqualTo(new DatabaseReference(sourceType, "experiment")); + } + @Test void testRequestCreatorAllowlistsAreComplete() throws Exception { Set> simpleRequests = diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index 4425e1352713..20868aadb3c0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -171,6 +171,55 @@ void testListPagesPreserveFilterAndTokens() throws Exception { assertThat(second.getRequestUrl().queryParameter("maxResults")).isNull(); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testMergeUsesCatalogConfiguration(DatabaseReferenceType sourceType) throws Exception { + enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); + DatabaseReference source = new DatabaseReference(sourceType, "experiment"); + + assertThat(trees.mergeBranch(DATABASE, "main", source)) + .isEqualTo(new DatabaseReference(BRANCH, "main")); + + RecordedRequest merge = takeRequest("POST", TREES_PATH + "/main/merge"); + assertBody( + merge, + sourceType == BRANCH + ? "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}}" + : "{\"source\":{\"type\":\"TAG\",\"name\":\"experiment\"}}"); + assertThat(server.getRequestCount()).isEqualTo(2); + } + + @Test + void testMergeErrorsPreserveDetails() throws Exception { + DatabaseReference source = new DatabaseReference(BRANCH, "experiment"); + enqueue( + 409, + "{\"code\":409,\"message\":\"Conflicting changes to table features\"," + + "\"resourceType\":\"TABLE\",\"resourceName\":\"training db.features\"}"); + assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) + .isInstanceOfSatisfying( + AlreadyExistsException.class, + conflict -> { + assertThat(conflict.resourceType()).isEqualTo("TABLE"); + assertThat(conflict.resourceName()).isEqualTo("training db.features"); + }) + .hasMessageContaining("Conflicting changes to table features"); + takeRequest("POST", TREES_PATH + "/main/merge"); + + enqueue(404, "{\"code\":404,\"message\":\"source reference missing\"}"); + assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) + .isInstanceOf(NoSuchResourceException.class) + .hasMessageContaining("source reference missing"); + takeRequest("POST", TREES_PATH + "/main/merge"); + + enqueue(501, "{\"code\":501,\"message\":\"merge unsupported\"}"); + assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) + .isInstanceOf(NotImplementedException.class) + .hasMessageContaining("merge unsupported"); + takeRequest("POST", TREES_PATH + "/main/merge"); + assertThat(server.getRequestCount()).isEqualTo(4); + } + @Test void testListAllTypesAndEmptyReferences() throws Exception { enqueue(200, "{\"references\":[" + MAIN_JSON + "," + TAG_JSON + "]}"); From 20366895e3cb2b771b2170760149f1f73641a17b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 17:43:39 +0800 Subject: [PATCH 07/12] [rest] Unify database merges with per-table merge modes --- docs/static/rest-catalog-open-api.yaml | 115 ++++++++---------- .../paimon/management/TreeManagement.java | 33 ++++- .../org/apache/paimon/rest/MergeMode.java | 34 ++++++ .../java/org/apache/paimon/rest/RESTApi.java | 43 ++++--- .../paimon/rest/RESTTreeManagement.java | 17 +-- .../org/apache/paimon/rest/ResourcePaths.java | 6 - ...BranchRequest.java => TableMergeMode.java} | 34 ++++-- .../exceptions/MergeConflictException.java | 48 ++++++++ .../requests/MergeDatabaseBranchRequest.java | 44 ++++++- .../rest/RESTApiDatabaseReferenceTest.java | 37 +++++- .../RequestJacksonCompatibilityTest.java | 74 +++++++++-- .../rest/RESTCatalogTreeManagementTest.java | 80 ++++++++++-- 12 files changed, 417 insertions(+), 148 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java rename paimon-api/src/main/java/org/apache/paimon/rest/{requests/FastForwardDatabaseBranchRequest.java => TableMergeMode.java} (62%) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index d2086de62596..0d9c1b85bcfb 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -380,61 +380,6 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/trees/{name}/forward: - post: - tags: - - database-reference - summary: Fast-forward database branch from a branch or tag - description: >- - The target must be a branch. The source branch or tag is resolved in the same database - when the request is processed. Fast-forward succeeds only if the target is an ancestor - of the source; divergent histories return a conflict. - operationId: fastForwardDatabaseBranch - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - - name: name - in: path - required: true - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/FastForwardDatabaseBranchRequest' - responses: - "200": - description: Updated target branch. - content: - application/json: - schema: - $ref: '#/components/schemas/DatabaseReferenceResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - description: Database, target branch, or source reference does not exist. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - "409": - description: Target is not a branch or is not an ancestor of the source reference. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' /v1/{prefix}/databases/{database}/trees/{name}/merge: post: tags: @@ -442,13 +387,23 @@ paths: summary: Merge a branch or tag into a database branch description: >- The target must be a branch. The source branch or tag is resolved in the same database - when the request is processed. Merge compares table entries with their common ancestor, - including table creation and deletion. Changes made only on one side are preserved; - identical changes on both sides are accepted. Different changes to the same table entry - conflict; table row data is not merged. The server checks for conflicts before publishing - the result. A conflict leaves the target unchanged, and the source reference is never - modified. Fast-forward and no-op merges succeed. The server must retain ancestry and - merge relationships to resolve subsequent merges. + when the request is processed. Merge compares complete table versions with their merge + base, including schemas, properties, snapshots, and table creation or deletion; table row + data is not merged. Changes made only on the target are preserved. Source-side changes + use defaultMergeMode (NORMAL when omitted), overridden by tableMergeModes for individual + table names. NORMAL accepts one-sided or identical changes and rejects different changes + to the same table. FORCE accepts the source-side change even on conflict, including a + deletion. DROP skips all source-side changes to that table, even without a conflict, and + preserves its target state. Modes do not replace target-only changes with unchanged source + versions. The server checks for unresolved conflicts before publishing the result; a + conflict leaves the target unchanged, and the source reference is never modified. + Identical references or an already-merged source succeed without modifying the target. + When the target is an ancestor of the source, the server may fast-forward only if the + selected modes produce exactly the source state. Divergent histories use three-way merge; + no available merge base is a conflict. A successful merge must record the source as merged, + including changes skipped by DROP, even when the table contents remain unchanged. Repeating + a merge of the same source state does not reapply skipped changes. The server retains + ancestry and merge relationships to resolve subsequent merges. operationId: mergeDatabaseBranch parameters: - name: prefix @@ -479,6 +434,8 @@ paths: application/json: schema: $ref: '#/components/schemas/DatabaseReferenceResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -488,7 +445,7 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' "409": - description: Target is not a branch, no common ancestor is available, or table entries conflict. + description: Target is not a branch, no merge base is available, or table conflicts remain unresolved. content: application/json: schema: @@ -4072,13 +4029,24 @@ components: enum: [ "BRANCH", "TAG" ] source: $ref: '#/components/schemas/DatabaseReference' - FastForwardDatabaseBranchRequest: + MergeMode: + type: string + enum: [ "NORMAL", "FORCE", "DROP" ] + description: >- + NORMAL performs three-way conflict detection. FORCE accepts source-side table changes + even on conflict. DROP skips all source-side changes to the table and preserves its + target state. Each mode operates on complete table versions, not individual rows. + TableMergeMode: type: object required: - - source + - table + - mergeMode properties: - source: - $ref: '#/components/schemas/DatabaseReference' + table: + type: string + description: Exact table name within the database being merged. + mergeMode: + $ref: '#/components/schemas/MergeMode' MergeDatabaseBranchRequest: type: object required: @@ -4086,6 +4054,19 @@ components: properties: source: $ref: '#/components/schemas/DatabaseReference' + defaultMergeMode: + description: Mode for tables without a per-table override. Defaults to NORMAL. + default: NORMAL + allOf: + - $ref: '#/components/schemas/MergeMode' + tableMergeModes: + type: array + description: >- + Per-table modes override defaultMergeMode. Omit or use an empty array to apply the + default to every table. Each table name may appear at most once; duplicates are a + bad request. Names without source-side changes have no effect. + items: + $ref: '#/components/schemas/TableMergeMode' DeleteDatabaseReferenceRequest: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java index 920dbec16245..8f864017676a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -22,9 +22,13 @@ import org.apache.paimon.annotation.Experimental; import org.apache.paimon.rest.DatabaseReference; import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.MergeMode; +import org.apache.paimon.rest.TableMergeMode; import javax.annotation.Nullable; +import java.util.List; + /** Control-plane contract for database-level writable branches and immutable tags. */ @Experimental public interface TreeManagement { @@ -52,19 +56,36 @@ DatabaseReference createReference( DatabaseReferenceType type, DatabaseReference source); - /** Fast-forwards a branch to another branch or immutable tag in the same database. */ - DatabaseReference fastForwardBranch( - String databaseName, String targetBranch, DatabaseReference source); - /** * Merges a branch or immutable tag into a target branch in the same database. * *

Table entries are merged relative to a common ancestor. Conflicting changes fail the merge * without modifying the target; the source reference is never modified. A merge with no changes - * succeeds. + * succeeds. The server automatically fast-forwards when possible. + */ + default DatabaseReference mergeBranch( + String databaseName, String targetBranch, DatabaseReference source) { + return mergeBranch(databaseName, targetBranch, source, null, null); + } + + /** + * Merges a branch or immutable tag using default and per-table merge modes. + * + *

Modes apply to source-side changes to complete table versions, including creation and + * deletion; table row data is not merged. Per-table modes override the default. Unresolved + * conflicts leave the target unchanged, and the source is never modified. A successful merge + * records the source as merged, including changes skipped by {@link MergeMode#DROP}. + * + * @param defaultMergeMode mode for tables without an override; null means {@link + * MergeMode#NORMAL} + * @param tableMergeModes per-table overrides; null or empty uses the default for every table */ DatabaseReference mergeBranch( - String databaseName, String targetBranch, DatabaseReference source); + String databaseName, + String targetBranch, + DatabaseReference source, + @Nullable MergeMode defaultMergeMode, + @Nullable List tableMergeModes); /** * Deletes and returns a named reference. A missing reference is an error. diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java b/paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java new file mode 100644 index 000000000000..8b18529e9a30 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.annotation.Experimental; + +/** How source-side table changes are handled when merging database references. */ +@Experimental +public enum MergeMode { + /** Merge changes relative to the common ancestor, failing on conflicting table versions. */ + NORMAL, + + /** Accept source-side changes even when they conflict with the target table version. */ + FORCE, + + /** Skip all source-side changes to the table, preserving its target state. */ + DROP +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index ea4ffb2f65f0..decefe3e3712 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -39,6 +39,7 @@ import org.apache.paimon.rest.auth.RESTAuthFunction; import org.apache.paimon.rest.exceptions.AlreadyExistsException; import org.apache.paimon.rest.exceptions.ForbiddenException; +import org.apache.paimon.rest.exceptions.MergeConflictException; import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterFunctionRequest; @@ -57,7 +58,6 @@ import org.apache.paimon.rest.requests.DeleteDatabaseReferenceRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.DropPolicyRequest; -import org.apache.paimon.rest.requests.FastForwardDatabaseBranchRequest; import org.apache.paimon.rest.requests.ForwardBranchRequest; import org.apache.paimon.rest.requests.GrantPermissionRequest; import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; @@ -417,30 +417,35 @@ public DatabaseReference createDatabaseReference( return checkNotNull(response.getReference(), "Reference response must contain reference"); } - /** Fast-forward a database-level branch to another branch or immutable tag. */ + /** Merge a branch or immutable tag into a database-level branch, failing on conflicts. */ @Experimental - public DatabaseReference fastForwardDatabaseBranch( + public DatabaseReference mergeDatabaseBranch( String databaseName, String targetBranch, DatabaseReference source) { - DatabaseReferenceResponse response = - client.post( - resourcePaths.forwardDatabaseBranch(databaseName, targetBranch), - new FastForwardDatabaseBranchRequest(source), - DatabaseReferenceResponse.class, - restAuthFunction); - return checkNotNull(response.getReference(), "Reference response must contain reference"); + return mergeDatabaseBranch(databaseName, targetBranch, source, null, null); } - /** Merge a branch or immutable tag into a database-level branch, failing on conflicts. */ + /** Merge a branch or immutable tag using default and per-table merge modes. */ @Experimental public DatabaseReference mergeDatabaseBranch( - String databaseName, String targetBranch, DatabaseReference source) { - DatabaseReferenceResponse response = - client.post( - resourcePaths.mergeDatabaseBranch(databaseName, targetBranch), - new MergeDatabaseBranchRequest(source), - DatabaseReferenceResponse.class, - restAuthFunction); - return checkNotNull(response.getReference(), "Reference response must contain reference"); + String databaseName, + String targetBranch, + DatabaseReference source, + @Nullable MergeMode defaultMergeMode, + @Nullable List tableMergeModes) { + try { + DatabaseReferenceResponse response = + client.post( + resourcePaths.mergeDatabaseBranch(databaseName, targetBranch), + new MergeDatabaseBranchRequest( + source, defaultMergeMode, tableMergeModes), + DatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull( + response.getReference(), "Reference response must contain reference"); + } catch (AlreadyExistsException e) { + throw new MergeConflictException( + e, e.resourceType(), e.resourceName(), "%s", e.getMessage()); + } } /** Delete one database-level branch or immutable tag. */ diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java index 87e84b223775..f5cf357b877b 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java @@ -24,6 +24,8 @@ import javax.annotation.Nullable; +import java.util.List; + /** REST implementation of tree management, bound to the configured REST catalog prefix. */ @Experimental public class RESTTreeManagement implements TreeManagement { @@ -57,16 +59,15 @@ public DatabaseReference createReference( return api.createDatabaseReference(databaseName, referenceName, type, source); } - @Override - public DatabaseReference fastForwardBranch( - String databaseName, String targetBranch, DatabaseReference source) { - return api.fastForwardDatabaseBranch(databaseName, targetBranch, source); - } - @Override public DatabaseReference mergeBranch( - String databaseName, String targetBranch, DatabaseReference source) { - return api.mergeDatabaseBranch(databaseName, targetBranch, source); + String databaseName, + String targetBranch, + DatabaseReference source, + @Nullable MergeMode defaultMergeMode, + @Nullable List tableMergeModes) { + return api.mergeDatabaseBranch( + databaseName, targetBranch, source, defaultMergeMode, tableMergeModes); } @Override diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 01422fdb7e1f..0b25fa3de9d2 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -157,12 +157,6 @@ public String databaseTree(String databaseName, String referenceName) { return SLASH.join(databaseTrees(databaseName), encodeString(referenceName)); } - /** Action endpoint for fast-forwarding a database-level branch. */ - @Experimental - public String forwardDatabaseBranch(String databaseName, String branch) { - return SLASH.join(databaseTree(databaseName, branch), "forward"); - } - /** Action endpoint for merging a branch or tag into a database-level branch. */ @Experimental public String mergeDatabaseBranch(String databaseName, String branch) { diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java similarity index 62% rename from paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java rename to paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java index d35f98afc16c..b7eab7f34ed0 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java @@ -16,11 +16,9 @@ * limitations under the License. */ -package org.apache.paimon.rest.requests; +package org.apache.paimon.rest; import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.rest.DatabaseReference; -import org.apache.paimon.rest.RESTRequest; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; @@ -29,23 +27,33 @@ import java.beans.ConstructorProperties; -/** Request for fast-forwarding a database branch to another branch or immutable tag. */ +/** Overrides the default merge mode for one table name within the database being merged. */ @Experimental @JsonIgnoreProperties(ignoreUnknown = true) -public class FastForwardDatabaseBranchRequest implements RESTRequest { +public class TableMergeMode { - private static final String FIELD_SOURCE = "source"; + private static final String FIELD_TABLE = "table"; + private static final String FIELD_MERGE_MODE = "mergeMode"; - private final DatabaseReference source; + private final String table; + private final MergeMode mergeMode; @JsonCreator - @ConstructorProperties({FIELD_SOURCE}) - public FastForwardDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE) DatabaseReference source) { - this.source = source; + @ConstructorProperties({FIELD_TABLE, FIELD_MERGE_MODE}) + public TableMergeMode( + @JsonProperty(FIELD_TABLE) String table, + @JsonProperty(FIELD_MERGE_MODE) MergeMode mergeMode) { + this.table = table; + this.mergeMode = mergeMode; } - @JsonGetter(FIELD_SOURCE) - public DatabaseReference getSource() { - return source; + @JsonGetter(FIELD_TABLE) + public String getTable() { + return table; + } + + @JsonGetter(FIELD_MERGE_MODE) + public MergeMode getMergeMode() { + return mergeMode; } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java b/paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java new file mode 100644 index 000000000000..ba8433bbe754 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.exceptions; + +import org.apache.paimon.annotation.Experimental; + +/** Exception thrown when an HTTP 409 prevents a database branch merge. */ +@Experimental +public class MergeConflictException extends RESTException { + + private final String resourceType; + private final String resourceName; + + public MergeConflictException( + Throwable cause, + String resourceType, + String resourceName, + String message, + Object... args) { + super(cause, message, args); + this.resourceType = resourceType; + this.resourceName = resourceName; + } + + public String resourceType() { + return resourceType; + } + + public String resourceName() { + return resourceName; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java index 34c11f261b9f..5beb2e7973ec 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java @@ -20,14 +20,22 @@ import org.apache.paimon.annotation.Experimental; import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.MergeMode; import org.apache.paimon.rest.RESTRequest; +import org.apache.paimon.rest.TableMergeMode; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.Nullable; + import java.beans.ConstructorProperties; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; /** Request for merging a branch or immutable tag into a database branch. */ @Experimental @@ -35,17 +43,49 @@ public class MergeDatabaseBranchRequest implements RESTRequest { private static final String FIELD_SOURCE = "source"; + private static final String FIELD_DEFAULT_MERGE_MODE = "defaultMergeMode"; + private static final String FIELD_TABLE_MERGE_MODES = "tableMergeModes"; private final DatabaseReference source; + @Nullable private final MergeMode defaultMergeMode; + @Nullable private final List tableMergeModes; + + public MergeDatabaseBranchRequest(DatabaseReference source) { + this(source, null, null); + } @JsonCreator - @ConstructorProperties({FIELD_SOURCE}) - public MergeDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE) DatabaseReference source) { + @ConstructorProperties({FIELD_SOURCE, FIELD_DEFAULT_MERGE_MODE, FIELD_TABLE_MERGE_MODES}) + public MergeDatabaseBranchRequest( + @JsonProperty(FIELD_SOURCE) DatabaseReference source, + @Nullable @JsonProperty(FIELD_DEFAULT_MERGE_MODE) MergeMode defaultMergeMode, + @Nullable @JsonProperty(FIELD_TABLE_MERGE_MODES) List tableMergeModes) { this.source = source; + this.defaultMergeMode = defaultMergeMode; + this.tableMergeModes = + tableMergeModes == null + ? null + : Collections.unmodifiableList(new ArrayList<>(tableMergeModes)); } @JsonGetter(FIELD_SOURCE) public DatabaseReference getSource() { return source; } + + /** Null uses the server default, {@link MergeMode#NORMAL}. */ + @Nullable + @JsonGetter(FIELD_DEFAULT_MERGE_MODE) + @JsonInclude(JsonInclude.Include.NON_NULL) + public MergeMode getDefaultMergeMode() { + return defaultMergeMode; + } + + /** Per-table modes override the default; null or empty supplies no overrides. */ + @Nullable + @JsonGetter(FIELD_TABLE_MERGE_MODES) + @JsonInclude(JsonInclude.Include.NON_NULL) + public List getTableMergeModes() { + return tableMergeModes; + } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index 91b838320857..d8991249eac4 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -36,6 +36,7 @@ import java.net.InetSocketAddress; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -154,9 +155,9 @@ void testBranchAndImmutableTagHappyPath(DatabaseReferenceType sourceType) throws enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); DatabaseReference source = sourceType == DatabaseReferenceType.BRANCH ? branch : tag; - assertThat(api.fastForwardDatabaseBranch("training db", "main", source)) + assertThat(api.mergeDatabaseBranch("training db", "main", source)) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - assertRequest(4, "POST", TREES_PATH + "/main/forward"); + assertRequest(4, "POST", TREES_PATH + "/main/merge"); assertBody( requests.get(4), sourceType == DatabaseReferenceType.BRANCH @@ -219,6 +220,38 @@ void testMergeBranchOrTag(DatabaseReferenceType sourceType) throws Exception { assertThat(requests).hasSize(1); } + @ParameterizedTest + @EnumSource(MergeMode.class) + void testMergeModesAreSentInBody(MergeMode defaultMergeMode) throws Exception { + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); + DatabaseReference source = + new DatabaseReference(DatabaseReferenceType.BRANCH, "experiment"); + + assertThat( + api.mergeDatabaseBranch( + "training db", + "main", + source, + defaultMergeMode, + Arrays.asList( + new TableMergeMode("features.v2", MergeMode.FORCE), + new TableMergeMode("scratch", MergeMode.DROP), + new TableMergeMode("labels", MergeMode.NORMAL)))) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + + assertRequest(0, "POST", TREES_PATH + "/main/merge"); + assertBody( + requests.get(0), + "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}," + + "\"defaultMergeMode\":\"" + + defaultMergeMode.name() + + "\"," + + "\"tableMergeModes\":[{\"table\":\"features.v2\",\"mergeMode\":\"FORCE\"}," + + "{\"table\":\"scratch\",\"mergeMode\":\"DROP\"}," + + "{\"table\":\"labels\",\"mergeMode\":\"NORMAL\"}]}"); + assertThat(requests).hasSize(1); + } + @Test void testDeleteReferenceWithoutType() throws Exception { enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 952ef5ac388b..9e67468e6324 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -20,8 +20,10 @@ import org.apache.paimon.rest.DatabaseReference; import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.MergeMode; import org.apache.paimon.rest.RESTApi; import org.apache.paimon.rest.RESTRequest; +import org.apache.paimon.rest.TableMergeMode; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; @@ -29,6 +31,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import java.beans.ConstructorProperties; import java.lang.reflect.Constructor; @@ -181,7 +184,6 @@ public class RequestJacksonCompatibilityTest { CreateViewRequest.class, DeleteDatabaseReferenceRequest.class, DropPolicyRequest.class, - FastForwardDatabaseBranchRequest.class, GrantPermissionRequest.class, MergeDatabaseBranchRequest.class, PolicyRequest.class, @@ -251,18 +253,6 @@ void testDeleteDatabaseReferenceRequestRoundTrips() throws Exception { assertThat(RESTApi.fromJson("{}", DeleteDatabaseReferenceRequest.class).getType()).isNull(); } - @ParameterizedTest - @EnumSource(DatabaseReferenceType.class) - void testFastForwardDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) - throws Exception { - String json = "{\"source\":{\"type\":\"" + sourceType.name() + "\",\"name\":\"training\"}}"; - FastForwardDatabaseBranchRequest request = - EXTERNAL_MAPPER.readValue(json, FastForwardDatabaseBranchRequest.class); - FastForwardDatabaseBranchRequest roundTrip = - RESTApi.fromJson(RESTApi.toJson(request), FastForwardDatabaseBranchRequest.class); - assertThat(roundTrip.getSource()).isEqualTo(new DatabaseReference(sourceType, "training")); - } - @ParameterizedTest @EnumSource(DatabaseReferenceType.class) void testMergeDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) @@ -275,6 +265,64 @@ void testMergeDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); assertThat(roundTrip.getSource()) .isEqualTo(new DatabaseReference(sourceType, "experiment")); + assertThat(roundTrip.getDefaultMergeMode()).isNull(); + assertThat(roundTrip.getTableMergeModes()).isNull(); + assertThat(RESTApi.fromJson(RESTApi.toJson(roundTrip), Map.class)) + .isEqualTo(RESTApi.fromJson(json, Map.class)); + } + + @ParameterizedTest + @EnumSource(MergeMode.class) + void testMergeModesRoundTrip(MergeMode mode) throws Exception { + String json = + "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}," + + "\"defaultMergeMode\":\"" + + mode.name() + + "\"," + + "\"tableMergeModes\":[{\"table\":\"features.v2\",\"mergeMode\":\"FORCE\"}," + + "{\"table\":\"scratch\",\"mergeMode\":\"DROP\"}," + + "{\"table\":\"labels\",\"mergeMode\":\"NORMAL\"}]}"; + MergeDatabaseBranchRequest request = + EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class); + MergeDatabaseBranchRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); + assertThat(roundTrip.getDefaultMergeMode()).isEqualTo(mode); + assertThat(roundTrip.getTableMergeModes()) + .extracting(TableMergeMode::getTable) + .containsExactly("features.v2", "scratch", "labels"); + assertThat(roundTrip.getTableMergeModes()) + .extracting(TableMergeMode::getMergeMode) + .containsExactly(MergeMode.FORCE, MergeMode.DROP, MergeMode.NORMAL); + assertThat(RESTApi.fromJson(RESTApi.toJson(roundTrip), Map.class)) + .isEqualTo(RESTApi.fromJson(json, Map.class)); + } + + @Test + void testMergeWithEmptyOverrides() throws Exception { + String json = + "{\"source\":{\"type\":\"TAG\",\"name\":\"train-v1\"},\"tableMergeModes\":[]}"; + MergeDatabaseBranchRequest request = + EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class); + MergeDatabaseBranchRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); + assertThat(roundTrip.getDefaultMergeMode()).isNull(); + assertThat(roundTrip.getTableMergeModes()).isEmpty(); + assertThat(RESTApi.fromJson(RESTApi.toJson(roundTrip), Map.class)) + .isEqualTo(RESTApi.fromJson(json, Map.class)); + } + + @ParameterizedTest + @ValueSource( + strings = { + "\"defaultMergeMode\":\"UNKNOWN\"", + "\"tableMergeModes\":[{\"table\":\"features\",\"mergeMode\":\"UNKNOWN\"}]" + }) + void testUnknownMergeModesAreRejected(String modes) { + String json = "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}," + modes + "}"; + assertThatThrownBy(() -> EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class)) + .hasMessageContaining("UNKNOWN"); + assertThatThrownBy(() -> RESTApi.fromJson(json, MergeDatabaseBranchRequest.class)) + .hasMessageContaining("UNKNOWN"); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index 20868aadb3c0..e4cb58ebc7a9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -23,6 +23,8 @@ import org.apache.paimon.management.TreeManagement; import org.apache.paimon.options.Options; import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.exceptions.BadRequestException; +import org.apache.paimon.rest.exceptions.MergeConflictException; import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.exceptions.NotImplementedException; @@ -35,6 +37,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -122,11 +125,9 @@ void testBranchAndTagOperationsUseCatalogConfiguration(DatabaseReferenceType sou enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); DatabaseReference source = sourceType == BRANCH ? branch : tag; - assertThat(trees.fastForwardBranch(DATABASE, "main", source)).isEqualTo(main); - RecordedRequest fastForward = takeRequest("POST", TREES_PATH + "/main/forward"); - assertBody( - fastForward, - "{\"source\":" + (sourceType == BRANCH ? BRANCH_JSON : TAG_JSON) + "}"); + assertThat(trees.mergeBranch(DATABASE, "main", source)).isEqualTo(main); + RecordedRequest merge = takeRequest("POST", TREES_PATH + "/main/merge"); + assertBody(merge, "{\"source\":" + (sourceType == BRANCH ? BRANCH_JSON : TAG_JSON) + "}"); enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); assertThat(trees.deleteReference(DATABASE, "exp-1", BRANCH)).isEqualTo(branch); @@ -189,21 +190,76 @@ void testMergeUsesCatalogConfiguration(DatabaseReferenceType sourceType) throws assertThat(server.getRequestCount()).isEqualTo(2); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testMergeModesUseCatalogConfiguration(DatabaseReferenceType sourceType) throws Exception { + enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); + DatabaseReference source = new DatabaseReference(sourceType, "experiment"); + + assertThat( + trees.mergeBranch( + DATABASE, + "main", + source, + MergeMode.NORMAL, + Arrays.asList( + new TableMergeMode("features", MergeMode.FORCE), + new TableMergeMode("scratch", MergeMode.DROP)))) + .isEqualTo(new DatabaseReference(BRANCH, "main")); + + assertBody( + takeRequest("POST", TREES_PATH + "/main/merge"), + "{\"source\":{\"type\":\"" + + sourceType.name() + + "\",\"name\":\"experiment\"}," + + "\"defaultMergeMode\":\"NORMAL\",\"tableMergeModes\":[" + + "{\"table\":\"features\",\"mergeMode\":\"FORCE\"}," + + "{\"table\":\"scratch\",\"mergeMode\":\"DROP\"}]}"); + assertThat(server.getRequestCount()).isEqualTo(2); + } + @Test void testMergeErrorsPreserveDetails() throws Exception { DatabaseReference source = new DatabaseReference(BRANCH, "experiment"); - enqueue( - 409, - "{\"code\":409,\"message\":\"Conflicting changes to table features\"," - + "\"resourceType\":\"TABLE\",\"resourceName\":\"training db.features\"}"); + server.enqueue( + new MockResponse() + .setResponseCode(409) + .setHeader("Content-Type", "application/json") + .setHeader("x-request-id", "merge-request") + .setBody( + "{\"message\":\"Conflicting changes to table features (100%)\"," + + "\"resourceType\":\"TABLE\",\"resourceName\":\"training db.features\"}")); assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) .isInstanceOfSatisfying( - AlreadyExistsException.class, + MergeConflictException.class, conflict -> { assertThat(conflict.resourceType()).isEqualTo("TABLE"); assertThat(conflict.resourceName()).isEqualTo("training db.features"); + assertThat(conflict.getCause()) + .isInstanceOf(AlreadyExistsException.class) + .hasMessage(conflict.getMessage()); }) - .hasMessageContaining("Conflicting changes to table features"); + .hasMessage("Conflicting changes to table features (100%) requestId:merge-request"); + takeRequest("POST", TREES_PATH + "/main/merge"); + + enqueue(409, "{\"code\":409,\"message\":\"reference already exists\"}"); + assertThatThrownBy(() -> trees.createReference(DATABASE, "existing", BRANCH, source)) + .isExactlyInstanceOf(AlreadyExistsException.class); + takeRequest("POST", TREES_PATH); + + enqueue(400, "{\"code\":400,\"message\":\"duplicate table merge mode\"}"); + assertThatThrownBy( + () -> + trees.mergeBranch( + DATABASE, + "main", + source, + MergeMode.NORMAL, + Arrays.asList( + new TableMergeMode("features", MergeMode.FORCE), + new TableMergeMode("features", MergeMode.DROP)))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("duplicate table merge mode"); takeRequest("POST", TREES_PATH + "/main/merge"); enqueue(404, "{\"code\":404,\"message\":\"source reference missing\"}"); @@ -217,7 +273,7 @@ void testMergeErrorsPreserveDetails() throws Exception { .isInstanceOf(NotImplementedException.class) .hasMessageContaining("merge unsupported"); takeRequest("POST", TREES_PATH + "/main/merge"); - assertThat(server.getRequestCount()).isEqualTo(4); + assertThat(server.getRequestCount()).isEqualTo(6); } @Test From da8e1af2eb3d7ce88b934efcee747789c93044f5 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 18:07:39 +0800 Subject: [PATCH 08/12] [docs] Describe database versioning APIs and server MVP --- .../docs/concepts/rest/database-versioning.md | 488 ++++++++++++++++++ docs/docs/concepts/rest/index.md | 2 + docs/docs/concepts/rest/rest-api.md | 7 +- docs/docs/program-api/rest-api.mdx | 1 + docs/sidebars.js | 1 + 5 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 docs/docs/concepts/rest/database-versioning.md diff --git a/docs/docs/concepts/rest/database-versioning.md b/docs/docs/concepts/rest/database-versioning.md new file mode 100644 index 000000000000..ea956a615452 --- /dev/null +++ b/docs/docs/concepts/rest/database-versioning.md @@ -0,0 +1,488 @@ +--- +title: "Database Branches and Tags" +--- + + + +# Database Branches and Tags + +Database references group the versions of several tables under one branch or tag. A typical +training workflow starts an experiment from `main`, writes derived data on the experiment branch, +freezes the inputs under a tag, and merges accepted changes back into `main`. + +This page describes the experimental REST management contract and a proposed server MVP that +reuses Paimon's existing [table branches](../../maintenance/manage-branches) and +[table tags](../../maintenance/manage-tags). + +:::info Implementation status + +The Java reference-management client and the `/trees` wire contract are implemented. Reference +storage, table-level orchestration, and database merge execution must be implemented by the catalog +server. The server implementation below is a design, not a claim that an existing service supports it. + +The MVP uses explicit table-level branch and tag addressing. This page introduces no new reference +header or catalog option. Selecting a complete database view for table listing and DDL remains +additional work, described under [Beyond the fixed-table MVP](#beyond-the-fixed-table-mvp). + +::: + +## Scope and terminology + +| Term | Meaning | +| --- | --- | +| Database branch | A writable reference to a database's table membership and table versions. | +| Database tag | An immutable reference to a captured membership and versions. Deleting a tag is allowed; moving it is not. | +| Table branch/tag | The existing Paimon storage and read/write mechanism used behind a database reference. | +| Table membership | The names and identities of the tables visible in a database reference. | +| Merge base | The historical state used to distinguish source changes from target changes. It includes previous merge relationships. | + +References belong to one database, not the whole catalog. Branches and tags share a name namespace +within that database. A reference contains `type` (`BRANCH` or `TAG`) and `name`. Names match +`[A-Za-z0-9][A-Za-z0-9._-]{0,127}`. Public references have no hash or reference ID. + +### Initial server MVP + +Start with managed native Paimon tables and a fixed set of logical table names. Create and populate +those tables on `main` before starting the experiment. Use batch writers and pause writes during +branch creation, tag creation, and merge. Resume with freshly loaded tables after publication. + +This scope can demonstrate isolated table writes, multi-table training inputs frozen under a tag, +and table-version merge. It does not require a public multi-table transaction API, public hashes, +row-level conflict resolution, or concurrent streaming publication. + +Branch-local table creation, deletion, and rename need reference-aware namespace handling. The +merge contract covers table creation and deletion, but the first fixed-table server can defer those +operations until the table APIs can address the corresponding database view. Format Tables, +Object Tables, external tables, views, functions, and catalog permissions are outside this initial +versioned-table scope. + +## Reference management API + +All paths use the configured catalog `prefix`. For brevity, the following table uses +`B = /v1/{prefix}/databases/{database}`. Encode each path segment; names in JSON remain unencoded. + +| Method and path | Request | Result | +| --- | --- | --- | +| `GET B/trees` | Optional `type`, `maxResults`, and `pageToken` query parameters. | One page of references. | +| `GET B/trees/{name}` | No body. | One reference. | +| `POST B/trees` | New name, type, and an existing source reference. | The created reference. | +| `POST B/trees/{name}/merge` | Source reference and optional merge modes. The path names the target branch. | The target reference after success. | +| `DELETE B/trees/{name}` | Optional expected `type` in the body. | The deleted reference. | + +Database merge includes fast-forward when applicable. There is no database-level `/forward` +endpoint. The existing table-level forward API is separate. + +### Create a branch or tag + +Create an experiment branch from `main`: + +```http +POST /v1/catalog/databases/training/trees +Content-Type: application/json + +{ + "name": "experiment", + "type": "BRANCH", + "source": {"type": "BRANCH", "name": "main"} +} +``` + +Freeze the experiment under a database tag: + +```json +{ + "name": "train_v1", + "type": "TAG", + "source": {"type": "BRANCH", "name": "experiment"} +} +``` + +Both requests use the same path. The source must exist in the same database. A source can be a +branch or an immutable tag; the new reference can also be either type. Successful singular +operations return `DatabaseReferenceResponse`: + +```json +{"reference": {"type": "TAG", "name": "train_v1"}} +``` + +### Inspect and list + +```http +GET /v1/catalog/databases/training/trees/train_v1 +GET /v1/catalog/databases/training/trees?type=tag&maxResults=100 +``` + +The list filter uses lowercase `branch` or `tag`; JSON reference types use uppercase enum names. +Omitting `type` includes both. A missing or zero `maxResults` uses the server default. Pass the +returned `nextPageToken` unchanged to request the next page; a missing token ends iteration. + +```json +{ + "references": [{"type": "TAG", "name": "train_v1"}], + "nextPageToken": "next-page" +} +``` + +Getting a reference returns its name and type, not the table membership, source branch, or table +version map. Pagination discovers references; it does not create a frozen view across pages. + +### Merge + +```http +POST /v1/catalog/databases/training/trees/main/merge +Content-Type: application/json + +{ + "source": {"type": "BRANCH", "name": "experiment"}, + "defaultMergeMode": "NORMAL", + "tableMergeModes": [ + {"table": "features", "mergeMode": "FORCE"}, + {"table": "scratch", "mergeMode": "DROP"} + ] +} +``` + +Only `source` is required. Omitting modes gives `NORMAL` for every table. Per-table modes override +the default, and an omitted or empty override list applies the default everywhere. Table names +are exact names within this database. Duplicate table overrides are a bad request; an override +for a table without source-side changes has no effect. + +The target is always a branch. A source tag is allowed and remains immutable. The response remains +`DatabaseReferenceResponse`; it does not include a commit hash or a detailed merge report. + +### Delete + +```http +DELETE /v1/catalog/databases/training/trees/train_v1 +Content-Type: application/json + +{"type": "TAG"} +``` + +The optional type checks the reference before deletion. Omitting the body or sending `{}` omits +that check. An absent reference is an error. The MVP server should protect the default `main` +branch. Logical deletion does not authorize deleting table versions still needed by another +reference. + +### Errors + +| Situation | HTTP behavior | +| --- | --- | +| Missing database or reference | `404`; merge distinguishes the missing source or target in its error details. | +| Creating an existing reference | `409`. | +| Merge target is a tag, no merge base is available, or unresolved table conflicts remain | `409`; the target stays unchanged. | +| Invalid merge request, such as duplicate per-table modes | `400`. | +| Deleting a protected default branch or supplying the wrong expected type | `409`. | +| Server does not implement an operation | No client fallback; the server error is propagated. | + +Errors use `ErrorResponse`. The Java merge client converts `409` to `MergeConflictException` and +preserves the resource type/name, message, request ID, and cause. Resource creation still uses +`AlreadyExistsException`. See the [OpenAPI specification](/rest-catalog-open-api.yaml) for the +individual operations and their documented responses. + +## Java management usage + +Obtain tree management from an already configured `RESTCatalog`. It shares that catalog's +prefix, authentication, and HTTP configuration: + +```java +import org.apache.paimon.PagedList; +import org.apache.paimon.management.TreeManagement; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.MergeMode; +import org.apache.paimon.rest.TableMergeMode; + +import java.util.Collections; + +TreeManagement trees = restCatalog.treeManagement(); +DatabaseReference main = new DatabaseReference(DatabaseReferenceType.BRANCH, "main"); +DatabaseReference experiment = trees.createReference( + "training", "experiment", DatabaseReferenceType.BRANCH, main); + +// Run batch writes on the corresponding table branches before freezing this tag. +DatabaseReference trainingTag = trees.createReference( + "training", "train_v1", DatabaseReferenceType.TAG, experiment); + +PagedList page = trees.listReferencesPaged( + "training", DatabaseReferenceType.TAG, 100, null); + +// Default three-way merge, failing on conflicting table versions. +trees.mergeBranch("training", "main", trainingTag); +``` + +When resolving a conflict, use the following call instead of the default merge to accept the +source version of `features`. Changing modes after a successful merge does not reapply that source: + +```java +trees.mergeBranch( + "training", "main", trainingTag, MergeMode.NORMAL, + Collections.singletonList(new TableMergeMode("features", MergeMode.FORCE))); +``` + +`RESTApi` exposes equivalent methods: `listDatabaseReferencesPaged`, `getDatabaseReference`, +`createDatabaseReference`, `mergeDatabaseBranch`, and `deleteDatabaseReference`. Listing is paged; +there is no non-paged database-reference helper. + +These are management calls. Creating a database branch does not switch the catalog's ordinary +table operations to that branch. + +## Reusing table branches and tags on the server + +The server coordinates existing table-level operations and keeps database metadata around them. +An illustrative mapping is: + +```text +training / main [BRANCH] + features -> table identity A, table branch main + labels -> table identity B, table branch main + +training / experiment [BRANCH] + features -> table identity A, table branch experiment + labels -> table identity B, table branch experiment + +training / train_v1 [TAG] + features -> table identity A, experiment branch, pinned table tag train_v1 + labels -> table identity B, experiment branch, pinned table tag train_v1 +``` + +For the explicit-addressing MVP, names such as `experiment` can also name the corresponding table +branches, and `train_v1` can name each table tag in its source table branch. These names are owned +by the service. Reject collisions with unrelated existing table references; do not adopt them just +because the names match. Additional internal baseline tags can use private, service-generated names. + +The public database-reference name rules and native table-branch rules are not identical. For +example, the database protocol permits a purely numeric name, while native table branch creation +rejects it. A server supporting the full name contract needs an alias mapping to valid physical +branch names and must resolve the logical table-branch address through that mapping. It must not +silently narrow the database API's name rules. The examples use names valid in both layers. + +### Minimal metadata + +The server needs: + +- A database reference record: name, type, current membership, and internal ancestry/merge history. +- A mapping from logical table name to stable table identity and backing table branch or tag. +- Captured table versions at branch points, tag creation, and merges, including the schema and + snapshot state needed for comparisons and reads. + +A captured version can reuse a snapshot UUID, a pinned schema, and relevant table properties. +Empty tables need an explicit no-snapshot state. Numeric snapshot/schema IDs alone are not enough +to compare independently written branches. Table identity distinguishes a dropped-and-recreated +table from its predecessor. Copying metadata to a new physical branch does not itself constitute a +logical table change. + +These records can live in the catalog backend. Their internal identities are not public hashes and +need not introduce a new versioned storage engine. The server must update its recorded table state +when a managed branch accepts a table commit or schema change; names alone cannot support merge. +Use the server's table commit and schema operations for those writes. Uncoordinated filesystem +writes or direct edits of service-owned table references would bypass this bookkeeping. + +### Bootstrap main + +A version-enabled new database starts with an empty `main` branch. Otherwise every create-reference +request would require a source that does not yet exist. For an existing database, the server can +initialize `main` from its current tables while writers are stopped. Automatic online conversion of +an actively written database is outside the first MVP. + +This is a server lifecycle rule, not an additional REST endpoint. Reference creation always keeps +its existing `source` field. + +### Create a database branch + +1. Capture the source membership and each selected table version while writes are paused. +2. For a populated table, pin the selected source snapshot with a service-owned table tag and create + the destination table branch from that tag. +3. For an empty table, create a schema-only table branch. Preserve the selected schema and properties. +4. Record the common baseline and publish the database branch after all table branches are ready. + +The existing `FileSystemBranchManager.createBranch(name)` creates an empty branch by copying +schemas. It does not clone the source data. `createBranch(name, tagName)` copies the selected +snapshot and its schemas. If the captured current schema is newer than the snapshot's schema, +the server must also preserve that schema-only change; snapshot cloning alone is insufficient. + +No data files need to be copied merely to create a branch. In the single-process MVP, table-level +setup can run sequentially; do not expose an incomplete database reference as successfully created. +Failures can leave private work to clean up or resume. + +### Create and retain a database tag + +Capture the table membership and pin a table tag for each populated table. Persist the source table +branch with each pin: native Paimon tags belong to a table branch, not a database-wide directory. +An empty table has no snapshot to tag, so its frozen entry must retain the schema and empty state; +the server cannot blindly call `createTag` on every table. + +A database tag never follows subsequent writes to its source. For an empty tagged table, reads must +remain empty even if the source later receives its first snapshot. The source's later schema +changes must also leave the tagged schema unchanged. A demonstration server that has not implemented +empty-table reads must restrict tagging to populated tables explicitly. + +Service-owned pins must not expire through ordinary automatic tag-retention settings or be replaced +through user table-tag operations. Table branch deletion removes its metadata directory, including +the tags in that directory. Keep a backing branch while a database tag or merge baseline still needs +it, or relocate the retained metadata before deleting it. + +The first MVP can defer physical deletion and cleanup. Removing a logical database reference need +not immediately drop its underlying table branches or files. Enable physical cleanup only when it +accounts for all retained database references and merge baselines. + +## Merge semantics and execution + +Merge operates on complete table versions, including schema, properties, and snapshot state. It +also defines how table presence or absence is combined once branch-aware DDL is available. + +Let `B`, `S`, and `T` be a table's base, source, and target version, with absence represented as a +state. First determine whether the source changed relative to `B`: + +| Condition or mode | Result | +| --- | --- | +| `S = B` | Keep `T`, including target-only changes. | +| Source changed, mode `DROP` | Keep `T`, even when there would be no conflict. | +| Source changed, mode `FORCE` | Use `S`, including source-side deletion. | +| Source changed, mode `NORMAL`, and `T = B` | Use `S`. | +| Source changed, mode `NORMAL`, and `S = T` | Accept the identical result. | +| Source changed, mode `NORMAL`, and both sides changed differently | Fail the merge with `409`. | + +Different tables can therefore change independently and merge successfully. Different versions of +the same table conflict under `NORMAL`, even when an application might know how to combine their +rows. `FORCE` selects a complete source version; `DROP` skips all source changes to the selected +table, not just conflicting changes. + +### Publication + +1. Resolve the source and target and find their merge base, including earlier merges. +2. Compare table versions and compute the complete result using the selected modes. +3. If any unresolved conflict exists, return `409` before changing target tables. +4. Prepare the selected target table versions with table-level snapshot/schema mechanisms and publish + the database result. Record the source as merged; never modify the source reference. + +When the target is an ancestor of the source, fast-forward is possible only if the chosen modes +produce exactly the source state. Already-merged sources and identical reference states succeed +without changing the target. Divergent histories use three-way merge. + +The existing table `mergeBranch` implementation merges append-only data-file changes. It is not an +implementation of this whole-table-version algorithm. Existing table `fastForward` also has its own +replacement semantics and can remove target metadata and tags. A server needs an adapter that +checks the database result first and preserves retained references; looping over either operation +without that adapter is insufficient. + +Preparing fresh backing branches and publishing a new mapping is one possible server implementation. +A server retaining the explicit table branch names must instead provide a safe way to install the +prepared versions behind those names. The client-visible table address must continue to resolve +correctly. In either case, source and target must remain independently writable: pointing both at +the same mutable table branch would make future source writes modify the target as well. + +### Repeated merge + +After a successful merge, the server records the integrated source version even if `DROP` preserved +all target table contents. Repeating a merge of that same source state must not bring skipped +changes back. A later source write can participate in a subsequent merge using the updated history. + +Do not identify prior merges only by the source branch name; that branch can continue to advance. +Internal ancestry is required even though the public API has no hash. The first MVP can serialize +these operations and pause writers instead of introducing public concurrency tokens or multi-table +transactions. Reads during a multi-table publication need not provide an atomic database view in +this restricted MVP. Partial backend execution still needs a recoverable server operation record; +an HTTP success must mean that the planned result is installed. + +## Exercise the fixed-table MVP + +The following workflow requires a server that implements the orchestration above. The current Java +client tests alone do not provide that server. + +1. Create database `training` and two populated managed tables, `features` and `labels`, on `main`. + Stop writes and create database branch `experiment` from `main` using `/trees`. +2. Read and write the corresponding table branches through existing table addressing. For example, + Spark uses these table names: + + ```sql + SELECT * FROM training.`features$branch_experiment`; + SELECT * FROM training.`labels$branch_experiment`; + ``` + + Batch writes use the same branch-qualified names. An unqualified table name still addresses its + ordinary main table; creating the database branch does not switch the current catalog. +3. Stop experiment writes and create database tag `train_v1` from `experiment`. The server creates + and protects the corresponding table tags. In this explicit-addressing workflow, the training + job records both the source branch and tag name: + + ```sql + SELECT * FROM training.`features$branch_experiment` VERSION AS OF 'train_v1'; + SELECT * FROM training.`labels$branch_experiment` VERSION AS OF 'train_v1'; + ``` + + These examples compose existing table branch and time-travel syntax. The server integration + must verify that both snapshot and schema lookups resolve the protected tag. The database tag + response itself does not return the source branch or a table mapping. +4. Advance the experiment tables, then repeat the tagged reads. They must return the earlier data + and schemas. Keep the tag's backing table branches while these reads are needed. +5. With main and experiment writers stopped, merge `train_v1` into `main`. This publishes the + evaluated source version. Merging the live `experiment` branch would instead include its newer + state. If both sides changed a table, inspect the conflict and deliberately choose a per-table + mode when appropriate. +6. Reload main tables and verify the published state. Resume writes separately on `main` and + `experiment` and verify that neither changes the other. Merge the same source state again to + check no-op behavior. Delete unused database references through tree management. + +## Beyond the fixed-table MVP + +A complete database view needs a defined way for ordinary table APIs to identify the selected +database branch or tag. Existing table branch suffixes identify one table branch; they do not scope +`listTables`, create-table, drop-table, or rename operations to a database reference. + +The following work remains separate from the management API: + +- Address and return the selected reference's table membership, including tables absent on `main`. +- Apply branch-local DDL to that membership, with stable table identities for rename and new + identities for drop-and-recreate. +- Resolve a database tag to its table mappings without requiring the caller to remember its source + branch; expose the frozen schema and empty-table state as well as the snapshot. +- Preserve the selected context through schema/snapshot access, catalog loaders, task serialization, + and caches. Table identifiers by ID and catalog-wide listings need equally explicit semantics. + +This document does not select a new header, query parameter, catalog option, or endpoint for those +operations. Such an extension needs its own agreed wire contract and client integration. Database +reference management can be implemented first, and a fixed-table training workflow can use explicit +table addressing while that namespace design is settled. + +## Validation and implementation sequence + +The existing reference tests validate HTTP paths, request bodies, authentication/configuration, +pagination, JSON compatibility, and exception propagation. Mocked success responses do not test +server branch isolation, snapshot retention, or the merge algorithm. + +Implement and verify in this order: + +1. **Reference records and bootstrap:** create `main`, list/get/create/delete references, and protect + managed table-reference names. +2. **Table orchestration:** clone populated and empty tables correctly; record baselines; route + explicit table branches through existing Paimon readers and writers. +3. **Frozen training inputs:** pin table tags and schemas, validate repeated reads after source + writes, and retain dependencies after logical reference deletion. Add empty-table coverage when + that case is enabled. +4. **Merge:** verify automatic fast-forward, independent changes to different tables, same-table + conflicts leaving the target unchanged, all three modes, repeated merge including `DROP`, and + continued independent writes after merge. +5. **Complete database views:** settle reference-aware listing and DDL, then add membership changes + and tag-only discovery to the integration tests. + +A useful acceptance test uses real Paimon snapshots for two tables and exercises the workflow above +against a stateful server. Passing that test establishes the fixed-table MVP; a full database-view +MVP additionally requires the final namespace step. diff --git a/docs/docs/concepts/rest/index.md b/docs/docs/concepts/rest/index.md index f2974ef44190..d62f8807a26e 100644 --- a/docs/docs/concepts/rest/index.md +++ b/docs/docs/concepts/rest/index.md @@ -74,6 +74,8 @@ Choose the authentication guide for your service: ## API References - [REST Catalog API](./rest-api): the OpenAPI contract for catalog operations. +- [Database Branches and Tags](./database-versioning): experimental reference-management APIs and + the server MVP design using existing table branches and tags. - [REST Management API](./management-api): permissions, row filters, column masking, and the corresponding Spark SQL procedures. diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index 18e54faea4db..c310bb09c1b7 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -52,7 +52,8 @@ payloads, and error responses are defined in the OpenAPI specification. | Commits and snapshots | Commit, roll back, and inspect table versions. | Table-scoped `commit`, `rollback`, `rollback-schema`, `snapshot`, and `snapshots`. | | Data access | Request storage credentials and authorize a query. | Table-scoped `token` and `auth`. | | Partitions | List, create, drop, and mark partitions done. | Table-scoped `partitions`. | -| Branches and tags | Manage named histories and retained snapshots. | Table-scoped `branches` and `tags`. | +| Table branches and tags | Manage named histories and retained snapshots. | Table-scoped `branches` and `tags`. | +| Database branches and tags | List, get, create, delete, and merge references. | Database-scoped `trees` and `trees/{name}/merge`. | | Consumers | List and reset streaming consumer progress. | Table-scoped `consumers`. | | Views and functions | Manage reusable SQL and function definitions. | Database- and catalog-scoped `views` and `functions`. | @@ -60,6 +61,10 @@ In this table, **table-scoped** means `/v1/{prefix}/databases/{database}/tables/{table}`. Catalog-wide listing and detail-listing endpoints are described in the specification alongside their database-scoped counterparts. +See [Database Branches and Tags](./database-versioning) for reference-management examples, merge +modes, and the server MVP design. These management APIs do not switch ordinary table requests to +a database reference automatically. + ## Partition Compatibility Partition options use the existing `POST .../partitions` request. `partitionOptions` follows the diff --git a/docs/docs/program-api/rest-api.mdx b/docs/docs/program-api/rest-api.mdx index ed19d4e380c9..8e7739bb6c7b 100644 --- a/docs/docs/program-api/rest-api.mdx +++ b/docs/docs/program-api/rest-api.mdx @@ -36,6 +36,7 @@ metadata requests without bringing in the full table read/write bundle. | Load a `Table` and read or write rows | [Java API](java-api) with a REST catalog | | Implement an HTTP client or catalog server | [REST API specification](../concepts/rest/rest-api) | | Administrative endpoints | [Management API](../concepts/rest/management-api) | +| Database branch/tag management | [Database Branches and Tags](../concepts/rest/database-versioning#java-management-usage) | ## Dependency diff --git a/docs/sidebars.js b/docs/sidebars.js index 9c9f11a2991c..cdc959aa0bb5 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -44,6 +44,7 @@ const sidebars = { "concepts/rest/tables", "concepts/rest/pvfs", "concepts/rest/rest-api", + "concepts/rest/database-versioning", "concepts/rest/management-api" ] }, From e376e0f5b80cbe8ebf82e5ecf09d356c5b9523ac Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 21:55:09 +0800 Subject: [PATCH 09/12] [rest] Scope table operations to database references --- .../docs/concepts/rest/database-versioning.md | 213 ++++-- docs/docs/concepts/rest/index.md | 2 +- docs/docs/concepts/rest/rest-api.md | 5 +- docs/docs/program-api/rest-api.mdx | 1 + docs/scripts/validate-rest-openapi.js | 37 + docs/static/rest-catalog-open-api.yaml | 720 ++++++++++++++++++ .../apache/paimon/rest/DatabaseReference.java | 8 +- .../java/org/apache/paimon/rest/RESTApi.java | 20 + .../org/apache/paimon/rest/ResourcePaths.java | 143 ++-- .../org/apache/paimon/rest/RESTCatalog.java | 52 +- .../apache/paimon/rest/RESTCatalogLoader.java | 15 +- .../paimon/rest/RESTCatalogReferenceTest.java | 469 ++++++++++++ 12 files changed, 1545 insertions(+), 140 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java diff --git a/docs/docs/concepts/rest/database-versioning.md b/docs/docs/concepts/rest/database-versioning.md index ea956a615452..eef723837b83 100644 --- a/docs/docs/concepts/rest/database-versioning.md +++ b/docs/docs/concepts/rest/database-versioning.md @@ -27,19 +27,20 @@ Database references group the versions of several tables under one branch or tag training workflow starts an experiment from `main`, writes derived data on the experiment branch, freezes the inputs under a tag, and merges accepted changes back into `main`. -This page describes the experimental REST management contract and a proposed server MVP that +This page describes the experimental REST reference and table contracts and a proposed server MVP that reuses Paimon's existing [table branches](../../maintenance/manage-branches) and [table tags](../../maintenance/manage-tags). :::info Implementation status -The Java reference-management client and the `/trees` wire contract are implemented. Reference -storage, table-level orchestration, and database merge execution must be implemented by the catalog -server. The server implementation below is a design, not a claim that an existing service supports it. +The Java reference-management client, reference-scoped table client, and their wire contracts are +implemented. Reference storage, table-level orchestration, and database merge execution must be +implemented by the catalog server. The server implementation below is a design, not a claim that +an existing service supports it. -The MVP uses explicit table-level branch and tag addressing. This page introduces no new reference -header or catalog option. Selecting a complete database view for table listing and DDL remains -additional work, described under [Beyond the fixed-table MVP](#beyond-the-fixed-table-mvp). +Table operations select a database reference through `/trees/{reference}` in the resource path. +Callers use logical table names without constructing table branch suffixes or remembering a tag's +source branch. This requires no new reference header or catalog option. ::: @@ -69,7 +70,8 @@ row-level conflict resolution, or concurrent streaming publication. Branch-local table creation, deletion, and rename need reference-aware namespace handling. The merge contract covers table creation and deletion, but the first fixed-table server can defer those -operations until the table APIs can address the corresponding database view. Format Tables, +operations until reference-aware namespace storage is implemented. Their scoped REST routes already +reuse the ordinary table request and response schemas; rename is deferred. Format Tables, Object Tables, external tables, views, functions, and catalog permissions are outside this initial versioned-table scope. @@ -197,6 +199,108 @@ preserves the resource type/name, message, request ID, and cause. Resource creat `AlreadyExistsException`. See the [OpenAPI specification](/rest-catalog-open-api.yaml) for the individual operations and their documented responses. +## Reference-scoped table API + +Let `S = /v1/{prefix}/databases/{database}/trees/{reference}`. The reference name selects either +an existing branch or an immutable tag. It is resolved by the server; the client need not first +fetch its type. These endpoints reuse the ordinary table request and response structures: + +| Method and path | Existing request / response | Scope | +| --- | --- | --- | +| `GET S/tables` | `ListTablesResponse`; existing paging/filter query parameters. | Table membership of the reference. | +| `GET S/table-details` | `ListTableDetailsResponse`; existing paging/filter query parameters. | Table definitions within the reference. | +| `GET S/tables/{table}` | `GetTableResponse`. | Selected schema, storage options and path. | +| `POST S/tables` | `CreateTableRequest`. | Create a table in a branch. | +| `POST S/tables/{table}` | `AlterTableRequest`. | Alter a table in a branch. | +| `DELETE S/tables/{table}` | Existing drop-table response. | Remove a table from a branch. | +| `GET S/tables/{table}/snapshot` | `GetTableSnapshotResponse`. | Current branch snapshot or pinned tag snapshot. | +| `GET S/tables/{table}/snapshots/{version}` | `GetVersionSnapshotResponse`. | Resolve a version within this reference. | +| `GET S/tables/{table}/snapshots` | `ListSnapshotsResponse`; existing pagination. | Snapshot history visible through this reference. | +| `GET S/tables/{table}/schemas/{version}` | `GetSchemaResponse`. | Resolve a schema ID or `LATEST` within this reference. | +| `GET S/tables/{table}/schemas` | `ListSchemasResponse`; existing pagination. | Schema history retained for this reference. | +| `POST S/tables/{table}/commit` | `CommitTableRequest` / `CommitTableResponse`. | Commit a snapshot to the selected branch. | +| `GET S/tables/{table}/token` | `GetTableTokenResponse`. | Credentials for the resolved table version. | +| `POST S/tables/{table}/auth` | `AuthTableQueryRequest` / `AuthTableQueryResponse`. | Authorize a read of the resolved table. | + +For example, read the same logical table through a live experiment and a frozen training tag: + +```http +GET /v1/catalog/databases/training/trees/experiment/tables/features +GET /v1/catalog/databases/training/trees/train_v1/tables/features +``` + +The response name remains `features`. `GetTableResponse` carries the resolved schema, path and +storage options; the server may supply an internal physical branch in the existing schema options. +A commit keeps the existing `tableId`, `baseSnapshotUuid`, `snapshot`, and `statistics` fields. +The reference path determines the target; identifiers and table IDs in the request must agree with +the table resolved from that path. Caller-supplied table branch suffixes are not part of this contract. + +### Branch and tag behavior + +A branch resolves to its current membership and table versions. A tag resolves to the membership, +schemas, options and snapshots captured when it was created, even after its source branch advances. +Tag snapshot listing exposes only the pinned snapshot. `LATEST` and `EARLIEST` select that snapshot; +other version selectors must resolve to it or return `404`. Schema reads may access the captured +schema and older schemas retained for reading the captured data, but never later source schemas. +An empty captured table is still returned by `GET table`; snapshot lookup returns `404` with +`resourceType: SNAPSHOT`. + +The server rejects content changes through a tag with `409`. Read authorization remains allowed +through `POST .../auth`; HTTP method alone does not determine whether an operation is a write. +Tag credentials must permit reading without allowing mutation of retained metadata or data. + +Missing references and tables return `404`. Unsupported scoped operations return `501`, without +falling back to the ordinary main-table path. Existing unscoped URLs retain their behavior. + +### Java table usage + +Bind a separate client instance to the desired database reference: + +```java +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.rest.RESTCatalog; +import org.apache.paimon.table.Table; + +RESTCatalog experimentCatalog = restCatalog.withReference("training", "experiment"); +RESTCatalog trainingCatalog = restCatalog.withReference("training", "train_v1"); + +Table experimentFeatures = experimentCatalog.getTable(Identifier.create("training", "features")); +Table trainingFeatures = trainingCatalog.getTable(Identifier.create("training", "features")); + +// Use experimentFeatures with the ordinary Paimon batch write API. +// Use trainingFeatures with the ordinary Paimon read API. +``` + +`withReference` leaves the original catalog unchanged and does not fetch catalog configuration +again. Each returned catalog keeps its own binding and local caches. Its serialized +`RESTCatalogLoader`, and loaders inside serialized table objects, retain the binding for later +snapshot reads, commits, schema changes and token requests. Storage commits can supply a physical +table branch internally; the bound catalog sends the logical table name and keeps the reference +path authoritative. + +The lightweight client supports the same binding through +`RESTApi.withReference("training", "experiment")`. Existing table methods and DTOs remain usable. +Table operations must use the bound database. Binding is currently a Java API, not a SQL catalog +option; engine configuration for selecting a reference is additional integration work. + +The scoped client does not yet support global table listing, lookup by table ID, rename, register, +replace, rollback, partition/consumer endpoints, or nested table branch/tag management. Such calls +fail locally instead of reaching an unscoped table route. Database and reference management, +functions, views and catalog-level management retain their existing meaning; this binding versions +only the supported table endpoints. Table policy endpoints are also outside the scoped MVP. + +### Server routing and reuse + +Resolve `(database, reference, logical table)` once into an internal request context containing +reference type, table identity and backing table version. Pass that context into the existing table +handlers. Validate authentication against the actual scoped request, and authorize access to the +resolved table. A path rewrite alone is insufficient: listing must use the selected membership, +tags need frozen metadata, and commits must update the selected branch's recorded table state. + +The additional routing and DTO work is small. Runtime work is a reference/table mapping lookup, +which can be cached; adding the scope does not require proxying or copying table data. Reference +creation, retention, namespace changes and merge still require the server orchestration below. + ## Java management usage Obtain tree management from an already configured `RESTCatalog`. It shares that catalog's @@ -263,7 +367,7 @@ training / train_v1 [TAG] labels -> table identity B, experiment branch, pinned table tag train_v1 ``` -For the explicit-addressing MVP, names such as `experiment` can also name the corresponding table +Names such as `experiment` can also name the corresponding backing table branches, and `train_v1` can name each table tag in its source table branch. These names are owned by the service. Reject collisions with unrelated existing table references; do not adopt them just because the names match. Additional internal baseline tags can use private, service-generated names. @@ -271,7 +375,7 @@ because the names match. Additional internal baseline tags can use private, serv The public database-reference name rules and native table-branch rules are not identical. For example, the database protocol permits a purely numeric name, while native table branch creation rejects it. A server supporting the full name contract needs an alias mapping to valid physical -branch names and must resolve the logical table-branch address through that mapping. It must not +branch names and must resolve the scoped logical table address through that mapping. It must not silently narrow the database API's name rules. The examples use names valid in both layers. ### Minimal metadata @@ -384,9 +488,9 @@ checks the database result first and preserves retained references; looping over without that adapter is insufficient. Preparing fresh backing branches and publishing a new mapping is one possible server implementation. -A server retaining the explicit table branch names must instead provide a safe way to install the -prepared versions behind those names. The client-visible table address must continue to resolve -correctly. In either case, source and target must remain independently writable: pointing both at +The server can update the reference mapping to those prepared versions while retaining any +physical branches needed by tags or merge baselines. The client-visible scoped table address must +continue to resolve correctly. In either case, source and target must remain independently writable: pointing both at the same mutable table branch would make future source writes modify the target as well. ### Repeated merge @@ -404,84 +508,63 @@ an HTTP success must mean that the planned result is installed. ## Exercise the fixed-table MVP -The following workflow requires a server that implements the orchestration above. The current Java -client tests alone do not provide that server. +The following workflow requires a server that implements the orchestration above. The client tests +exercise scoped HTTP routing and table loaders; they do not implement database reference storage +or the database merge algorithm. 1. Create database `training` and two populated managed tables, `features` and `labels`, on `main`. - Stop writes and create database branch `experiment` from `main` using `/trees`. -2. Read and write the corresponding table branches through existing table addressing. For example, - Spark uses these table names: - - ```sql - SELECT * FROM training.`features$branch_experiment`; - SELECT * FROM training.`labels$branch_experiment`; - ``` - - Batch writes use the same branch-qualified names. An unqualified table name still addresses its - ordinary main table; creating the database branch does not switch the current catalog. -3. Stop experiment writes and create database tag `train_v1` from `experiment`. The server creates - and protects the corresponding table tags. In this explicit-addressing workflow, the training - job records both the source branch and tag name: - - ```sql - SELECT * FROM training.`features$branch_experiment` VERSION AS OF 'train_v1'; - SELECT * FROM training.`labels$branch_experiment` VERSION AS OF 'train_v1'; - ``` - - These examples compose existing table branch and time-travel syntax. The server integration - must verify that both snapshot and schema lookups resolve the protected tag. The database tag - response itself does not return the source branch or a table mapping. -4. Advance the experiment tables, then repeat the tagged reads. They must return the earlier data - and schemas. Keep the tag's backing table branches while these reads are needed. + Stop writes and create database branch `experiment` from `main` using tree management. +2. Bind `restCatalog.withReference("training", "experiment")`. List and load `features` and `labels` + by their ordinary names, then write experiment data with the usual batch write API. Their + metadata reads and commits use `/trees/experiment/tables/...`. +3. Stop experiment writes and create database tag `train_v1` from `experiment`. Bind a second + catalog with `restCatalog.withReference("training", "train_v1")`. Load the same logical table + names for training; the service resolves the pinned table versions without a source-branch hint. +4. Advance the experiment tables, then reload and read them through the tag-bound catalog. The + tagged data and schemas must remain unchanged. Verify that writes through the tag are rejected. 5. With main and experiment writers stopped, merge `train_v1` into `main`. This publishes the evaluated source version. Merging the live `experiment` branch would instead include its newer - state. If both sides changed a table, inspect the conflict and deliberately choose a per-table - mode when appropriate. + state. If both sides changed a table, choose a per-table merge mode when appropriate. 6. Reload main tables and verify the published state. Resume writes separately on `main` and `experiment` and verify that neither changes the other. Merge the same source state again to check no-op behavior. Delete unused database references through tree management. ## Beyond the fixed-table MVP -A complete database view needs a defined way for ordinary table APIs to identify the selected -database branch or tag. Existing table branch suffixes identify one table branch; they do not scope -`listTables`, create-table, drop-table, or rename operations to a database reference. - -The following work remains separate from the management API: - -- Address and return the selected reference's table membership, including tables absent on `main`. -- Apply branch-local DDL to that membership, with stable table identities for rename and new - identities for drop-and-recreate. -- Resolve a database tag to its table mappings without requiring the caller to remember its source - branch; expose the frozen schema and empty-table state as well as the snapshot. -- Preserve the selected context through schema/snapshot access, catalog loaders, task serialization, - and caches. Table identifiers by ID and catalog-wide listings need equally explicit semantics. +The REST scope and client binding now identify the selected database view for table listing, +reads, commits and the ordinary create/alter/drop endpoints. A complete server namespace still +needs branch-local membership changes, stable identities across rename, and new identities for +drop-and-recreate. The fixed-table server may return `501` for unsupported scoped DDL. -This document does not select a new header, query parameter, catalog option, or endpoint for those -operations. Such an extension needs its own agreed wire contract and client integration. Database -reference management can be implemented first, and a fixed-table training workflow can use explicit -table addressing while that namespace design is settled. +Global table IDs, global listings, rename and the other deferred endpoints need explicit scope +semantics before they can be enabled on a bound client. SQL engine configuration also needs to +preserve the same binding when constructing catalogs. These additions do not require callers to +construct per-table branch names. ## Validation and implementation sequence -The existing reference tests validate HTTP paths, request bodies, authentication/configuration, -pagination, JSON compatibility, and exception propagation. Mocked success responses do not test -server branch isolation, snapshot retention, or the merge algorithm. +The reference tests validate HTTP paths, request bodies, authentication/configuration, pagination, +JSON compatibility, exception propagation and reference preservation through serialized catalogs +and tables. The OpenAPI validator checks that scoped endpoints reuse the corresponding ordinary +request and success-response structures. A stateful test fixture also uses real Paimon data files +to exercise batch writes on separate branches, frozen tag reads after source writes, and tag write +rejection. This validates client integration with a resolving server; production reference +lifecycle, snapshot retention and database merge still require server integration tests. Implement and verify in this order: 1. **Reference records and bootstrap:** create `main`, list/get/create/delete references, and protect managed table-reference names. 2. **Table orchestration:** clone populated and empty tables correctly; record baselines; route - explicit table branches through existing Paimon readers and writers. + scoped logical table names through existing Paimon readers and writers. 3. **Frozen training inputs:** pin table tags and schemas, validate repeated reads after source writes, and retain dependencies after logical reference deletion. Add empty-table coverage when that case is enabled. 4. **Merge:** verify automatic fast-forward, independent changes to different tables, same-table conflicts leaving the target unchanged, all three modes, repeated merge including `DROP`, and continued independent writes after merge. -5. **Complete database views:** settle reference-aware listing and DDL, then add membership changes - and tag-only discovery to the integration tests. +5. **Complete database views:** implement branch-local DDL storage and verify membership changes, + then extend the scoped protocol to the deferred operations as needed. A useful acceptance test uses real Paimon snapshots for two tables and exercises the workflow above against a stateful server. Passing that test establishes the fixed-table MVP; a full database-view diff --git a/docs/docs/concepts/rest/index.md b/docs/docs/concepts/rest/index.md index d62f8807a26e..6f2c9d18cde0 100644 --- a/docs/docs/concepts/rest/index.md +++ b/docs/docs/concepts/rest/index.md @@ -74,7 +74,7 @@ Choose the authentication guide for your service: ## API References - [REST Catalog API](./rest-api): the OpenAPI contract for catalog operations. -- [Database Branches and Tags](./database-versioning): experimental reference-management APIs and +- [Database Branches and Tags](./database-versioning): experimental reference management, scoped table APIs, and the server MVP design using existing table branches and tags. - [REST Management API](./management-api): permissions, row filters, column masking, and the corresponding Spark SQL procedures. diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index c310bb09c1b7..cae9e3b6e18c 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -62,8 +62,9 @@ In this table, **table-scoped** means endpoints are described in the specification alongside their database-scoped counterparts. See [Database Branches and Tags](./database-versioning) for reference-management examples, merge -modes, and the server MVP design. These management APIs do not switch ordinary table requests to -a database reference automatically. +modes, and the server MVP design. Supported table operations can select a reference through +`/v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}`, using the existing request and +response structures. Java clients bind the scope with `withReference(database, reference)`. ## Partition Compatibility diff --git a/docs/docs/program-api/rest-api.mdx b/docs/docs/program-api/rest-api.mdx index 8e7739bb6c7b..45864e944493 100644 --- a/docs/docs/program-api/rest-api.mdx +++ b/docs/docs/program-api/rest-api.mdx @@ -37,6 +37,7 @@ metadata requests without bringing in the full table read/write bundle. | Implement an HTTP client or catalog server | [REST API specification](../concepts/rest/rest-api) | | Administrative endpoints | [Management API](../concepts/rest/management-api) | | Database branch/tag management | [Database Branches and Tags](../concepts/rest/database-versioning#java-management-usage) | +| Tables within a database branch/tag | [Reference-scoped table usage](../concepts/rest/database-versioning#java-table-usage) | ## Dependency diff --git a/docs/scripts/validate-rest-openapi.js b/docs/scripts/validate-rest-openapi.js index e51eb2295da0..6eaaaa6b6794 100644 --- a/docs/scripts/validate-rest-openapi.js +++ b/docs/scripts/validate-rest-openapi.js @@ -225,6 +225,43 @@ function requireExactEnum(contract, schemaName, expectedValues) { function validateCatalogOpenApi() { const contract = validateCommon('rest-catalog-open-api.yaml'); + const databasePath = '/v1/{prefix}/databases/{database}'; + [ + '/tables', + '/table-details', + '/tables/{table}', + '/tables/{table}/commit', + '/tables/{table}/token', + '/tables/{table}/auth', + '/tables/{table}/snapshot', + '/tables/{table}/snapshots', + '/tables/{table}/snapshots/{version}', + '/tables/{table}/schemas', + '/tables/{table}/schemas/{version}', + ].forEach((suffix) => { + const original = contract.spec.paths[databasePath + suffix]; + const scopedPath = databasePath + '/trees/{reference}' + suffix; + const scoped = contract.spec.paths[scopedPath]; + contract.checkSpec(scoped, `Missing reference-scoped table path: ${scopedPath}`); + Object.entries(original).forEach(([method, operation]) => { + if (!HTTP_METHODS.has(method)) { + return; + } + const counterpart = scoped[method]; + contract.checkSpec(counterpart, `Missing ${method} on ${scopedPath}`); + ['requestBody', 'responses'].forEach((field) => { + const value = (op) => field === 'responses' ? op.responses['200'] : op[field]; + contract.checkSpec( + JSON.stringify(value(operation)) === JSON.stringify(value(counterpart)), + `${scopedPath} must reuse the unscoped ${method} ${field} contract`, + ); + }); + contract.requireResponses(counterpart.operationId, ['404', '501']); + if (method !== 'get' && !suffix.endsWith('/auth')) { + contract.requireResponses(counterpart.operationId, ['409']); + } + }); + }); [ 'getConfig', 'createDatabase', diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 0d9c1b85bcfb..133944ea412e 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -452,6 +452,569 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/trees/{reference}/tables: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + get: + tags: + - database-reference + summary: List tables in a database reference + operationId: listTablesInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. + parameters: + - name: maxResults + in: query + schema: + type: integer + format: int32 + - name: pageToken + in: query + schema: + type: string + - name: tableNamePattern + description: A sql LIKE pattern (%) for table names. Currently, only prefix matching is supported. + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListTablesResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + post: + tags: + - database-reference + summary: Create table in a database reference + operationId: createTableInReference + description: >- + Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table + IDs in the body must agree with the logical table resolved from the path. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CreateTableRequest" + responses: + "200": + description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/table-details: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + get: + tags: + - database-reference + summary: List table details in a database reference + operationId: listTableDetailsInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. + parameters: + - name: maxResults + in: query + schema: + type: integer + format: int32 + - name: pageToken + in: query + schema: + type: string + - name: tableNamePattern + description: A sql LIKE pattern (%) for table names. Currently, only prefix matching is supported. + in: query + schema: + type: string + - name: tableType + description: Filter tables by table type. All table types will be returned if not set or empty. + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListTableDetailsResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: Get table in a database reference + operationId: getTableInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. Return the logical table name and the + resolved schema, path and storage options. Internal branch aliases may be carried in schema + options. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetTableResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + post: + tags: + - database-reference + summary: Alter table in a database reference + operationId: alterTableInReference + description: >- + Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table + IDs in the body must agree with the logical table resolved from the path. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AlterTableRequest" + responses: + "200": + description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + delete: + tags: + - database-reference + summary: Drop table in a database reference + operationId: dropTableInReference + description: >- + Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table + IDs in the body must agree with the logical table resolved from the path. + responses: + "200": + description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/commit: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + post: + tags: + - database-reference + summary: Commit table in a database reference + operationId: commitTableInReference + description: >- + Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table + IDs in the body must agree with the logical table resolved from the path. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CommitTableRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CommitTableResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/token: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: Get table token in a database reference + operationId: getTableTokenInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. Credentials for a tag must allow reading + without allowing mutation of retained metadata or data. + responses: + "200": + description: DataToken for visit data. + content: + application/json: + schema: + $ref: "#/components/schemas/GetTableDataTokenResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/auth: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + post: + tags: + - database-reference + summary: Auth table query in a database reference + operationId: authTableQueryInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AuthTableQueryRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/AuthTableQueryResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshot: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: Get table snapshot in a database reference + operationId: getTableSnapshotInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. An empty captured table returns 404 with + resourceType SNAPSHOT. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetTableSnapshotResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshots/{version}: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + - name: version + in: path + required: true + schema: + type: string + get: + tags: + - database-reference + summary: Get version snapshot in a database reference + operationId: getVersionSnapshotInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. A tag exposes only its pinned snapshot: + LATEST and EARLIEST select it, and other versions must resolve to that snapshot or return 404. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetVersionSnapshotResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshots: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: List snapshots in a database reference + operationId: listSnapshotsInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. A tag exposes only its pinned snapshot: + LATEST and EARLIEST select it, and other versions must resolve to that snapshot or return 404. + parameters: + - name: maxResults + in: query + schema: + type: integer + format: int32 + - name: pageToken + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListSnapshotsResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/schemas: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: List table schemas in a database reference + operationId: listSchemasInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. LATEST selects the captured schema. Schema + history is limited to schemas retained for the captured table version; later source schemas + are not visible. + parameters: + - name: maxResults + in: query + schema: + type: integer + minimum: 0 + - name: pageToken + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListSchemasResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/schemas/{version}: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + - $ref: "#/components/parameters/Version" + get: + tags: + - database-reference + summary: Get table schema in a database reference + operationId: getSchemaInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. LATEST selects the captured schema. Schema + history is limited to schemas retained for the captured table version; later source schemas + are not visible. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetSchemaResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/register: post: tags: @@ -1180,6 +1743,72 @@ paths: $ref: '#/components/responses/TableNotExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tables/{table}/schemas: + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Table" + get: + tags: + - table + summary: List table schemas + operationId: listSchemas + parameters: + - name: maxResults + in: query + schema: + type: integer + minimum: 0 + - name: pageToken + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListSchemasResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "404": + description: Table or schema does not exist. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + /v1/{prefix}/databases/{database}/tables/{table}/schemas/{version}: + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Table" + - $ref: "#/components/parameters/Version" + get: + tags: + - table + summary: Get table schema + operationId: getSchema + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetSchemaResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "404": + description: Table or schema does not exist. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + /v1/{prefix}/databases/{database}/tables/{table}/partitions: get: tags: @@ -2655,10 +3284,70 @@ paths: $ref: '#/components/responses/SemanticViewNotImplementedErrorResponse' components: + parameters: + Prefix: + name: prefix + in: path + required: true + schema: + type: string + Database: + name: database + in: path + required: true + schema: + type: string + Reference: + name: reference + in: path + required: true + schema: + type: string + pattern: ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ + description: An existing database branch or immutable tag. Never falls back to main. + Table: + name: table + in: path + required: true + schema: + type: string + description: Logical table name. A database reference does not require a table branch suffix. + Version: + name: version + in: path + required: true + schema: + type: string + description: A schema ID or LATEST; resolved within the selected table version. + ############################# # Reusable Response Objects # ############################# responses: + ReferenceTableNotExistErrorResponse: + description: >- + Database, reference, table, snapshot or schema does not exist within the selected reference. + Return resourceType SNAPSHOT for an existing table with no snapshot. Never fall back to main. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + ReferenceTableConflictErrorResponse: + description: >- + The target is an immutable tag, the table already exists, or the operation conflicts with the + selected table state. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + ReferenceTableNotImplementedErrorResponse: + description: >- + The server does not implement this reference-scoped table operation. No fallback to an unscoped + route is allowed. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" SemanticViewNotExistErrorResponse: description: Database or semantic view does not exist. content: @@ -2914,6 +3603,37 @@ components: message: Internal Server Error code: 500 schemas: + TableSchema: + allOf: + - $ref: "#/components/schemas/Schema" + - type: object + properties: + version: + type: integer + id: + type: integer + format: int64 + highestFieldId: + type: integer + timeMillis: + type: integer + format: int64 + GetSchemaResponse: + type: object + properties: + schema: + $ref: "#/components/schemas/TableSchema" + ListSchemasResponse: + type: object + properties: + schemas: + type: array + items: + $ref: "#/components/schemas/TableSchema" + nextPageToken: + type: + - string + - "null" SemanticViewDefinition: type: object required: [ format, content ] diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java index 845a68deb003..353ac2d605cb 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java @@ -49,12 +49,16 @@ public DatabaseReference( @JsonProperty(FIELD_TYPE) DatabaseReferenceType type, @JsonProperty(FIELD_NAME) String name) { checkArgument(type != null, "Reference type must not be null"); - checkArgument( - name != null && name.matches(NAME_PATTERN), "Invalid reference name: %s", name); + validateName(name); this.type = type; this.name = name; } + static void validateName(String name) { + checkArgument( + name != null && name.matches(NAME_PATTERN), "Invalid reference name: %s", name); + } + @JsonGetter(FIELD_TYPE) public DatabaseReferenceType getType() { return type; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index decefe3e3712..723bc1e8f7db 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -257,6 +257,26 @@ public RESTApi(Options options, boolean configRequired) { this.resourcePaths = ResourcePaths.forCatalogProperties(options); } + private RESTApi(RESTApi api, ResourcePaths resourcePaths) { + this.client = api.client; + this.restAuthFunction = api.restAuthFunction; + this.options = api.options; + this.resourcePaths = resourcePaths; + } + + /** + * Returns a client whose table operations address one database branch or immutable tag. + * + *

The original client is unchanged. Table names remain logical names, without a table branch + * suffix. The server resolves the reference and enforces tag immutability. Operations without a + * reference-scoped table route are unsupported on this client; database and reference + * management retain their catalog-wide meaning. + */ + @Experimental + public RESTApi withReference(String database, String reference) { + return new RESTApi(this, resourcePaths.withReference(database, reference)); + } + /** Get the configured options which has been merged from REST Server. */ public Options options() { return options; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 0b25fa3de9d2..e1b9d8615c2e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -19,11 +19,14 @@ package org.apache.paimon.rest; import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.management.PermissionResource; import org.apache.paimon.options.Options; import org.apache.paimon.shade.guava30.com.google.common.base.Joiner; +import javax.annotation.Nullable; + import static org.apache.paimon.rest.RESTUtil.encodeString; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -64,9 +67,47 @@ public static ResourcePaths forCatalogProperties(Options options) { } private final String prefix; + @Nullable private final String referenceDatabase; + @Nullable private final String referenceName; public ResourcePaths(String prefix) { - this.prefix = encodeString(prefix); + this(encodeString(prefix), null, null); + } + + private ResourcePaths( + String encodedPrefix, + @Nullable String referenceDatabase, + @Nullable String referenceName) { + this.prefix = encodedPrefix; + this.referenceDatabase = referenceDatabase; + this.referenceName = referenceName; + } + + /** Returns paths for table operations within one database branch or immutable tag. */ + @Experimental + public ResourcePaths withReference(String database, String reference) { + checkArgument(database != null && !database.trim().isEmpty(), "database must not be blank"); + DatabaseReference.validateName(reference); + return new ResourcePaths(prefix, database, reference); + } + + private String tableScope(String database) { + if (referenceName == null) { + return database(database); + } + checkArgument( + referenceDatabase.equals(database), + "Table operation must use reference database %s, not %s", + referenceDatabase, + database); + return databaseTree(database, referenceName); + } + + private void checkUnscoped(String operation) { + if (referenceName != null) { + throw new UnsupportedOperationException( + operation + " is not supported in a database reference scope"); + } } /** Labels attached to one entity, whose canonical name is encoded as a single segment. */ @@ -127,6 +168,7 @@ public String revokePermission() { /** Policy collection nested below its attachment resource. */ @Experimental public String policies(PermissionResource resource) { + checkUnscoped("policies"); resource.validatePolicyAttachment(); return SLASH.join(table(resource.getDatabase(), resource.getTable()), POLICIES); } @@ -164,36 +206,38 @@ public String mergeDatabaseBranch(String databaseName, String branch) { } public String tables(String databaseName) { - return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLES); + return SLASH.join(tableScope(databaseName), TABLES); } public String tableDetails(String databaseName) { - return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLE_DETAILS); + return SLASH.join(tableScope(databaseName), TABLE_DETAILS); } public String tables() { + checkUnscoped("tables"); return SLASH.join(V1, prefix, TABLES); } public String table(String tableId) { + checkUnscoped("table"); return SLASH.join(V1, prefix, TABLES, ID, encodeString(tableId)); } public String table(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName)); + checkArgument( + referenceName == null + || Identifier.create(databaseName, objectName).getBranchName() == null, + "Table branch suffixes cannot be combined with a database reference"); + return SLASH.join(tables(databaseName), encodeString(objectName)); } public String renameTable() { + checkUnscoped("renameTable"); return SLASH.join(V1, prefix, TABLES, "rename"); } public String replaceTable(String databaseName, String objectName) { + checkUnscoped("replaceTable"); return SLASH.join( V1, prefix, @@ -205,17 +249,11 @@ public String replaceTable(String databaseName, String objectName) { } public String commitTable(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - "commit"); + return SLASH.join(table(databaseName, objectName), "commit"); } public String rollbackTable(String databaseName, String objectName) { + checkUnscoped("rollbackTable"); return SLASH.join( V1, prefix, @@ -227,6 +265,7 @@ public String rollbackTable(String databaseName, String objectName) { } public String rollbackSchemaTable(String databaseName, String objectName) { + checkUnscoped("rollbackSchemaTable"); return SLASH.join( V1, prefix, @@ -238,63 +277,28 @@ public String rollbackSchemaTable(String databaseName, String objectName) { } public String registerTable(String databaseName) { + checkUnscoped("registerTable"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), REGISTER); } public String tableToken(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - "token"); + return SLASH.join(table(databaseName, objectName), "token"); } public String tableSnapshot(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - "snapshot"); + return SLASH.join(table(databaseName, objectName), "snapshot"); } public String tableSnapshot(String databaseName, String objectName, String version) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - SNAPSHOTS, - version); + return SLASH.join(snapshots(databaseName, objectName), encodeString(version)); } public String snapshots(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - SNAPSHOTS); + return SLASH.join(table(databaseName, objectName), SNAPSHOTS); } public String schemas(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - SCHEMAS); + return SLASH.join(table(databaseName, objectName), SCHEMAS); } public String schemas(String databaseName, String objectName, String version) { @@ -302,17 +306,11 @@ public String schemas(String databaseName, String objectName, String version) { } public String authTable(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - "auth"); + return SLASH.join(table(databaseName, objectName), "auth"); } public String partitions(String databaseName, String objectName) { + checkUnscoped("partitions"); return SLASH.join( V1, prefix, @@ -324,6 +322,7 @@ public String partitions(String databaseName, String objectName) { } public String dropPartitions(String databaseName, String objectName) { + checkUnscoped("dropPartitions"); return SLASH.join( V1, prefix, @@ -336,6 +335,7 @@ public String dropPartitions(String databaseName, String objectName) { } public String markDonePartitions(String databaseName, String objectName) { + checkUnscoped("markDonePartitions"); return SLASH.join( V1, prefix, @@ -348,6 +348,7 @@ public String markDonePartitions(String databaseName, String objectName) { } public String listPartitionsByNames(String databaseName, String objectName) { + checkUnscoped("listPartitionsByNames"); return SLASH.join( V1, prefix, @@ -360,6 +361,7 @@ public String listPartitionsByNames(String databaseName, String objectName) { } public String listPartitionsByFilter(String databaseName, String objectName) { + checkUnscoped("listPartitionsByFilter"); return SLASH.join( V1, prefix, @@ -372,6 +374,7 @@ public String listPartitionsByFilter(String databaseName, String objectName) { } public String branches(String databaseName, String objectName) { + checkUnscoped("branches"); return SLASH.join( V1, prefix, @@ -383,6 +386,7 @@ public String branches(String databaseName, String objectName) { } public String branch(String databaseName, String objectName, String branchName) { + checkUnscoped("branch"); return SLASH.join( V1, prefix, @@ -395,6 +399,7 @@ public String branch(String databaseName, String objectName, String branchName) } public String forwardBranch(String databaseName, String tableName, String branch) { + checkUnscoped("forwardBranch"); return SLASH.join( V1, prefix, @@ -408,6 +413,7 @@ public String forwardBranch(String databaseName, String tableName, String branch } public String tags(String databaseName, String objectName) { + checkUnscoped("tags"); return SLASH.join( V1, prefix, @@ -419,6 +425,7 @@ public String tags(String databaseName, String objectName) { } public String consumers(String databaseName, String objectName) { + checkUnscoped("consumers"); return SLASH.join( V1, prefix, @@ -430,6 +437,7 @@ public String consumers(String databaseName, String objectName) { } public String resetConsumer(String databaseName, String objectName) { + checkUnscoped("resetConsumer"); return SLASH.join( V1, prefix, @@ -442,6 +450,7 @@ public String resetConsumer(String databaseName, String objectName) { } public String tag(String databaseName, String objectName, String tagName) { + checkUnscoped("tag"); return SLASH.join( V1, prefix, diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 58ffc5edb518..e830e40a1011 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -115,6 +115,8 @@ public class RESTCatalog implements Catalog { private final RESTApi api; private final CatalogContext context; + @Nullable private final String referenceDatabase; + @Nullable private final String referenceName; private final boolean dataTokenEnabled; protected final Map tableDefaultOptions; private final @Nullable LocalCacheManager cacheManager; @@ -124,7 +126,30 @@ public RESTCatalog(CatalogContext context) { } public RESTCatalog(CatalogContext context, boolean configRequired) { - this.api = new RESTApi(context.options(), configRequired); + this(context, configRequired, null, null); + } + + RESTCatalog( + CatalogContext context, + boolean configRequired, + @Nullable String referenceDatabase, + @Nullable String referenceName) { + this( + context, + new RESTApi(context.options(), configRequired), + referenceDatabase, + referenceName); + } + + private RESTCatalog( + CatalogContext context, + RESTApi api, + @Nullable String referenceDatabase, + @Nullable String referenceName) { + this.api = + referenceName == null ? api : api.withReference(referenceDatabase, referenceName); + this.referenceDatabase = referenceDatabase; + this.referenceName = referenceName; this.context = CatalogContext.create( api.options(), @@ -143,7 +168,20 @@ public Map options() { @Override public RESTCatalogLoader catalogLoader() { - return new RESTCatalogLoader(context); + return new RESTCatalogLoader(context, referenceDatabase, referenceName); + } + + /** + * Returns a separate catalog whose table operations use one database branch or immutable tag. + * + *

The binding is preserved by {@link #catalogLoader()}. Use ordinary logical table names; + * the server resolves their backing versions. No additional configuration request is made. + * Database and reference management are not versioned by this binding. + */ + @Experimental + public RESTCatalog withReference(String database, String reference) { + DatabaseReference.validateName(reference); + return new RESTCatalog(context, api, database, reference); } @Experimental @@ -533,6 +571,16 @@ public boolean commitSnapshot( Snapshot snapshot, List statistics) throws TableNotExistException { + // CatalogSnapshotCommit supplies the physical storage branch. A database reference + // already selects the write target, so keep its logical table name on the wire. + if (referenceName != null && identifier.getBranchName() != null) { + identifier = + new Identifier( + identifier.getDatabaseName(), + identifier.getTableName(), + null, + identifier.getSystemTableName()); + } try { return api.commitSnapshot( identifier, tableUuid, baseSnapshotUuid, snapshot, statistics); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java index efc5a0b46ca4..05aa369eaa4d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java @@ -21,15 +21,28 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogLoader; +import javax.annotation.Nullable; + /** Loader to create {@link RESTCatalog}. */ public class RESTCatalogLoader implements CatalogLoader { private static final long serialVersionUID = 1L; private final CatalogContext context; + @Nullable private final String referenceDatabase; + @Nullable private final String referenceName; public RESTCatalogLoader(CatalogContext context) { + this(context, null, null); + } + + RESTCatalogLoader( + CatalogContext context, + @Nullable String referenceDatabase, + @Nullable String referenceName) { this.context = context; + this.referenceDatabase = referenceDatabase; + this.referenceName = referenceName; } public CatalogContext context() { @@ -38,6 +51,6 @@ public CatalogContext context() { @Override public RESTCatalog load() { - return new RESTCatalog(context, false); + return new RESTCatalog(context, false, referenceDatabase, referenceName); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java new file mode 100644 index 000000000000..9ab1269b467f --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java @@ -0,0 +1,469 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.SnapshotCommit; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.Options; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.requests.CommitTableRequest; +import org.apache.paimon.rest.responses.GetSchemaResponse; +import org.apache.paimon.rest.responses.GetTableResponse; +import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.SnapshotManager; + +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.apache.paimon.CoreOptions.BRANCH; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.URI; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies reference scope through ordinary table APIs, serialization and storage commits. */ +class RESTCatalogReferenceTest { + + private static final String DATABASE = "training db"; + private static final Identifier TABLE = Identifier.create(DATABASE, "features"); + private static final String DATABASE_PATH = "/v1/catalog%2Fid/databases/training+db"; + private static final String SNAPSHOT_JSON = + "{\"version\":3,\"id\":7,\"schemaId\":2,\"uuid\":\"snapshot-7\"," + + "\"commitKind\":\"APPEND\",\"commitUser\":\"writer\",\"commitIdentifier\":1," + + "\"timeMillis\":1000,\"totalRecordCount\":3,\"deltaRecordCount\":3}"; + + @TempDir Path tempDir; + + private MockWebServer server; + private RESTCatalog catalog; + + @BeforeEach + void setUp() throws Exception { + server = new MockWebServer(); + server.start(); + enqueue( + 200, + "{\"defaults\":{},\"overrides\":{\"prefix\":\"catalog/id\"," + + "\"header.X-Catalog-Context\":\"configured\"}}"); + Options options = new Options(); + options.set(URI, server.url("/").toString()); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "test-token"); + catalog = new RESTCatalog(CatalogContext.create(options)); + assertThat(server.takeRequest(10, TimeUnit.SECONDS).getPath()).isEqualTo("/v1/config"); + } + + @AfterEach + void tearDown() throws Exception { + catalog.close(); + server.shutdown(); + } + + @ParameterizedTest + @ValueSource(strings = {"experiment", "train_v1"}) + void testTableAndSerializedLoaderKeepReference(String reference) throws Exception { + RESTCatalog scoped = catalog.withReference(DATABASE, reference); + String scope = DATABASE_PATH + "/trees/" + reference; + enqueue(200, "{\"tables\":[\"features\",\"labels\"]}"); + assertThat(scoped.listTables(DATABASE)).containsExactly("features", "labels"); + takeRequest("GET", scope + "/tables"); + + enqueue(200, tableResponse("physical-experiment")); + FileStoreTable table = (FileStoreTable) scoped.getTable(TABLE); + assertThat(table.catalogEnvironment().identifier()).isEqualTo(TABLE); + assertThat(table.snapshotManager().branch()).isEqualTo("physical-experiment"); + assertThat(table.schema().id()).isEqualTo(2); + takeRequest("GET", scope + "/tables/features"); + + // A task receives a serialized table. Its snapshot loader must still address the tree. + FileStoreTable restored = InstantiationUtil.clone(table); + enqueue(200, "{\"snapshot\":{\"snapshot\":" + SNAPSHOT_JSON + "}}"); + assertThat(restored.snapshotManager().latestSnapshot().id()).isEqualTo(7); + takeRequest("GET", scope + "/tables/features/snapshot"); + + RESTCatalog loaded = InstantiationUtil.clone(scoped.catalogLoader()).load(); + enqueue( + 200, + RESTApi.toJson( + new GetSchemaResponse( + TableSchema.create(2, schema("physical-experiment"))))); + assertThat(loaded.loadSchema(TABLE, "LATEST").get().id()).isEqualTo(2); + takeRequest("GET", scope + "/tables/features/schemas/LATEST"); + + // Binding another catalog must not change the original catalog's route or metadata. + enqueue(200, tableResponse("main")); + FileStoreTable main = (FileStoreTable) catalog.getTable(TABLE); + assertThat(main.snapshotManager().branch()).isEqualTo("main"); + takeRequest("GET", DATABASE_PATH + "/tables/features"); + assertThat(server.getRequestCount()).isEqualTo(6); + } + + @Test + void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { + RESTCatalog scoped = catalog.withReference(DATABASE, "experiment"); + enqueue(200, tableResponse("physical-experiment")); + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + takeRequest("GET", DATABASE_PATH + "/trees/experiment/tables/features"); + + Snapshot snapshot = Snapshot.fromJson(SNAPSHOT_JSON); + enqueue(200, "{\"success\":true}"); + try (SnapshotCommit commit = + table.catalogEnvironment().snapshotCommit(table.snapshotManager())) { + assertThat( + commit.commit( + "snapshot-6", + snapshot, + table.snapshotManager().branch(), + emptyList())) + .isTrue(); + } + RecordedRequest request = + takeRequest("POST", DATABASE_PATH + "/trees/experiment/tables/features/commit"); + CommitTableRequest body = + RESTApi.fromJson(request.getBody().readUtf8(), CommitTableRequest.class); + assertThat(body.getTableId()).isEqualTo("table-id"); + assertThat(body.getBaseSnapshotUuid()).isEqualTo("snapshot-6"); + assertThat(body.getSnapshot()).isEqualTo(snapshot); + assertThat(body.getStatistics()).isEmpty(); + } + + @Test + void testReadFollowUpsAndPaginationReuseProtocol() throws Exception { + RESTApi scoped = catalog.api().withReference(DATABASE, "train_v1"); + String scope = DATABASE_PATH + "/trees/train_v1"; + String tablePath = scope + "/tables/features"; + enqueue(200, "{\"tables\":[\"features\"],\"nextPageToken\":\"next\"}"); + assertThat(scoped.listTablesPaged(DATABASE, 1, null, "feat%", null).getNextPageToken()) + .isEqualTo("next"); + RecordedRequest first = takeRequest("GET", scope + "/tables"); + assertThat(first.getRequestUrl().queryParameter("tableNamePattern")).isEqualTo("feat%"); + enqueue(200, "{\"tables\":[\"labels\"]}"); + assertThat(scoped.listTablesPaged(DATABASE, 1, "next", null, null).getElements()) + .containsExactly("labels"); + assertThat( + takeRequest("GET", scope + "/tables") + .getRequestUrl() + .queryParameter("pageToken")) + .isEqualTo("next"); + + enqueue(200, "{\"tableDetails\":[" + tableResponse("physical-experiment") + "]}"); + assertThat(scoped.listTableDetails(DATABASE).get(0).getName()).isEqualTo("features"); + takeRequest("GET", scope + "/table-details"); + + enqueue(200, "{\"snapshot\":" + SNAPSHOT_JSON + "}"); + assertThat(scoped.loadSnapshot(TABLE, "LATEST").id()).isEqualTo(7); + takeRequest("GET", tablePath + "/snapshots/LATEST"); + enqueue(200, "{\"snapshots\":[" + SNAPSHOT_JSON + "]}"); + assertThat(scoped.listSnapshotsPaged(TABLE, 10, null).getElements().get(0).id()) + .isEqualTo(7); + takeRequest("GET", tablePath + "/snapshots"); + + TableSchema schema = TableSchema.create(2, schema("physical-experiment")); + enqueue(200, "{\"schemas\":[" + RESTApi.toJson(schema) + "]}"); + assertThat(scoped.listSchemasPaged(TABLE, 10, null).getElements()).containsExactly(schema); + takeRequest("GET", tablePath + "/schemas"); + + enqueue(200, "{\"token\":{\"key\":\"value\"},\"expiresAtMillis\":1234}"); + assertThat(scoped.loadTableToken(TABLE).getToken()).containsEntry("key", "value"); + takeRequest("GET", tablePath + "/token"); + enqueue(200, "{\"filter\":[],\"columnMasking\":{}}"); + scoped.authTableQuery(TABLE, singletonList("id")); + assertThat(takeRequest("POST", tablePath + "/auth").getBody().readUtf8()) + .isEqualTo("{\"select\":[\"id\"]}"); + } + + @Test + void testTableMutationsReuseRequestBodies() throws Exception { + RESTApi scoped = catalog.api().withReference(DATABASE, "experiment"); + for (RESTApi api : new RESTApi[] {catalog.api(), scoped}) { + enqueue(200, "{}"); + api.createTable(TABLE, schema("main")); + enqueue(200, "{}"); + api.alterTable(TABLE, singletonList(SchemaChange.setOption("key", "value"))); + enqueue(200, "{}"); + api.dropTable(TABLE); + } + RecordedRequest[] original = { + takeRequest("POST", DATABASE_PATH + "/tables"), + takeRequest("POST", DATABASE_PATH + "/tables/features"), + takeRequest("DELETE", DATABASE_PATH + "/tables/features") + }; + String scope = DATABASE_PATH + "/trees/experiment"; + RecordedRequest[] referenced = { + takeRequest("POST", scope + "/tables"), + takeRequest("POST", scope + "/tables/features"), + takeRequest("DELETE", scope + "/tables/features") + }; + for (int i = 0; i < original.length; i++) { + assertThat(referenced[i].getBody().readUtf8()) + .isEqualTo(original[i].getBody().readUtf8()); + } + } + + @Test + void testErrorsDoNotFallBackToDefaultBranch() throws Exception { + RESTCatalog scoped = catalog.withReference(DATABASE, "train_v1"); + enqueue(404, "{\"message\":\"reference missing\",\"code\":404}"); + assertThatThrownBy(() -> scoped.getTable(TABLE)) + .isInstanceOf(Catalog.TableNotExistException.class); + takeRequest("GET", DATABASE_PATH + "/trees/train_v1/tables/features"); + enqueue(409, "{\"message\":\"tag is immutable\",\"code\":409}"); + assertThatThrownBy( + () -> + scoped.commitSnapshot( + TABLE, + "table-id", + null, + Snapshot.fromJson(SNAPSHOT_JSON), + emptyList())) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageContaining("tag is immutable"); + takeRequest("POST", DATABASE_PATH + "/trees/train_v1/tables/features/commit"); + assertThat(server.getRequestCount()).isEqualTo(3); + } + + @Test + void testUnsupportedSelectorsNeverSendAnUnscopedRequest() { + RESTApi scoped = catalog.api().withReference(DATABASE, "experiment"); + assertThatThrownBy(() -> catalog.withReference(DATABASE, null)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> scoped.withReference(DATABASE, "../main")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> scoped.listTables("other")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(DATABASE); + assertThatThrownBy(() -> scoped.getTable(new Identifier(DATABASE, "features", "other"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("branch suffixes"); + assertThatThrownBy(() -> scoped.getTableById("table-id")) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> scoped.listTablesPagedGlobally(null, null, null, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> scoped.renameTable(TABLE, Identifier.create(DATABASE, "renamed"))) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> scoped.createBranch(TABLE, "nested", null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThat(server.getRequestCount()).isEqualTo(1); + } + + @Test + void testBatchReadWriteAndPinnedTagWithRealDataFiles() throws Exception { + // A small stateful fixture resolves references; production reference lifecycle is separate. + org.apache.paimon.fs.Path location = + new org.apache.paimon.fs.Path(tempDir.resolve("features").toUri()); + LocalFileIO fileIO = LocalFileIO.create(); + for (String branch : new String[] {"main", "physical-experiment"}) { + new FileSystemSchemaManager(fileIO, location, branch).createTable(schema(branch)); + } + Map snapshots = new ConcurrentHashMap<>(); + ConcurrentLinkedQueue unexpected = new ConcurrentLinkedQueue<>(); + server.setDispatcher( + new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + try { + String route = request.getRequestUrl().encodedPath(); + String prefix = DATABASE_PATH + "/trees/"; + if (!route.startsWith(prefix)) { + unexpected.add(route); + return response(500, "{}"); + } + String[] parts = route.substring(prefix.length()).split("/"); + String reference = parts[0]; + if (parts.length < 3 + || !parts[1].equals("tables") + || !parts[2].equals("features")) { + unexpected.add(route); + return response(500, "{}"); + } + String branch = + reference.equals("main") ? "main" : "physical-experiment"; + if (request.getMethod().equals("GET") && parts.length == 3) { + return response(200, tableResponse(branch, 0)); + } + if (request.getMethod().equals("GET") + && parts.length == 4 + && parts[3].equals("snapshot")) { + Snapshot snapshot = snapshots.get(reference); + return snapshot == null + ? response( + 404, + "{\"code\":404,\"resourceType\":\"SNAPSHOT\",\"message\":\"empty table\"}") + : response( + 200, + "{\"snapshot\":{\"snapshot\":" + + snapshot.toJson() + + "}}"); + } + if (request.getMethod().equals("POST") + && parts.length == 4 + && parts[3].equals("commit")) { + if (reference.equals("train_v1")) { + return response( + 409, "{\"code\":409,\"message\":\"tag is immutable\"}"); + } + CommitTableRequest commit = + RESTApi.fromJson( + request.getBody().readUtf8(), + CommitTableRequest.class); + Snapshot snapshot = commit.getSnapshot(); + fileIO.overwriteFileUtf8( + new SnapshotManager(fileIO, location, branch, null, null) + .snapshotPath(snapshot.id()), + snapshot.toJson()); + snapshots.put(reference, snapshot); + return response(200, "{\"success\":true}"); + } + unexpected.add(route); + return response(500, "{}"); + } catch (Exception e) { + unexpected.add(e.toString()); + return response(500, "{}"); + } + } + }); + + RESTCatalog main = catalog.withReference(DATABASE, "main"); + RESTCatalog experiment = catalog.withReference(DATABASE, "experiment"); + writeRows(main, 10); + writeRows(experiment, 20); + snapshots.put("train_v1", snapshots.get("experiment")); + RESTCatalog tag = catalog.withReference(DATABASE, "train_v1"); + assertThat(readRows(tag)).containsExactly(20); + + writeRows(experiment, 30); + assertThat(readRows(main)).containsExactly(10); + assertThat(readRows(experiment)).containsExactlyInAnyOrder(20, 30); + // A newly loaded tag table must not follow the source branch's latest snapshot. + assertThat(readRows(tag)).containsExactly(20); + assertThatThrownBy(() -> writeRows(tag, 99)).hasStackTraceContaining("tag is immutable"); + assertThat(readRows(tag)).containsExactly(20); + assertThat(snapshots.get("train_v1").id()).isEqualTo(1); + assertThat(snapshots.get("experiment").id()).isEqualTo(2); + assertThat(unexpected).isEmpty(); + } + + private void writeRows(RESTCatalog scoped, int value) throws Exception { + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + write.write(GenericRow.of(value)); + commit.commit(write.prepareCommit()); + } + } + + private List readRows(RESTCatalog scoped) throws Exception { + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + ReadBuilder builder = table.newReadBuilder(); + List rows = new ArrayList<>(); + try (RecordReader reader = + builder.newRead().createReader(builder.newScan().plan().splits())) { + reader.forEachRemaining(row -> rows.add(row.getInt(0))); + } + return rows; + } + + private Schema schema(String branch) { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .option("bucket", "-1") + .option(BRANCH.key(), branch) + .build(); + } + + private String tableResponse(String branch) throws Exception { + return tableResponse(branch, 2); + } + + private String tableResponse(String branch, long schemaId) throws Exception { + return RESTApi.toJson( + new GetTableResponse( + "table-id", + DATABASE, + "features", + tempDir.resolve("features").toUri().toString(), + false, + schemaId, + schema(branch), + null, + 0, + null, + 0, + null)); + } + + private void enqueue(int status, String body) { + server.enqueue(response(status, body)); + } + + private MockResponse response(int status, String body) { + return new MockResponse() + .setResponseCode(status) + .setHeader("Content-Type", "application/json") + .setBody(body); + } + + private RecordedRequest takeRequest(String method, String path) throws Exception { + RecordedRequest request = server.takeRequest(10, TimeUnit.SECONDS); + assertThat(request).isNotNull(); + assertThat(request.getMethod()).isEqualTo(method); + assertThat(request.getRequestUrl().encodedPath()).isEqualTo(path); + assertThat(request.getHeader("Authorization")).isEqualTo("Bearer test-token"); + assertThat(request.getHeader("X-Catalog-Context")).isEqualTo("configured"); + assertThat(request.getHeader("Paimon-Reference")).isNull(); + return request; + } +} From 217b3de0c6fe16b0ba3ed8e821df956caa79d201 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 22:15:45 +0800 Subject: [PATCH 10/12] [rest] Select database references through name suffixes --- .../docs/concepts/rest/database-versioning.md | 219 +-- docs/docs/concepts/rest/index.md | 2 +- docs/docs/concepts/rest/rest-api.md | 6 +- docs/docs/program-api/rest-api.mdx | 2 +- docs/scripts/validate-rest-openapi.js | 50 +- docs/static/rest-catalog-open-api.yaml | 1295 ++++++----------- .../paimon/rest/DatabaseIdentifier.java | 94 ++ .../apache/paimon/rest/DatabaseReference.java | 8 +- .../java/org/apache/paimon/rest/RESTApi.java | 28 +- .../org/apache/paimon/rest/ResourcePaths.java | 98 +- .../paimon/rest/DatabaseIdentifierTest.java | 81 ++ .../org/apache/paimon/rest/RESTCatalog.java | 50 +- .../apache/paimon/rest/RESTCatalogLoader.java | 15 +- .../paimon/rest/RESTCatalogReferenceTest.java | 189 ++- 14 files changed, 925 insertions(+), 1212 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/DatabaseIdentifier.java create mode 100644 paimon-api/src/test/java/org/apache/paimon/rest/DatabaseIdentifierTest.java diff --git a/docs/docs/concepts/rest/database-versioning.md b/docs/docs/concepts/rest/database-versioning.md index eef723837b83..af694d295358 100644 --- a/docs/docs/concepts/rest/database-versioning.md +++ b/docs/docs/concepts/rest/database-versioning.md @@ -33,14 +33,15 @@ reuses Paimon's existing [table branches](../../maintenance/manage-branches) and :::info Implementation status -The Java reference-management client, reference-scoped table client, and their wire contracts are -implemented. Reference storage, table-level orchestration, and database merge execution must be +The Java reference-management client, database-name selector parser, and their wire contracts are +implemented. Ordinary table APIs carry the selector in the database name. Reference storage, table-level orchestration, and database merge execution must be implemented by the catalog server. The server implementation below is a design, not a claim that an existing service supports it. -Table operations select a database reference through `/trees/{reference}` in the resource path. -Callers use logical table names without constructing table branch suffixes or remembering a tag's -source branch. This requires no new reference header or catalog option. +Table operations select a database reference through a `$branch_` or `$tag_` suffix on +the database name. The existing table paths and request/response structures are reused. Callers use +ordinary table names without remembering a tag's source branch. No reference header, catalog +option, or separately bound client is needed. ::: @@ -201,39 +202,76 @@ individual operations and their documented responses. ## Reference-scoped table API -Let `S = /v1/{prefix}/databases/{database}/trees/{reference}`. The reference name selects either -an existing branch or an immutable tag. It is resolved by the server; the client need not first -fetch its type. These endpoints reuse the ordinary table request and response structures: +A database name can include exactly one reference selector: -| Method and path | Existing request / response | Scope | -| --- | --- | --- | -| `GET S/tables` | `ListTablesResponse`; existing paging/filter query parameters. | Table membership of the reference. | -| `GET S/table-details` | `ListTableDetailsResponse`; existing paging/filter query parameters. | Table definitions within the reference. | -| `GET S/tables/{table}` | `GetTableResponse`. | Selected schema, storage options and path. | -| `POST S/tables` | `CreateTableRequest`. | Create a table in a branch. | -| `POST S/tables/{table}` | `AlterTableRequest`. | Alter a table in a branch. | -| `DELETE S/tables/{table}` | Existing drop-table response. | Remove a table from a branch. | -| `GET S/tables/{table}/snapshot` | `GetTableSnapshotResponse`. | Current branch snapshot or pinned tag snapshot. | -| `GET S/tables/{table}/snapshots/{version}` | `GetVersionSnapshotResponse`. | Resolve a version within this reference. | -| `GET S/tables/{table}/snapshots` | `ListSnapshotsResponse`; existing pagination. | Snapshot history visible through this reference. | -| `GET S/tables/{table}/schemas/{version}` | `GetSchemaResponse`. | Resolve a schema ID or `LATEST` within this reference. | -| `GET S/tables/{table}/schemas` | `ListSchemasResponse`; existing pagination. | Schema history retained for this reference. | -| `POST S/tables/{table}/commit` | `CommitTableRequest` / `CommitTableResponse`. | Commit a snapshot to the selected branch. | -| `GET S/tables/{table}/token` | `GetTableTokenResponse`. | Credentials for the resolved table version. | -| `POST S/tables/{table}/auth` | `AuthTableQueryRequest` / `AuthTableQueryResponse`. | Authorize a read of the resolved table. | - -For example, read the same logical table through a live experiment and a frozen training tag: +| Database name | Meaning | +| --- | --- | +| `training` | The ordinary physical database, with its existing main-table behavior. | +| `training$branch_experiment` | The writable database branch `experiment`. | +| `training$branch_main` | Explicit selection of the database branch `main`. | +| `training$tag_train_v1` | The immutable database tag `train_v1`. | + +The selector is carried in the existing database field, including inside `Identifier`. Encode the +complete database name once as one REST path segment. JSON names remain decoded. For example: ```http -GET /v1/catalog/databases/training/trees/experiment/tables/features -GET /v1/catalog/databases/training/trees/train_v1/tables/features +GET /v1/catalog/databases/training%24branch_experiment/tables/features +GET /v1/catalog/databases/training%24tag_train_v1/tables/features +POST /v1/catalog/databases/training%24branch_experiment/tables/features/commit ``` -The response name remains `features`. `GetTableResponse` carries the resolved schema, path and -storage options; the server may supply an internal physical branch in the existing schema options. -A commit keeps the existing `tableId`, `baseSnapshotUuid`, `snapshot`, and `statistics` fields. -The reference path determines the target; identifiers and table IDs in the request must agree with -the table resolved from that path. Caller-supplied table branch suffixes are not part of this contract. +There are no additional table routes below `/trees/{reference}`. `/trees` remains the reference +management resource and always takes the physical database name, such as `training`. + +Let `D = /v1/{prefix}/databases/{database}` below, where `database` may carry a reference suffix. +These are the existing operations and request/response structures: + +| Method and path | Existing request / response | Scope | +| --- | --- | --- | +| `GET D` | `GetDatabaseResponse`. | Validate the database and selected reference; return virtual database metadata. | +| `GET D/tables` | `ListTablesResponse`; existing paging/filter query parameters. | Table membership of the reference. | +| `GET D/table-details` | `ListTableDetailsResponse`; existing paging/filter query parameters. | Table definitions within the reference. | +| `GET D/tables/{table}` | `GetTableResponse`. | Selected schema, storage options and path. | +| `POST D/tables` | `CreateTableRequest`. | Create a table in a branch. | +| `POST D/tables/{table}` | `AlterTableRequest`. | Alter a table in a branch. | +| `DELETE D/tables/{table}` | Existing drop-table response. | Remove a table from a branch. | +| `GET D/tables/{table}/snapshot` | `GetTableSnapshotResponse`. | Current branch snapshot or pinned tag snapshot. | +| `GET D/tables/{table}/snapshots/{version}` | `GetVersionSnapshotResponse`. | Resolve a version within this reference. | +| `GET D/tables/{table}/snapshots` | `ListSnapshotsResponse`; existing pagination. | Snapshot history visible through this reference. | +| `GET D/tables/{table}/schemas/{version}` | `GetSchemaResponse`. | Resolve a schema ID or `LATEST` within this reference. | +| `GET D/tables/{table}/schemas` | `ListSchemasResponse`; existing pagination. | Schema history retained for this reference. | +| `POST D/tables/{table}/commit` | `CommitTableRequest` / `CommitTableResponse`. | Commit a snapshot to the selected branch. | +| `GET D/tables/{table}/token` | `GetTableTokenResponse`. | Credentials for the resolved table version. | +| `POST D/tables/{table}/auth` | `AuthTableQueryRequest` / `AuthTableQueryResponse`. | Authorize a read of the resolved table. | + +`GetTableResponse` retains the requested database name including its suffix and the logical table +name, such as `features`. It carries the resolved schema, path and storage options; the server may +supply a physical branch alias through existing schema options. Request identifiers retain the +same full database name. A commit keeps the existing `tableId`, `baseSnapshotUuid`, `snapshot`, and +`statistics` fields. The path selects the reference; request identifiers and table IDs must agree +with the resolved table. + +### Database lookup and naming rules + +`GET database` must resolve a suffixed name, because SQL engines can check namespace existence +before accessing a table. The response represents the virtual database and retains its full name. +Database listing returns physical database names only; use `/trees` to discover branches and tags. + +CREATE, DROP and ALTER DATABASE do not accept reference suffixes. In particular, dropping a +virtual database must never drop its physical database. Create, delete and merge references through +`/databases/training/trees` instead. This does not prevent ordinary create/alter/drop **table** +operations from modifying membership or metadata in a writable branch. + +The markers `$branch_` and `$tag_` are case-sensitive reserved syntax. The base database must be +nonblank, and the reference follows the name rules above. Missing names, multiple selectors, or +invalid reference names are rejected rather than interpreted as literal database names. Other +uses of `$`, such as `training$archive`, remain literal. Catalogs adopting this contract must resolve +any pre-existing physical database names containing the reserved markers before enabling it; +lookup must not switch between literal and reference meanings based on which object exists. + +Caller-supplied table branch suffixes cannot be combined with a database selector. For example, +`training$branch_a.features$branch_b` is rejected. Storage commits can supply a physical table branch +internally; RESTCatalog removes that internal table suffix while preserving the database selector. ### Branch and tag behavior @@ -249,57 +287,68 @@ The server rejects content changes through a tag with `409`. Read authorization through `POST .../auth`; HTTP method alone does not determine whether an operation is a write. Tag credentials must permit reading without allowing mutation of retained metadata or data. -Missing references and tables return `404`. Unsupported scoped operations return `501`, without -falling back to the ordinary main-table path. Existing unscoped URLs retain their behavior. +Missing databases, references and tables return `404`. A selector whose type does not match the +reference, such as `$branch_train_v1` for a tag, returns `409`. Malformed selectors return `400`. +Unsupported operations on references return `501`. None of these errors permits retrying the +request against the physical database without its suffix. ### Java table usage -Bind a separate client instance to the desired database reference: +Use the same catalog for ordinary databases and any number of database references: ```java import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.rest.RESTCatalog; import org.apache.paimon.table.Table; -RESTCatalog experimentCatalog = restCatalog.withReference("training", "experiment"); -RESTCatalog trainingCatalog = restCatalog.withReference("training", "train_v1"); +Identifier experiment = Identifier.create("training$branch_experiment", "features"); +Identifier trainingTag = Identifier.create("training$tag_train_v1", "features"); -Table experimentFeatures = experimentCatalog.getTable(Identifier.create("training", "features")); -Table trainingFeatures = trainingCatalog.getTable(Identifier.create("training", "features")); +Table experimentFeatures = restCatalog.getTable(experiment); +Table trainingFeatures = restCatalog.getTable(trainingTag); +restCatalog.listTables("training$branch_experiment"); +restCatalog.getDatabase("training$tag_train_v1"); // Use experimentFeatures with the ordinary Paimon batch write API. // Use trainingFeatures with the ordinary Paimon read API. ``` -`withReference` leaves the original catalog unchanged and does not fetch catalog configuration -again. Each returned catalog keeps its own binding and local caches. Its serialized -`RESTCatalogLoader`, and loaders inside serialized table objects, retain the binding for later -snapshot reads, commits, schema changes and token requests. Storage commits can supply a physical -table branch internally; the bound catalog sends the logical table name and keeps the reference -path authoritative. +`RESTApi` uses these same identifiers with its existing table methods. `Identifier` already retains +the full database name through serialization and in table loaders; no extra reference fields are +stored in RESTCatalog or RESTCatalogLoader. Subsequent snapshot reads, schema changes, commits, +auth and token requests carry the same database name. Caches keyed by full table identifiers +naturally distinguish the physical database, branches and tags. + +SQL clients can pass the selector as a quoted database name, using their ordinary identifier +quoting rules. For example: + +```sql +SELECT * FROM `training$branch_experiment`.features; +SELECT * FROM `training$tag_train_v1`.features; +``` -The lightweight client supports the same binding through -`RESTApi.withReference("training", "experiment")`. Existing table methods and DTOs remain usable. -Table operations must use the bound database. Binding is currently a Java API, not a SQL catalog -option; engine configuration for selecting a reference is additional integration work. +A REST server implementing virtual database lookup and table resolution is required. There is no +new engine catalog option or reference-switch operation. -The scoped client does not yet support global table listing, lookup by table ID, rename, register, -replace, rollback, partition/consumer endpoints, or nested table branch/tag management. Such calls -fail locally instead of reaching an unscoped table route. Database and reference management, -functions, views and catalog-level management retain their existing meaning; this binding versions -only the supported table endpoints. Table policy endpoints are also outside the scoped MVP. +Rename, register, replace, rollback, partition/consumer endpoints, nested table branch/tag +management, views, functions and table policies do not yet accept database reference suffixes in +the Java client. Global table listing and lookup by table ID retain their physical-catalog meaning; +they have no database selector. Extending those operations to discover or address references is +additional work. Catalog-level permissions and reference management continue to use physical names. ### Server routing and reuse -Resolve `(database, reference, logical table)` once into an internal request context containing -reference type, table identity and backing table version. Pass that context into the existing table -handlers. Validate authentication against the actual scoped request, and authorize access to the -resolved table. A path rewrite alone is insufficient: listing must use the selected membership, -tags need frozen metadata, and commits must update the selected branch's recorded table state. +Decode the database path segment and parse it with `DatabaseIdentifier.parse(name)`. The result +contains the physical database name and an optional typed `DatabaseReference`. Resolve that +reference and the logical table once into a request context with table identity and backing version, +then reuse the existing table handlers. Validate authentication against the actual request path +and authorize access to the resolved table. Preserve the full requested database name in returned +identifiers so follow-up calls stay on the same reference. -The additional routing and DTO work is small. Runtime work is a reference/table mapping lookup, -which can be cached; adding the scope does not require proxying or copying table data. Reference -creation, retention, namespace changes and merge still require the server orchestration below. +Parsing the suffix does not replace reference management: listing still needs the selected +membership, tags need frozen metadata, and commits must update the selected branch's recorded +table state. The additional request cost is a reference/table mapping lookup, which can be cached; +this addressing scheme does not require proxying or copying table data. The storage and merge work +remains the server orchestration described below. ## Java management usage @@ -489,9 +538,9 @@ without that adapter is insufficient. Preparing fresh backing branches and publishing a new mapping is one possible server implementation. The server can update the reference mapping to those prepared versions while retaining any -physical branches needed by tags or merge baselines. The client-visible scoped table address must -continue to resolve correctly. In either case, source and target must remain independently writable: pointing both at -the same mutable table branch would make future source writes modify the target as well. +physical branches needed by tags or merge baselines. The client-visible table address must continue +to resolve correctly. Source and target must remain independently writable: pointing both at the +same mutable table branch would make future source writes modify the target as well. ### Repeated merge @@ -509,19 +558,19 @@ an HTTP success must mean that the planned result is installed. ## Exercise the fixed-table MVP The following workflow requires a server that implements the orchestration above. The client tests -exercise scoped HTTP routing and table loaders; they do not implement database reference storage -or the database merge algorithm. +exercise suffix-based HTTP addressing and table loaders; they do not implement database reference +storage or the database merge algorithm. 1. Create database `training` and two populated managed tables, `features` and `labels`, on `main`. Stop writes and create database branch `experiment` from `main` using tree management. -2. Bind `restCatalog.withReference("training", "experiment")`. List and load `features` and `labels` - by their ordinary names, then write experiment data with the usual batch write API. Their - metadata reads and commits use `/trees/experiment/tables/...`. -3. Stop experiment writes and create database tag `train_v1` from `experiment`. Bind a second - catalog with `restCatalog.withReference("training", "train_v1")`. Load the same logical table - names for training; the service resolves the pinned table versions without a source-branch hint. -4. Advance the experiment tables, then reload and read them through the tag-bound catalog. The - tagged data and schemas must remain unchanged. Verify that writes through the tag are rejected. +2. List and load `features` and `labels` from database `training$branch_experiment`, then write + experiment data with the usual batch write API. Their metadata reads and commits use the + existing table paths with the complete suffixed database name. +3. Stop experiment writes and create database tag `train_v1` from `experiment`. Use the same + catalog to access database `training$tag_train_v1`. Load the same logical table names for training; + the service resolves the pinned table versions without a source-branch hint. +4. Advance the experiment tables, then reload and read them through the tag-suffixed database name. + The tagged data and schemas must remain unchanged. Verify that writes through the tag are rejected. 5. With main and experiment writers stopped, merge `train_v1` into `main`. This publishes the evaluated source version. Merging the live `experiment` branch would instead include its newer state. If both sides changed a table, choose a per-table merge mode when appropriate. @@ -531,22 +580,22 @@ or the database merge algorithm. ## Beyond the fixed-table MVP -The REST scope and client binding now identify the selected database view for table listing, +The database name suffix now identifies the selected database view for database lookup, table listing, reads, commits and the ordinary create/alter/drop endpoints. A complete server namespace still needs branch-local membership changes, stable identities across rename, and new identities for drop-and-recreate. The fixed-table server may return `501` for unsupported scoped DDL. -Global table IDs, global listings, rename and the other deferred endpoints need explicit scope -semantics before they can be enabled on a bound client. SQL engine configuration also needs to -preserve the same binding when constructing catalogs. These additions do not require callers to -construct per-table branch names. +Global table IDs, global listings, rename and the other deferred endpoints need explicit reference +semantics before they can be extended. Engine integrations must preserve the full database name in +identifiers and perform database existence checks through the catalog. These additions do not +require callers to construct per-table branch names. ## Validation and implementation sequence The reference tests validate HTTP paths, request bodies, authentication/configuration, pagination, -JSON compatibility, exception propagation and reference preservation through serialized catalogs -and tables. The OpenAPI validator checks that scoped endpoints reuse the corresponding ordinary -request and success-response structures. A stateful test fixture also uses real Paimon data files +JSON compatibility, exception propagation and suffix preservation through serialized tables and +catalog loaders. Tests cover virtual database lookup, mutation guards and malformed or mixed +selectors. The OpenAPI validator checks that reference access uses the ordinary table paths. A stateful test fixture also uses real Paimon data files to exercise batch writes on separate branches, frozen tag reads after source writes, and tag write rejection. This validates client integration with a resolving server; production reference lifecycle, snapshot retention and database merge still require server integration tests. diff --git a/docs/docs/concepts/rest/index.md b/docs/docs/concepts/rest/index.md index 6f2c9d18cde0..8e372fcb6cff 100644 --- a/docs/docs/concepts/rest/index.md +++ b/docs/docs/concepts/rest/index.md @@ -74,7 +74,7 @@ Choose the authentication guide for your service: ## API References - [REST Catalog API](./rest-api): the OpenAPI contract for catalog operations. -- [Database Branches and Tags](./database-versioning): experimental reference management, scoped table APIs, and +- [Database Branches and Tags](./database-versioning): experimental reference management, database-name selectors, and the server MVP design using existing table branches and tags. - [REST Management API](./management-api): permissions, row filters, column masking, and the corresponding Spark SQL procedures. diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index cae9e3b6e18c..c099dff39691 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -62,9 +62,9 @@ In this table, **table-scoped** means endpoints are described in the specification alongside their database-scoped counterparts. See [Database Branches and Tags](./database-versioning) for reference-management examples, merge -modes, and the server MVP design. Supported table operations can select a reference through -`/v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}`, using the existing request and -response structures. Java clients bind the scope with `withReference(database, reference)`. +modes, and the server MVP design. Supported table operations select a reference with a database +name such as `training$branch_experiment` or `training$tag_train_v1`. The existing table paths, +request/response structures and Java methods carry the full database name. ## Partition Compatibility diff --git a/docs/docs/program-api/rest-api.mdx b/docs/docs/program-api/rest-api.mdx index 45864e944493..ec89c6dcb249 100644 --- a/docs/docs/program-api/rest-api.mdx +++ b/docs/docs/program-api/rest-api.mdx @@ -37,7 +37,7 @@ metadata requests without bringing in the full table read/write bundle. | Implement an HTTP client or catalog server | [REST API specification](../concepts/rest/rest-api) | | Administrative endpoints | [Management API](../concepts/rest/management-api) | | Database branch/tag management | [Database Branches and Tags](../concepts/rest/database-versioning#java-management-usage) | -| Tables within a database branch/tag | [Reference-scoped table usage](../concepts/rest/database-versioning#java-table-usage) | +| Tables within a database branch/tag | [Database-name reference selectors](../concepts/rest/database-versioning#java-table-usage) | ## Dependency diff --git a/docs/scripts/validate-rest-openapi.js b/docs/scripts/validate-rest-openapi.js index 6eaaaa6b6794..a7bc8a1029b4 100644 --- a/docs/scripts/validate-rest-openapi.js +++ b/docs/scripts/validate-rest-openapi.js @@ -225,43 +225,19 @@ function requireExactEnum(contract, schemaName, expectedValues) { function validateCatalogOpenApi() { const contract = validateCommon('rest-catalog-open-api.yaml'); - const databasePath = '/v1/{prefix}/databases/{database}'; - [ - '/tables', - '/table-details', - '/tables/{table}', - '/tables/{table}/commit', - '/tables/{table}/token', - '/tables/{table}/auth', - '/tables/{table}/snapshot', - '/tables/{table}/snapshots', - '/tables/{table}/snapshots/{version}', - '/tables/{table}/schemas', - '/tables/{table}/schemas/{version}', - ].forEach((suffix) => { - const original = contract.spec.paths[databasePath + suffix]; - const scopedPath = databasePath + '/trees/{reference}' + suffix; - const scoped = contract.spec.paths[scopedPath]; - contract.checkSpec(scoped, `Missing reference-scoped table path: ${scopedPath}`); - Object.entries(original).forEach(([method, operation]) => { - if (!HTTP_METHODS.has(method)) { - return; - } - const counterpart = scoped[method]; - contract.checkSpec(counterpart, `Missing ${method} on ${scopedPath}`); - ['requestBody', 'responses'].forEach((field) => { - const value = (op) => field === 'responses' ? op.responses['200'] : op[field]; - contract.checkSpec( - JSON.stringify(value(operation)) === JSON.stringify(value(counterpart)), - `${scopedPath} must reuse the unscoped ${method} ${field} contract`, - ); - }); - contract.requireResponses(counterpart.operationId, ['404', '501']); - if (method !== 'get' && !suffix.endsWith('/auth')) { - contract.requireResponses(counterpart.operationId, ['409']); - } - }); - }); + contract.checkSpec( + !Object.keys(contract.spec.paths).some((path) => /\/trees\/\{[^}]+\}\/(tables|table-details)/.test(path)), + 'Database reference access must reuse ordinary table paths', + ); + const databaseParameter = contract.spec.components.parameters.Database; + contract.checkSpec( + databaseParameter.examples.branch.value === 'training$branch_experiment' && + databaseParameter.examples.tag.value === 'training$tag_train_v1', + 'Database reference examples must use the reserved branch and tag suffixes', + ); + ['getDatabase', 'listTables', 'getTable', 'commitTable', 'getSchema', 'listSchemas'].forEach( + (operationId) => contract.requireResponses(operationId, ['404', '409', '501']), + ); [ 'getConfig', 'createDatabase', diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 133944ea412e..0bf557b2eba1 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -84,11 +84,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ListDatabasesResponse' + $ref: "#/components/schemas/ListDatabasesResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + description: >- + List physical databases. Branch and tag access names are not additional database entries; + discover references through /databases/{database}/trees. post: tags: - database @@ -104,16 +107,21 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CreateDatabaseRequest' + $ref: "#/components/schemas/CreateDatabaseRequest" responses: "200": description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "409": - $ref: '#/components/responses/DatabaseAlreadyExistErrorResponse' + $ref: "#/components/responses/DatabaseAlreadyExistErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + description: >- + Create a physical database. The reserved $branch_ and $tag_ suffix markers are not allowed; + create database references through /trees. /v1/{prefix}/databases/{database}: get: tags: @@ -126,24 +134,31 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" responses: "200": description: Get a database by database name. content: application/json: schema: - $ref: '#/components/schemas/GetDatabaseResponse' + $ref: "#/components/schemas/GetDatabaseResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' + $ref: "#/components/responses/DatabaseNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + A database name with $branch_ or $tag_ selects an existing reference. Return + metadata for the virtual database, retaining the full requested name. This lookup supports + engine namespace existence checks. Missing databases or references return 404; a reference + type mismatch returns 409. Never resolve a missing reference to the base database. delete: tags: - database @@ -163,12 +178,17 @@ paths: responses: "200": description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' + $ref: "#/components/responses/DatabaseNotExistErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + description: >- + Database reference suffixes are not allowed for database mutation. Use the /trees management + endpoints with the physical database name to manage references. post: tags: - database @@ -189,20 +209,25 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AlterDatabaseRequest' + $ref: "#/components/schemas/AlterDatabaseRequest" responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/AlterDatabaseResponse' + $ref: "#/components/schemas/AlterDatabaseResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' + $ref: "#/components/responses/DatabaseNotExistErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + description: >- + Database reference suffixes are not allowed for database mutation. Use the /trees management + endpoints with the physical database name to manage references. /v1/{prefix}/databases/{database}/trees: get: tags: @@ -452,23 +477,54 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/trees/{reference}/tables: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" + /v1/{prefix}/databases/{database}/register: + post: + tags: + - table + summary: Register table + operationId: RegisterTable + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RegisterTableRequest' + responses: + "200": + description: Success, no content + "400": + $ref: '#/components/responses/BadRequestErrorResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/DatabaseNotExistErrorResponse' + "409": + $ref: '#/components/responses/TableAlreadyExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tables: get: tags: - - database-reference - summary: List tables in a database reference - operationId: listTablesInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. + - table + summary: List tables + operationId: listTables parameters: + - name: prefix + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/Database" - name: maxResults in: query schema: @@ -498,18 +554,29 @@ paths: $ref: "#/components/responses/ForbiddenErrorResponse" "404": $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": $ref: "#/components/responses/ServerErrorResponse" "501": $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. post: tags: - - database-reference - summary: Create table in a database reference - operationId: createTableInReference - description: >- - Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table - IDs in the body must agree with the logical table resolved from the path. + - table + summary: Create table + operationId: createTable + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/Database" requestBody: content: application/json: @@ -532,23 +599,25 @@ paths: $ref: "#/components/responses/ServerErrorResponse" "501": $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/table-details: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + body must retain the full database name including its suffix and agree with the path. Table + IDs must match the resolved table. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. + /v1/{prefix}/databases/{database}/table-details: get: tags: - - database-reference - summary: List table details in a database reference - operationId: listTableDetailsInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. + - table + summary: List table details + operationId: listTableDetails parameters: + - name: prefix + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/Database" - name: maxResults in: query schema: @@ -583,709 +652,137 @@ paths: $ref: "#/components/responses/ForbiddenErrorResponse" "404": $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": $ref: "#/components/responses/ServerErrorResponse" "501": $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. + /v1/{prefix}/tables: get: tags: - - database-reference - summary: Get table in a database reference - operationId: getTableInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. Return the logical table name and the - resolved schema, path and storage options. Internal branch aliases may be carried in schema - options. + - table + summary: List tables globally + operationId: ListTablesGlobally + description: list tables paged globally which matches the given database name pattern and table name pattern both. + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: databaseNamePattern + description: A sql LIKE pattern (%) for database names. All databases will be returned if not set or empty. Currently, only prefix matching is supported. + in: query + schema: + type: string + - name: tableNamePattern + description: A sql LIKE pattern (%) for table names. All tables will be returned if not set or empty. Currently, only prefix matching is supported. + in: query + schema: + type: string + - name: maxResults + in: query + schema: + type: integer + format: int32 + - name: pageToken + in: query + schema: + type: string + responses: "200": description: OK content: application/json: schema: - $ref: "#/components/schemas/GetTableResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" + $ref: '#/components/schemas/ListTablesGloballyResponse' "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + $ref: '#/components/responses/UnauthorizedErrorResponse' "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - post: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/tables/id/{tableId}: + get: tags: - - database-reference - summary: Alter table in a database reference - operationId: alterTableInReference - description: >- - Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table - IDs in the body must agree with the logical table resolved from the path. - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/AlterTableRequest" + - table + summary: Get table by id + operationId: getTableById + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: tableId + in: path + required: true + schema: + type: string responses: "200": - description: Success, no content - "400": - $ref: "#/components/responses/BadRequestErrorResponse" + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetTableResponse' "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" + $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "409": - $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + $ref: '#/components/responses/TableNotExistErrorResponse' "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - delete: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tables/{table}: + get: tags: - - database-reference - summary: Drop table in a database reference - operationId: dropTableInReference - description: >- - Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table - IDs in the body must agree with the logical table resolved from the path. - responses: - "200": - description: Success, no content - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "409": - $ref: "#/components/responses/ReferenceTableConflictErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/commit: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - post: - tags: - - database-reference - summary: Commit table in a database reference - operationId: commitTableInReference - description: >- - Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table - IDs in the body must agree with the logical table resolved from the path. - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/CommitTableRequest" - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/CommitTableResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "409": - $ref: "#/components/responses/ReferenceTableConflictErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/token: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - get: - tags: - - database-reference - summary: Get table token in a database reference - operationId: getTableTokenInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. Credentials for a tag must allow reading - without allowing mutation of retained metadata or data. - responses: - "200": - description: DataToken for visit data. - content: - application/json: - schema: - $ref: "#/components/schemas/GetTableDataTokenResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/auth: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - post: - tags: - - database-reference - summary: Auth table query in a database reference - operationId: authTableQueryInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/AuthTableQueryRequest" - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/AuthTableQueryResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshot: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - get: - tags: - - database-reference - summary: Get table snapshot in a database reference - operationId: getTableSnapshotInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. An empty captured table returns 404 with - resourceType SNAPSHOT. - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/GetTableSnapshotResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshots/{version}: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - - name: version - in: path - required: true - schema: - type: string - get: - tags: - - database-reference - summary: Get version snapshot in a database reference - operationId: getVersionSnapshotInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. A tag exposes only its pinned snapshot: - LATEST and EARLIEST select it, and other versions must resolve to that snapshot or return 404. - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/GetVersionSnapshotResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshots: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - get: - tags: - - database-reference - summary: List snapshots in a database reference - operationId: listSnapshotsInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. A tag exposes only its pinned snapshot: - LATEST and EARLIEST select it, and other versions must resolve to that snapshot or return 404. - parameters: - - name: maxResults - in: query - schema: - type: integer - format: int32 - - name: pageToken - in: query - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/ListSnapshotsResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/schemas: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - get: - tags: - - database-reference - summary: List table schemas in a database reference - operationId: listSchemasInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. LATEST selects the captured schema. Schema - history is limited to schemas retained for the captured table version; later source schemas - are not visible. - parameters: - - name: maxResults - in: query - schema: - type: integer - minimum: 0 - - name: pageToken - in: query - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/ListSchemasResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/schemas/{version}: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - - $ref: "#/components/parameters/Version" - get: - tags: - - database-reference - summary: Get table schema in a database reference - operationId: getSchemaInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. LATEST selects the captured schema. Schema - history is limited to schemas retained for the captured table version; later source schemas - are not visible. - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/GetSchemaResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - - /v1/{prefix}/databases/{database}/register: - post: - tags: - - table - summary: Register table - operationId: RegisterTable - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/RegisterTableRequest' - responses: - "200": - description: Success, no content - "400": - $ref: '#/components/responses/BadRequestErrorResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' - "409": - $ref: '#/components/responses/TableAlreadyExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/tables: - get: - tags: - - table - summary: List tables - operationId: listTables - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - - name: maxResults - in: query - schema: - type: integer - format: int32 - - name: pageToken - in: query - schema: - type: string - - name: tableNamePattern - description: A sql LIKE pattern (%) for table names. Currently, only prefix matching is supported. - in: query - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/ListTablesResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - post: - tags: - - table - summary: Create table - operationId: createTable - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/CreateTableRequest' - responses: - "200": - description: Success, no content - "400": - $ref: '#/components/responses/BadRequestErrorResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' - "409": - $ref: '#/components/responses/TableAlreadyExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/table-details: - get: - tags: - - table - summary: List table details - operationId: listTableDetails - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - - name: maxResults - in: query - schema: - type: integer - format: int32 - - name: pageToken - in: query - schema: - type: string - - name: tableNamePattern - description: A sql LIKE pattern (%) for table names. Currently, only prefix matching is supported. - in: query - schema: - type: string - - name: tableType - description: Filter tables by table type. All table types will be returned if not set or empty. - in: query - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/ListTableDetailsResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/tables: - get: - tags: - - table - summary: List tables globally - operationId: ListTablesGlobally - description: list tables paged globally which matches the given database name pattern and table name pattern both. - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: databaseNamePattern - description: A sql LIKE pattern (%) for database names. All databases will be returned if not set or empty. Currently, only prefix matching is supported. - in: query - schema: - type: string - - name: tableNamePattern - description: A sql LIKE pattern (%) for table names. All tables will be returned if not set or empty. Currently, only prefix matching is supported. - in: query - schema: - type: string - - name: maxResults - in: query - schema: - type: integer - format: int32 - - name: pageToken - in: query - schema: - type: string - - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/ListTablesGloballyResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/tables/id/{tableId}: - get: - tags: - - table - summary: Get table by id - operationId: getTableById - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: tableId - in: path - required: true - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/GetTableResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/TableNotExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/tables/{table}: - get: - tags: - - table - summary: Get table - operationId: getTable - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - - name: table - in: path - required: true - schema: - type: string + - table + summary: Get table + operationId: getTable + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/Database" + - name: table + in: path + required: true + schema: + type: string responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/GetTableResponse' + $ref: "#/components/schemas/GetTableResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Return the requested database name including its suffix, + the logical table name, and resolved schema, path and storage options. Internal branch aliases + may be carried in schema options. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. post: tags: - table @@ -1297,11 +794,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1311,20 +804,30 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AlterTableRequest' + $ref: "#/components/schemas/AlterTableRequest" responses: "200": description: Success, no content "400": - $ref: '#/components/responses/BadRequestErrorResponse' + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" "409": - $ref: '#/components/responses/TableAlreadyExistErrorResponse' + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + body must retain the full database name including its suffix and agree with the path. Table + IDs must match the resolved table. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. delete: tags: - table @@ -1336,11 +839,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1349,12 +848,26 @@ paths: responses: "200": description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + body must retain the full database name including its suffix and agree with the path. Table + IDs must match the resolved table. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. /v1/{prefix}/tables/rename: post: tags: @@ -1397,11 +910,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1411,22 +920,34 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CommitTableRequest' + $ref: "#/components/schemas/CommitTableRequest" responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/CommitTableResponse' + $ref: "#/components/schemas/CommitTableResponse" "400": - $ref: '#/components/responses/BadRequestErrorResponse' + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + body must retain the full database name including its suffix and agree with the path. Table + IDs must match the resolved table. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/rollback: post: tags: @@ -1526,11 +1047,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1542,13 +1059,27 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GetTableDataTokenResponse' + $ref: "#/components/schemas/GetTableDataTokenResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Tag credentials must allow reading without permitting + mutation of retained metadata or data. Missing references never fall back to the base + database. A suffix whose type does not match the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/auth: post: tags: @@ -1561,11 +1092,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1575,31 +1102,33 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AuthTableQueryRequest' + $ref: "#/components/schemas/AuthTableQueryRequest" responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/AuthTableQueryResponse' + $ref: "#/components/schemas/AuthTableQueryResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "403": - $ref: '#/components/responses/ForbiddenErrorResponse' - 404: - description: - Not Found - - TableNotExistException, table does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - examples: - TableNotExist: - $ref: '#/components/examples/TableNotExistError' + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/snapshot: get: tags: @@ -1612,11 +1141,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1628,25 +1153,27 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GetTableSnapshotResponse' + $ref: "#/components/schemas/GetTableSnapshotResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - 404: - description: - Not Found - - TableNotExistException, table does not exist - - SnapshotNotExistException, the requested snapshot does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - examples: - TableNotExist: - $ref: '#/components/examples/TableNotExistError' - SnapshotNotExist: - $ref: '#/components/examples/SnapshotNotExistError' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. An existing empty table returns 404 with resourceType + SNAPSHOT. Missing references never fall back to the base database. A suffix whose type does + not match the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/snapshots/{version}: get: tags: @@ -1659,11 +1186,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1680,25 +1203,28 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GetVersionSnapshotResponse' + $ref: "#/components/schemas/GetVersionSnapshotResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - 404: - description: - Not Found - - TableNotExistException, table does not exist - - SnapshotNotExistException, the requested snapshot does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - examples: - TableNotExist: - $ref: '#/components/examples/TableNotExistError' - SnapshotNotExist: - $ref: '#/components/examples/SnapshotNotExistError' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. A database tag exposes only its pinned snapshot: LATEST + and EARLIEST select it; other versions must resolve to it or return 404. Missing references + never fall back to the base database. A suffix whose type does not match the reference returns + 409. /v1/{prefix}/databases/{database}/tables/{table}/snapshots: get: tags: @@ -1711,11 +1237,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1736,13 +1258,28 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ListSnapshotsResponse' + $ref: "#/components/schemas/ListSnapshotsResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. A database tag exposes only its pinned snapshot: LATEST + and EARLIEST select it; other versions must resolve to it or return 404. Missing references + never fall back to the base database. A suffix whose type does not match the reference returns + 409. /v1/{prefix}/databases/{database}/tables/{table}/schemas: parameters: - $ref: "#/components/parameters/Prefix" @@ -1770,16 +1307,27 @@ paths: application/json: schema: $ref: "#/components/schemas/ListSchemasResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - description: Table or schema does not exist. - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. For a database tag, LATEST selects the captured schema. + History is limited to schemas retained for its captured data, excluding newer source schemas. + Missing references never fall back to the base database. A suffix whose type does not match + the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/schemas/{version}: parameters: - $ref: "#/components/parameters/Prefix" @@ -1798,17 +1346,27 @@ paths: application/json: schema: $ref: "#/components/schemas/GetSchemaResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - description: Table or schema does not exist. - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": $ref: "#/components/responses/ServerErrorResponse" - + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. For a database tag, LATEST selects the captured schema. + History is limited to schemas retained for its captured data, excluding newer source schemas. + Missing references never fall back to the base database. A suffix whose type does not match + the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/partitions: get: tags: @@ -3297,14 +2855,17 @@ components: required: true schema: type: string - Reference: - name: reference - in: path - required: true - schema: - type: string - pattern: ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ - description: An existing database branch or immutable tag. Never falls back to main. + description: >- + Decoded database name. On supported table operations and GET database, + $branch_ selects a writable branch and $tag_ selects + an immutable tag. The markers are case-sensitive and reserved; exactly one valid reference + suffix is allowed. Encode the complete name as one path segment. A suffix is not a physical + database name. + examples: + branch: + value: training$branch_experiment + tag: + value: training$tag_train_v1 Table: name: table in: path @@ -3334,16 +2895,16 @@ components: $ref: "#/components/schemas/ErrorResponse" ReferenceTableConflictErrorResponse: description: >- - The target is an immutable tag, the table already exists, or the operation conflicts with the - selected table state. + The reference type does not match the suffix, the target is an immutable tag, the table already + exists, or the operation conflicts with the selected table state. content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" ReferenceTableNotImplementedErrorResponse: description: >- - The server does not implement this reference-scoped table operation. No fallback to an unscoped - route is allowed. + The server does not implement this operation on a database reference. No fallback to the + physical database is allowed. content: application/json: schema: diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseIdentifier.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseIdentifier.java new file mode 100644 index 000000000000..6b21c26f7241 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseIdentifier.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.catalog.Identifier; + +import javax.annotation.Nullable; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** A REST database name and its optional database-level branch or immutable tag selector. */ +@Experimental +public final class DatabaseIdentifier { + + private static final String BRANCH_SUFFIX = "$branch_"; + private static final String TAG_SUFFIX = "$tag_"; + + private final String databaseName; + @Nullable private final DatabaseReference reference; + + private DatabaseIdentifier(String databaseName, @Nullable DatabaseReference reference) { + this.databaseName = databaseName; + this.reference = reference; + } + + /** + * Parses a decoded database name, such as {@code training$branch_experiment}. + * + *

The suffixes {@code $branch_} and {@code $tag_} are reserved. A name containing either + * marker must have exactly one valid reference suffix. Other dollar signs remain literal. + * Callers retain the original name in table identifiers and encode it as one REST path segment. + */ + public static DatabaseIdentifier parse(String name) { + checkArgument(name != null && !name.trim().isEmpty(), "Database name must not be blank"); + int branch = name.indexOf(BRANCH_SUFFIX); + int tag = name.indexOf(TAG_SUFFIX); + if (branch < 0 && tag < 0) { + return new DatabaseIdentifier(name, null); + } + boolean isBranch = branch >= 0 && (tag < 0 || branch < tag); + int separator = isBranch ? branch : tag; + String database = name.substring(0, separator); + checkArgument(!database.trim().isEmpty(), "Database name must not be blank"); + String reference = + name.substring( + separator + (isBranch ? BRANCH_SUFFIX.length() : TAG_SUFFIX.length())); + return new DatabaseIdentifier( + database, + new DatabaseReference( + isBranch ? DatabaseReferenceType.BRANCH : DatabaseReferenceType.TAG, + reference)); + } + + /** The physical database name, without the reference suffix. */ + public String getDatabaseName() { + return databaseName; + } + + @Nullable + public DatabaseReference getReference() { + return reference; + } + + static void checkNoReference(String database, String operation) { + if (parse(database).getReference() != null) { + throw new UnsupportedOperationException( + operation + " does not support database reference suffixes: " + database); + } + } + + static void checkTableName(String database, String table) { + checkArgument( + parse(database).getReference() == null + || Identifier.create(database, table).getBranchName() == null, + "Table branch suffixes cannot be combined with a database reference"); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java index 353ac2d605cb..845a68deb003 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java @@ -49,14 +49,10 @@ public DatabaseReference( @JsonProperty(FIELD_TYPE) DatabaseReferenceType type, @JsonProperty(FIELD_NAME) String name) { checkArgument(type != null, "Reference type must not be null"); - validateName(name); - this.type = type; - this.name = name; - } - - static void validateName(String name) { checkArgument( name != null && name.matches(NAME_PATTERN), "Invalid reference name: %s", name); + this.type = type; + this.name = name; } @JsonGetter(FIELD_TYPE) diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 723bc1e8f7db..e71f674266a7 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -257,26 +257,6 @@ public RESTApi(Options options, boolean configRequired) { this.resourcePaths = ResourcePaths.forCatalogProperties(options); } - private RESTApi(RESTApi api, ResourcePaths resourcePaths) { - this.client = api.client; - this.restAuthFunction = api.restAuthFunction; - this.options = api.options; - this.resourcePaths = resourcePaths; - } - - /** - * Returns a client whose table operations address one database branch or immutable tag. - * - *

The original client is unchanged. Table names remain logical names, without a table branch - * suffix. The server resolves the reference and enforces tag immutability. Operations without a - * reference-scoped table route are unsupported on this client; database and reference - * management retain their catalog-wide meaning. - */ - @Experimental - public RESTApi withReference(String database, String reference) { - return new RESTApi(this, resourcePaths.withReference(database, reference)); - } - /** Get the configured options which has been merged from REST Server. */ public Options options() { return options; @@ -340,6 +320,7 @@ public PagedList listDatabasesPaged( * this database */ public void createDatabase(String name, Map properties) { + DatabaseIdentifier.checkNoReference(name, "createDatabase"); CreateDatabaseRequest request = new CreateDatabaseRequest(name, properties); client.post(resourcePaths.databases(), request, restAuthFunction); } @@ -367,6 +348,7 @@ public GetDatabaseResponse getDatabase(String name) { * this database */ public void dropDatabase(String name) { + DatabaseIdentifier.checkNoReference(name, "dropDatabase"); client.delete(resourcePaths.database(name), restAuthFunction); } @@ -381,6 +363,7 @@ public void dropDatabase(String name) { * this database */ public void alterDatabase(String name, List removals, Map updates) { + DatabaseIdentifier.checkNoReference(name, "alterDatabase"); client.post( resourcePaths.database(name), new AlterDatabaseRequest(removals, updates), @@ -934,6 +917,7 @@ public PagedList listSchemasPaged( * creating table */ public void createTable(Identifier identifier, Schema schema) { + DatabaseIdentifier.checkTableName(identifier.getDatabaseName(), identifier.getObjectName()); CreateTableRequest request = new CreateTableRequest(identifier, schema); client.post(resourcePaths.tables(identifier.getDatabaseName()), request, restAuthFunction); } @@ -949,6 +933,8 @@ public void createTable(Identifier identifier, Schema schema) { * renaming table */ public void renameTable(Identifier fromTable, Identifier toTable) { + DatabaseIdentifier.checkNoReference(fromTable.getDatabaseName(), "renameTable"); + DatabaseIdentifier.checkNoReference(toTable.getDatabaseName(), "renameTable"); RenameTableRequest request = new RenameTableRequest(fromTable, toTable); client.post(resourcePaths.renameTable(), request, restAuthFunction); } @@ -2048,6 +2034,8 @@ public PagedList listViewsPagedGlobally( * views */ public void renameView(Identifier fromView, Identifier toView) { + DatabaseIdentifier.checkNoReference(fromView.getDatabaseName(), "renameView"); + DatabaseIdentifier.checkNoReference(toView.getDatabaseName(), "renameView"); RenameTableRequest request = new RenameTableRequest(fromView, toView); client.post(resourcePaths.renameView(), request, restAuthFunction); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index e1b9d8615c2e..22d6ce38d81f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -19,14 +19,11 @@ package org.apache.paimon.rest; import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.management.PermissionResource; import org.apache.paimon.options.Options; import org.apache.paimon.shade.guava30.com.google.common.base.Joiner; -import javax.annotation.Nullable; - import static org.apache.paimon.rest.RESTUtil.encodeString; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -67,47 +64,9 @@ public static ResourcePaths forCatalogProperties(Options options) { } private final String prefix; - @Nullable private final String referenceDatabase; - @Nullable private final String referenceName; public ResourcePaths(String prefix) { - this(encodeString(prefix), null, null); - } - - private ResourcePaths( - String encodedPrefix, - @Nullable String referenceDatabase, - @Nullable String referenceName) { - this.prefix = encodedPrefix; - this.referenceDatabase = referenceDatabase; - this.referenceName = referenceName; - } - - /** Returns paths for table operations within one database branch or immutable tag. */ - @Experimental - public ResourcePaths withReference(String database, String reference) { - checkArgument(database != null && !database.trim().isEmpty(), "database must not be blank"); - DatabaseReference.validateName(reference); - return new ResourcePaths(prefix, database, reference); - } - - private String tableScope(String database) { - if (referenceName == null) { - return database(database); - } - checkArgument( - referenceDatabase.equals(database), - "Table operation must use reference database %s, not %s", - referenceDatabase, - database); - return databaseTree(database, referenceName); - } - - private void checkUnscoped(String operation) { - if (referenceName != null) { - throw new UnsupportedOperationException( - operation + " is not supported in a database reference scope"); - } + this.prefix = encodeString(prefix); } /** Labels attached to one entity, whose canonical name is encoded as a single segment. */ @@ -139,6 +98,7 @@ private static String encodePathSegment(String value) { @Experimental public String semanticViews(String database) { checkArgument(database != null && !database.trim().isEmpty(), "database must not be blank"); + DatabaseIdentifier.checkNoReference(database, "semanticViews"); return SLASH.join(V1, prefix, DATABASES, encodePathSegment(database), SEMANTIC_VIEWS); } @@ -168,8 +128,8 @@ public String revokePermission() { /** Policy collection nested below its attachment resource. */ @Experimental public String policies(PermissionResource resource) { - checkUnscoped("policies"); resource.validatePolicyAttachment(); + DatabaseIdentifier.checkNoReference(resource.getDatabase(), "policies"); return SLASH.join(table(resource.getDatabase(), resource.getTable()), POLICIES); } @@ -184,12 +144,14 @@ public String databases() { } public String database(String databaseName) { + DatabaseIdentifier.parse(databaseName); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName)); } /** Database-level branches and immutable tags. */ @Experimental public String databaseTrees(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "tree management"); return SLASH.join(database(databaseName), TREES); } @@ -206,38 +168,32 @@ public String mergeDatabaseBranch(String databaseName, String branch) { } public String tables(String databaseName) { - return SLASH.join(tableScope(databaseName), TABLES); + return SLASH.join(database(databaseName), TABLES); } public String tableDetails(String databaseName) { - return SLASH.join(tableScope(databaseName), TABLE_DETAILS); + return SLASH.join(database(databaseName), TABLE_DETAILS); } public String tables() { - checkUnscoped("tables"); return SLASH.join(V1, prefix, TABLES); } public String table(String tableId) { - checkUnscoped("table"); return SLASH.join(V1, prefix, TABLES, ID, encodeString(tableId)); } public String table(String databaseName, String objectName) { - checkArgument( - referenceName == null - || Identifier.create(databaseName, objectName).getBranchName() == null, - "Table branch suffixes cannot be combined with a database reference"); + DatabaseIdentifier.checkTableName(databaseName, objectName); return SLASH.join(tables(databaseName), encodeString(objectName)); } public String renameTable() { - checkUnscoped("renameTable"); return SLASH.join(V1, prefix, TABLES, "rename"); } public String replaceTable(String databaseName, String objectName) { - checkUnscoped("replaceTable"); + DatabaseIdentifier.checkNoReference(databaseName, "replaceTable"); return SLASH.join( V1, prefix, @@ -253,7 +209,7 @@ public String commitTable(String databaseName, String objectName) { } public String rollbackTable(String databaseName, String objectName) { - checkUnscoped("rollbackTable"); + DatabaseIdentifier.checkNoReference(databaseName, "rollbackTable"); return SLASH.join( V1, prefix, @@ -265,7 +221,7 @@ public String rollbackTable(String databaseName, String objectName) { } public String rollbackSchemaTable(String databaseName, String objectName) { - checkUnscoped("rollbackSchemaTable"); + DatabaseIdentifier.checkNoReference(databaseName, "rollbackSchemaTable"); return SLASH.join( V1, prefix, @@ -277,7 +233,7 @@ public String rollbackSchemaTable(String databaseName, String objectName) { } public String registerTable(String databaseName) { - checkUnscoped("registerTable"); + DatabaseIdentifier.checkNoReference(databaseName, "registerTable"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), REGISTER); } @@ -310,7 +266,7 @@ public String authTable(String databaseName, String objectName) { } public String partitions(String databaseName, String objectName) { - checkUnscoped("partitions"); + DatabaseIdentifier.checkNoReference(databaseName, "partitions"); return SLASH.join( V1, prefix, @@ -322,7 +278,7 @@ public String partitions(String databaseName, String objectName) { } public String dropPartitions(String databaseName, String objectName) { - checkUnscoped("dropPartitions"); + DatabaseIdentifier.checkNoReference(databaseName, "dropPartitions"); return SLASH.join( V1, prefix, @@ -335,7 +291,7 @@ public String dropPartitions(String databaseName, String objectName) { } public String markDonePartitions(String databaseName, String objectName) { - checkUnscoped("markDonePartitions"); + DatabaseIdentifier.checkNoReference(databaseName, "markDonePartitions"); return SLASH.join( V1, prefix, @@ -348,7 +304,7 @@ public String markDonePartitions(String databaseName, String objectName) { } public String listPartitionsByNames(String databaseName, String objectName) { - checkUnscoped("listPartitionsByNames"); + DatabaseIdentifier.checkNoReference(databaseName, "listPartitionsByNames"); return SLASH.join( V1, prefix, @@ -361,7 +317,7 @@ public String listPartitionsByNames(String databaseName, String objectName) { } public String listPartitionsByFilter(String databaseName, String objectName) { - checkUnscoped("listPartitionsByFilter"); + DatabaseIdentifier.checkNoReference(databaseName, "listPartitionsByFilter"); return SLASH.join( V1, prefix, @@ -374,7 +330,7 @@ public String listPartitionsByFilter(String databaseName, String objectName) { } public String branches(String databaseName, String objectName) { - checkUnscoped("branches"); + DatabaseIdentifier.checkNoReference(databaseName, "branches"); return SLASH.join( V1, prefix, @@ -386,7 +342,7 @@ public String branches(String databaseName, String objectName) { } public String branch(String databaseName, String objectName, String branchName) { - checkUnscoped("branch"); + DatabaseIdentifier.checkNoReference(databaseName, "branch"); return SLASH.join( V1, prefix, @@ -399,7 +355,7 @@ public String branch(String databaseName, String objectName, String branchName) } public String forwardBranch(String databaseName, String tableName, String branch) { - checkUnscoped("forwardBranch"); + DatabaseIdentifier.checkNoReference(databaseName, "forwardBranch"); return SLASH.join( V1, prefix, @@ -413,7 +369,7 @@ public String forwardBranch(String databaseName, String tableName, String branch } public String tags(String databaseName, String objectName) { - checkUnscoped("tags"); + DatabaseIdentifier.checkNoReference(databaseName, "tags"); return SLASH.join( V1, prefix, @@ -425,7 +381,7 @@ public String tags(String databaseName, String objectName) { } public String consumers(String databaseName, String objectName) { - checkUnscoped("consumers"); + DatabaseIdentifier.checkNoReference(databaseName, "consumers"); return SLASH.join( V1, prefix, @@ -437,7 +393,7 @@ public String consumers(String databaseName, String objectName) { } public String resetConsumer(String databaseName, String objectName) { - checkUnscoped("resetConsumer"); + DatabaseIdentifier.checkNoReference(databaseName, "resetConsumer"); return SLASH.join( V1, prefix, @@ -450,7 +406,7 @@ public String resetConsumer(String databaseName, String objectName) { } public String tag(String databaseName, String objectName, String tagName) { - checkUnscoped("tag"); + DatabaseIdentifier.checkNoReference(databaseName, "tag"); return SLASH.join( V1, prefix, @@ -463,10 +419,12 @@ public String tag(String databaseName, String objectName, String tagName) { } public String views(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "views"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), VIEWS); } public String viewDetails(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "viewDetails"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), VIEW_DETAILS); } @@ -475,6 +433,7 @@ public String views() { } public String view(String databaseName, String viewName) { + DatabaseIdentifier.checkNoReference(databaseName, "view"); return SLASH.join( V1, prefix, DATABASES, encodeString(databaseName), VIEWS, encodeString(viewName)); } @@ -484,6 +443,7 @@ public String renameView() { } public String functions(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "functions"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), FUNCTIONS); } @@ -492,10 +452,12 @@ public String functions() { } public String functionDetails(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "functionDetails"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), FUNCTION_DETAILS); } public String function(String databaseName, String functionName) { + DatabaseIdentifier.checkNoReference(databaseName, "function"); return SLASH.join( V1, prefix, diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/DatabaseIdentifierTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/DatabaseIdentifierTest.java new file mode 100644 index 000000000000..edba6dcb8d54 --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/DatabaseIdentifierTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the reserved database reference suffix grammar independently of HTTP encoding. */ +class DatabaseIdentifierTest { + + @Test + void testBranchAndTagSelectors() { + DatabaseIdentifier branch = DatabaseIdentifier.parse("training db$branch_experiment"); + assertThat(branch.getDatabaseName()).isEqualTo("training db"); + assertThat(branch.getReference()) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "experiment")); + + DatabaseIdentifier tag = DatabaseIdentifier.parse("training$literal$tag_train_v1"); + assertThat(tag.getDatabaseName()).isEqualTo("training$literal"); + assertThat(tag.getReference()) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train_v1")); + + assertThat(DatabaseIdentifier.parse("training$branch_123").getReference()) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "123")); + } + + @ParameterizedTest + @ValueSource( + strings = { + "training", + "training$literal", + "training$Branch_main", + "training%24branch_main" + }) + void testOrdinaryNamesRemainLiteral(String name) { + DatabaseIdentifier identifier = DatabaseIdentifier.parse(name); + assertThat(identifier.getDatabaseName()).isEqualTo(name); + assertThat(identifier.getReference()).isNull(); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource( + strings = { + " ", + "$branch_main", + "$tag_v1", + "training$branch_", + "training$tag_", + "training$branch_a/b", + "training$tag_..", + "training$branch_a$branch_b", + "training$branch_a$tag_b", + "training$tag_a$branch_b" + }) + void testMalformedSelectorsAreNotLiteralDatabaseNames(String name) { + assertThatThrownBy(() -> DatabaseIdentifier.parse(name)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index e830e40a1011..acf362fc95e9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -115,8 +115,6 @@ public class RESTCatalog implements Catalog { private final RESTApi api; private final CatalogContext context; - @Nullable private final String referenceDatabase; - @Nullable private final String referenceName; private final boolean dataTokenEnabled; protected final Map tableDefaultOptions; private final @Nullable LocalCacheManager cacheManager; @@ -126,30 +124,7 @@ public RESTCatalog(CatalogContext context) { } public RESTCatalog(CatalogContext context, boolean configRequired) { - this(context, configRequired, null, null); - } - - RESTCatalog( - CatalogContext context, - boolean configRequired, - @Nullable String referenceDatabase, - @Nullable String referenceName) { - this( - context, - new RESTApi(context.options(), configRequired), - referenceDatabase, - referenceName); - } - - private RESTCatalog( - CatalogContext context, - RESTApi api, - @Nullable String referenceDatabase, - @Nullable String referenceName) { - this.api = - referenceName == null ? api : api.withReference(referenceDatabase, referenceName); - this.referenceDatabase = referenceDatabase; - this.referenceName = referenceName; + this.api = new RESTApi(context.options(), configRequired); this.context = CatalogContext.create( api.options(), @@ -168,20 +143,7 @@ public Map options() { @Override public RESTCatalogLoader catalogLoader() { - return new RESTCatalogLoader(context, referenceDatabase, referenceName); - } - - /** - * Returns a separate catalog whose table operations use one database branch or immutable tag. - * - *

The binding is preserved by {@link #catalogLoader()}. Use ordinary logical table names; - * the server resolves their backing versions. No additional configuration request is made. - * Database and reference management are not versioned by this binding. - */ - @Experimental - public RESTCatalog withReference(String database, String reference) { - DatabaseReference.validateName(reference); - return new RESTCatalog(context, api, database, reference); + return new RESTCatalogLoader(context); } @Experimental @@ -262,6 +224,7 @@ public Database getDatabase(String name) throws DatabaseNotExistException { public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException { checkNotSystemDatabase(name); + DatabaseIdentifier.checkNoReference(name, "dropDatabase"); try { if (!cascade && !this.listTables(name).isEmpty()) { throw new DatabaseNotEmptyException(name); @@ -571,9 +534,10 @@ public boolean commitSnapshot( Snapshot snapshot, List statistics) throws TableNotExistException { - // CatalogSnapshotCommit supplies the physical storage branch. A database reference - // already selects the write target, so keep its logical table name on the wire. - if (referenceName != null && identifier.getBranchName() != null) { + // CatalogSnapshotCommit supplies the physical storage branch. The database suffix + // already selects the write target; keep the logical table name on the wire. + if (DatabaseIdentifier.parse(identifier.getDatabaseName()).getReference() != null + && identifier.getBranchName() != null) { identifier = new Identifier( identifier.getDatabaseName(), diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java index 05aa369eaa4d..efc5a0b46ca4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java @@ -21,28 +21,15 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogLoader; -import javax.annotation.Nullable; - /** Loader to create {@link RESTCatalog}. */ public class RESTCatalogLoader implements CatalogLoader { private static final long serialVersionUID = 1L; private final CatalogContext context; - @Nullable private final String referenceDatabase; - @Nullable private final String referenceName; public RESTCatalogLoader(CatalogContext context) { - this(context, null, null); - } - - RESTCatalogLoader( - CatalogContext context, - @Nullable String referenceDatabase, - @Nullable String referenceName) { this.context = context; - this.referenceDatabase = referenceDatabase; - this.referenceName = referenceName; } public CatalogContext context() { @@ -51,6 +38,6 @@ public CatalogContext context() { @Override public RESTCatalog load() { - return new RESTCatalog(context, false, referenceDatabase, referenceName); + return new RESTCatalog(context, false); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java index 9ab1269b467f..b71d8c93a681 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java @@ -30,6 +30,7 @@ import org.apache.paimon.reader.RecordReader; import org.apache.paimon.rest.exceptions.AlreadyExistsException; import org.apache.paimon.rest.requests.CommitTableRequest; +import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.responses.GetSchemaResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.schema.FileSystemSchemaManager; @@ -112,37 +113,38 @@ void tearDown() throws Exception { } @ParameterizedTest - @ValueSource(strings = {"experiment", "train_v1"}) + @ValueSource(strings = {"$branch_experiment", "$tag_train_v1"}) void testTableAndSerializedLoaderKeepReference(String reference) throws Exception { - RESTCatalog scoped = catalog.withReference(DATABASE, reference); - String scope = DATABASE_PATH + "/trees/" + reference; + String database = DATABASE + reference; + Identifier selected = Identifier.create(database, "features"); + String scope = DATABASE_PATH + reference.replace("$", "%24"); enqueue(200, "{\"tables\":[\"features\",\"labels\"]}"); - assertThat(scoped.listTables(DATABASE)).containsExactly("features", "labels"); + assertThat(catalog.listTables(database)).containsExactly("features", "labels"); takeRequest("GET", scope + "/tables"); - enqueue(200, tableResponse("physical-experiment")); - FileStoreTable table = (FileStoreTable) scoped.getTable(TABLE); - assertThat(table.catalogEnvironment().identifier()).isEqualTo(TABLE); + enqueue(200, tableResponse(database, "physical-experiment", 2)); + FileStoreTable table = (FileStoreTable) catalog.getTable(selected); + assertThat(table.catalogEnvironment().identifier()).isEqualTo(selected); assertThat(table.snapshotManager().branch()).isEqualTo("physical-experiment"); assertThat(table.schema().id()).isEqualTo(2); takeRequest("GET", scope + "/tables/features"); - // A task receives a serialized table. Its snapshot loader must still address the tree. + // A task receives a serialized table. Its identifier must retain the database suffix. FileStoreTable restored = InstantiationUtil.clone(table); enqueue(200, "{\"snapshot\":{\"snapshot\":" + SNAPSHOT_JSON + "}}"); assertThat(restored.snapshotManager().latestSnapshot().id()).isEqualTo(7); takeRequest("GET", scope + "/tables/features/snapshot"); - RESTCatalog loaded = InstantiationUtil.clone(scoped.catalogLoader()).load(); + RESTCatalog loaded = InstantiationUtil.clone(catalog.catalogLoader()).load(); enqueue( 200, RESTApi.toJson( new GetSchemaResponse( TableSchema.create(2, schema("physical-experiment"))))); - assertThat(loaded.loadSchema(TABLE, "LATEST").get().id()).isEqualTo(2); + assertThat(loaded.loadSchema(selected, "LATEST").get().id()).isEqualTo(2); takeRequest("GET", scope + "/tables/features/schemas/LATEST"); - // Binding another catalog must not change the original catalog's route or metadata. + // The same catalog also loads the ordinary database without reference state. enqueue(200, tableResponse("main")); FileStoreTable main = (FileStoreTable) catalog.getTable(TABLE); assertThat(main.snapshotManager().branch()).isEqualTo("main"); @@ -152,10 +154,10 @@ void testTableAndSerializedLoaderKeepReference(String reference) throws Exceptio @Test void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { - RESTCatalog scoped = catalog.withReference(DATABASE, "experiment"); - enqueue(200, tableResponse("physical-experiment")); - FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); - takeRequest("GET", DATABASE_PATH + "/trees/experiment/tables/features"); + Identifier selected = Identifier.create(DATABASE + "$branch_experiment", "features"); + enqueue(200, tableResponse(selected.getDatabaseName(), "physical-experiment", 2)); + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) catalog.getTable(selected)); + takeRequest("GET", DATABASE_PATH + "%24branch_experiment/tables/features"); Snapshot snapshot = Snapshot.fromJson(SNAPSHOT_JSON); enqueue(200, "{\"success\":true}"); @@ -170,7 +172,7 @@ void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { .isTrue(); } RecordedRequest request = - takeRequest("POST", DATABASE_PATH + "/trees/experiment/tables/features/commit"); + takeRequest("POST", DATABASE_PATH + "%24branch_experiment/tables/features/commit"); CommitTableRequest body = RESTApi.fromJson(request.getBody().readUtf8(), CommitTableRequest.class); assertThat(body.getTableId()).isEqualTo("table-id"); @@ -181,16 +183,18 @@ void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { @Test void testReadFollowUpsAndPaginationReuseProtocol() throws Exception { - RESTApi scoped = catalog.api().withReference(DATABASE, "train_v1"); - String scope = DATABASE_PATH + "/trees/train_v1"; + RESTApi api = catalog.api(); + String database = DATABASE + "$tag_train_v1"; + Identifier selected = Identifier.create(database, "features"); + String scope = DATABASE_PATH + "%24tag_train_v1"; String tablePath = scope + "/tables/features"; enqueue(200, "{\"tables\":[\"features\"],\"nextPageToken\":\"next\"}"); - assertThat(scoped.listTablesPaged(DATABASE, 1, null, "feat%", null).getNextPageToken()) + assertThat(api.listTablesPaged(database, 1, null, "feat%", null).getNextPageToken()) .isEqualTo("next"); RecordedRequest first = takeRequest("GET", scope + "/tables"); assertThat(first.getRequestUrl().queryParameter("tableNamePattern")).isEqualTo("feat%"); enqueue(200, "{\"tables\":[\"labels\"]}"); - assertThat(scoped.listTablesPaged(DATABASE, 1, "next", null, null).getElements()) + assertThat(api.listTablesPaged(database, 1, "next", null, null).getElements()) .containsExactly("labels"); assertThat( takeRequest("GET", scope + "/tables") @@ -198,55 +202,67 @@ void testReadFollowUpsAndPaginationReuseProtocol() throws Exception { .queryParameter("pageToken")) .isEqualTo("next"); - enqueue(200, "{\"tableDetails\":[" + tableResponse("physical-experiment") + "]}"); - assertThat(scoped.listTableDetails(DATABASE).get(0).getName()).isEqualTo("features"); + enqueue( + 200, + "{\"tableDetails\":[" + tableResponse(database, "physical-experiment", 2) + "]}"); + GetTableResponse details = api.listTableDetails(database).get(0); + assertThat(details.getName()).isEqualTo("features"); + assertThat(details.getDatabase()).isEqualTo(database); takeRequest("GET", scope + "/table-details"); enqueue(200, "{\"snapshot\":" + SNAPSHOT_JSON + "}"); - assertThat(scoped.loadSnapshot(TABLE, "LATEST").id()).isEqualTo(7); + assertThat(api.loadSnapshot(selected, "LATEST").id()).isEqualTo(7); takeRequest("GET", tablePath + "/snapshots/LATEST"); enqueue(200, "{\"snapshots\":[" + SNAPSHOT_JSON + "]}"); - assertThat(scoped.listSnapshotsPaged(TABLE, 10, null).getElements().get(0).id()) + assertThat(api.listSnapshotsPaged(selected, 10, null).getElements().get(0).id()) .isEqualTo(7); takeRequest("GET", tablePath + "/snapshots"); TableSchema schema = TableSchema.create(2, schema("physical-experiment")); enqueue(200, "{\"schemas\":[" + RESTApi.toJson(schema) + "]}"); - assertThat(scoped.listSchemasPaged(TABLE, 10, null).getElements()).containsExactly(schema); + assertThat(api.listSchemasPaged(selected, 10, null).getElements()).containsExactly(schema); takeRequest("GET", tablePath + "/schemas"); enqueue(200, "{\"token\":{\"key\":\"value\"},\"expiresAtMillis\":1234}"); - assertThat(scoped.loadTableToken(TABLE).getToken()).containsEntry("key", "value"); + assertThat(api.loadTableToken(selected).getToken()).containsEntry("key", "value"); takeRequest("GET", tablePath + "/token"); enqueue(200, "{\"filter\":[],\"columnMasking\":{}}"); - scoped.authTableQuery(TABLE, singletonList("id")); + api.authTableQuery(selected, singletonList("id")); assertThat(takeRequest("POST", tablePath + "/auth").getBody().readUtf8()) .isEqualTo("{\"select\":[\"id\"]}"); } @Test void testTableMutationsReuseRequestBodies() throws Exception { - RESTApi scoped = catalog.api().withReference(DATABASE, "experiment"); - for (RESTApi api : new RESTApi[] {catalog.api(), scoped}) { + Identifier selected = Identifier.create(DATABASE + "$branch_experiment", "features"); + RESTApi api = catalog.api(); + for (Identifier identifier : new Identifier[] {TABLE, selected}) { enqueue(200, "{}"); - api.createTable(TABLE, schema("main")); + api.createTable(identifier, schema("main")); enqueue(200, "{}"); - api.alterTable(TABLE, singletonList(SchemaChange.setOption("key", "value"))); + api.alterTable(identifier, singletonList(SchemaChange.setOption("key", "value"))); enqueue(200, "{}"); - api.dropTable(TABLE); + api.dropTable(identifier); } RecordedRequest[] original = { takeRequest("POST", DATABASE_PATH + "/tables"), takeRequest("POST", DATABASE_PATH + "/tables/features"), takeRequest("DELETE", DATABASE_PATH + "/tables/features") }; - String scope = DATABASE_PATH + "/trees/experiment"; + String scope = DATABASE_PATH + "%24branch_experiment"; RecordedRequest[] referenced = { takeRequest("POST", scope + "/tables"), takeRequest("POST", scope + "/tables/features"), takeRequest("DELETE", scope + "/tables/features") }; - for (int i = 0; i < original.length; i++) { + CreateTableRequest plain = + RESTApi.fromJson(original[0].getBody().readUtf8(), CreateTableRequest.class); + CreateTableRequest branch = + RESTApi.fromJson(referenced[0].getBody().readUtf8(), CreateTableRequest.class); + assertThat(plain.getIdentifier()).isEqualTo(TABLE); + assertThat(branch.getIdentifier()).isEqualTo(selected); + assertThat(branch.getSchema()).isEqualTo(plain.getSchema()); + for (int i = 1; i < original.length; i++) { assertThat(referenced[i].getBody().readUtf8()) .isEqualTo(original[i].getBody().readUtf8()); } @@ -254,50 +270,77 @@ void testTableMutationsReuseRequestBodies() throws Exception { @Test void testErrorsDoNotFallBackToDefaultBranch() throws Exception { - RESTCatalog scoped = catalog.withReference(DATABASE, "train_v1"); + Identifier selected = Identifier.create(DATABASE + "$tag_train_v1", "features"); enqueue(404, "{\"message\":\"reference missing\",\"code\":404}"); - assertThatThrownBy(() -> scoped.getTable(TABLE)) + assertThatThrownBy(() -> catalog.getTable(selected)) .isInstanceOf(Catalog.TableNotExistException.class); - takeRequest("GET", DATABASE_PATH + "/trees/train_v1/tables/features"); + takeRequest("GET", DATABASE_PATH + "%24tag_train_v1/tables/features"); enqueue(409, "{\"message\":\"tag is immutable\",\"code\":409}"); assertThatThrownBy( () -> - scoped.commitSnapshot( - TABLE, + catalog.commitSnapshot( + selected, "table-id", null, Snapshot.fromJson(SNAPSHOT_JSON), emptyList())) .isInstanceOf(AlreadyExistsException.class) .hasMessageContaining("tag is immutable"); - takeRequest("POST", DATABASE_PATH + "/trees/train_v1/tables/features/commit"); + takeRequest("POST", DATABASE_PATH + "%24tag_train_v1/tables/features/commit"); assertThat(server.getRequestCount()).isEqualTo(3); } @Test - void testUnsupportedSelectorsNeverSendAnUnscopedRequest() { - RESTApi scoped = catalog.api().withReference(DATABASE, "experiment"); - assertThatThrownBy(() -> catalog.withReference(DATABASE, null)) + void testUnsupportedDatabaseOperationsAndMixedSelectorsDoNotSendRequests() { + RESTApi api = catalog.api(); + String database = DATABASE + "$branch_experiment"; + Identifier selected = Identifier.create(database, "features"); + Identifier mixed = new Identifier(database, "features", "other"); + assertThatThrownBy(() -> api.getTable(mixed)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> api.createTable(mixed, schema("main"))) .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> scoped.withReference(DATABASE, "../main")) + assertThatThrownBy(() -> api.listTables(DATABASE + "$tag_")) .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> scoped.listTables("other")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining(DATABASE); - assertThatThrownBy(() -> scoped.getTable(new Identifier(DATABASE, "features", "other"))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("branch suffixes"); - assertThatThrownBy(() -> scoped.getTableById("table-id")) + assertThatThrownBy(() -> api.renameTable(selected, TABLE)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.renameTable(TABLE, selected)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.createBranch(selected, "nested", null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.createDatabase(database, java.util.Collections.emptyMap())) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy( + () -> + api.alterDatabase( + database, emptyList(), java.util.Collections.emptyMap())) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.dropDatabase(database)) .isInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> scoped.listTablesPagedGlobally(null, null, null, null)) + assertThatThrownBy(() -> catalog.dropDatabase(database, true, false)) .isInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> scoped.renameTable(TABLE, Identifier.create(DATABASE, "renamed"))) + assertThatThrownBy(() -> catalog.dropDatabase(database, true, true)) .isInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> scoped.createBranch(TABLE, "nested", null)) + assertThatThrownBy(() -> catalog.treeManagement().getReference(database, "main")) .isInstanceOf(UnsupportedOperationException.class); assertThat(server.getRequestCount()).isEqualTo(1); } + @ParameterizedTest + @ValueSource(strings = {"$branch_experiment", "$tag_train_v1"}) + void testDatabaseLookupPreservesVirtualName(String suffix) throws Exception { + String database = DATABASE + suffix; + enqueue( + 200, + "{\"name\":\"" + database + "\",\"location\":\"file:///training\",\"options\":{}}"); + assertThat(catalog.getDatabase(database).name()).isEqualTo(database); + takeRequest("GET", DATABASE_PATH + suffix.replace("$", "%24")); + enqueue(404, "{\"code\":404,\"message\":\"reference missing\"}"); + assertThatThrownBy(() -> catalog.getDatabase(database)) + .isInstanceOf(Catalog.DatabaseNotExistException.class); + takeRequest("GET", DATABASE_PATH + suffix.replace("$", "%24")); + assertThat(server.getRequestCount()).isEqualTo(3); + } + @Test void testBatchReadWriteAndPinnedTagWithRealDataFiles() throws Exception { // A small stateful fixture resolves references; production reference lifecycle is separate. @@ -315,13 +358,22 @@ void testBatchReadWriteAndPinnedTagWithRealDataFiles() throws Exception { public MockResponse dispatch(RecordedRequest request) { try { String route = request.getRequestUrl().encodedPath(); - String prefix = DATABASE_PATH + "/trees/"; + String prefix = "/v1/catalog%2Fid/databases/"; if (!route.startsWith(prefix)) { unexpected.add(route); return response(500, "{}"); } String[] parts = route.substring(prefix.length()).split("/"); - String reference = parts[0]; + DatabaseIdentifier database = + DatabaseIdentifier.parse(RESTUtil.decodeString(parts[0])); + if (!database.getDatabaseName().equals(DATABASE)) { + unexpected.add(route); + return response(500, "{}"); + } + String reference = + database.getReference() == null + ? "main" + : database.getReference().getName(); if (parts.length < 3 || !parts[1].equals("tables") || !parts[2].equals("features")) { @@ -331,7 +383,9 @@ public MockResponse dispatch(RecordedRequest request) { String branch = reference.equals("main") ? "main" : "physical-experiment"; if (request.getMethod().equals("GET") && parts.length == 3) { - return response(200, tableResponse(branch, 0)); + return response( + 200, + tableResponse(RESTUtil.decodeString(parts[0]), branch, 0)); } if (request.getMethod().equals("GET") && parts.length == 4 @@ -375,12 +429,12 @@ public MockResponse dispatch(RecordedRequest request) { } }); - RESTCatalog main = catalog.withReference(DATABASE, "main"); - RESTCatalog experiment = catalog.withReference(DATABASE, "experiment"); + Identifier main = Identifier.create(DATABASE + "$branch_main", "features"); + Identifier experiment = Identifier.create(DATABASE + "$branch_experiment", "features"); writeRows(main, 10); writeRows(experiment, 20); snapshots.put("train_v1", snapshots.get("experiment")); - RESTCatalog tag = catalog.withReference(DATABASE, "train_v1"); + Identifier tag = Identifier.create(DATABASE + "$tag_train_v1", "features"); assertThat(readRows(tag)).containsExactly(20); writeRows(experiment, 30); @@ -395,8 +449,8 @@ public MockResponse dispatch(RecordedRequest request) { assertThat(unexpected).isEmpty(); } - private void writeRows(RESTCatalog scoped, int value) throws Exception { - FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + private void writeRows(Identifier selected, int value) throws Exception { + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) catalog.getTable(selected)); BatchWriteBuilder builder = table.newBatchWriteBuilder(); try (BatchTableWrite write = builder.newWrite(); BatchTableCommit commit = builder.newCommit()) { @@ -405,8 +459,8 @@ private void writeRows(RESTCatalog scoped, int value) throws Exception { } } - private List readRows(RESTCatalog scoped) throws Exception { - FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + private List readRows(Identifier selected) throws Exception { + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) catalog.getTable(selected)); ReadBuilder builder = table.newReadBuilder(); List rows = new ArrayList<>(); try (RecordReader reader = @@ -420,19 +474,20 @@ private Schema schema(String branch) { return Schema.newBuilder() .column("id", DataTypes.INT()) .option("bucket", "-1") + .option("commit.max-retries", "0") .option(BRANCH.key(), branch) .build(); } private String tableResponse(String branch) throws Exception { - return tableResponse(branch, 2); + return tableResponse(DATABASE, branch, 2); } - private String tableResponse(String branch, long schemaId) throws Exception { + private String tableResponse(String database, String branch, long schemaId) throws Exception { return RESTApi.toJson( new GetTableResponse( "table-id", - DATABASE, + database, "features", tempDir.resolve("features").toUri().toString(), false, From 8f03d1e55293ac104b3bf940a5519bd8efd5ee46 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 20 Sep 2026 10:50:22 +0800 Subject: [PATCH 11/12] [rest] Align database branch and tag APIs with table operations --- .../docs/concepts/rest/database-versioning.md | 667 ++++++++---------- docs/docs/concepts/rest/rest-api.md | 6 +- docs/scripts/validate-rest-openapi.js | 31 +- docs/static/rest-catalog-open-api.yaml | 484 +++++++------ .../paimon/management/TreeManagement.java | 87 +-- .../paimon/rest/DatabaseReferenceType.java | 9 +- .../org/apache/paimon/rest/HttpClient.java | 12 +- .../org/apache/paimon/rest/MergeMode.java | 34 - .../java/org/apache/paimon/rest/RESTApi.java | 141 ++-- .../paimon/rest/RESTTreeManagement.java | 59 +- .../org/apache/paimon/rest/ResourcePaths.java | 32 +- .../apache/paimon/rest/TableMergeMode.java | 59 -- .../exceptions/MergeConflictException.java | 48 -- .../CreateDatabaseReferenceRequest.java | 71 -- ...est.java => CreateDatabaseTagRequest.java} | 45 +- .../requests/MergeDatabaseBranchRequest.java | 91 --- .../responses/DatabaseReferenceResponse.java | 50 -- .../responses/GetDatabaseTagResponse.java | 81 +++ .../ListDatabaseReferencesResponse.java | 72 -- .../rest/RESTApiDatabaseBranchTagTest.java | 306 ++++++++ .../rest/RESTApiDatabaseReferenceTest.java | 336 --------- .../RequestJacksonCompatibilityTest.java | 129 +--- .../org/apache/paimon/rest/RESTCatalog.java | 28 + .../paimon/rest/RESTCatalogReferenceTest.java | 87 ++- .../rest/RESTCatalogTreeManagementTest.java | 289 +++----- 25 files changed, 1344 insertions(+), 1910 deletions(-) delete mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java delete mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java delete mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java delete mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java rename paimon-api/src/main/java/org/apache/paimon/rest/requests/{DeleteDatabaseReferenceRequest.java => CreateDatabaseTagRequest.java} (53%) delete mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java delete mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/GetDatabaseTagResponse.java delete mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java create mode 100644 paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseBranchTagTest.java delete mode 100644 paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java diff --git a/docs/docs/concepts/rest/database-versioning.md b/docs/docs/concepts/rest/database-versioning.md index af694d295358..c5569cdbc4bc 100644 --- a/docs/docs/concepts/rest/database-versioning.md +++ b/docs/docs/concepts/rest/database-versioning.md @@ -23,182 +23,212 @@ under the License. # Database Branches and Tags -Database references group the versions of several tables under one branch or tag. A typical -training workflow starts an experiment from `main`, writes derived data on the experiment branch, -freezes the inputs under a tag, and merges accepted changes back into `main`. - -This page describes the experimental REST reference and table contracts and a proposed server MVP that -reuses Paimon's existing [table branches](../../maintenance/manage-branches) and -[table tags](../../maintenance/manage-tags). +Database branches and tags extend Paimon's existing table branches and tags to a group of tables. +The catalog server coordinates the table operations and records database membership and table +versions. Table data stays in the existing Paimon storage layout. :::info Implementation status -The Java reference-management client, database-name selector parser, and their wire contracts are -implemented. Ordinary table APIs carry the selector in the database name. Reference storage, table-level orchestration, and database merge execution must be -implemented by the catalog server. The server implementation below is a design, not a claim that -an existing service supports it. - -Table operations select a database reference through a `$branch_` or `$tag_` suffix on -the database name. The existing table paths and request/response structures are reused. Callers use -ordinary table names without remembering a tag's source branch. No reference header, catalog -option, or separately bound client is needed. +The Java client, database-name selector parser, and REST contracts are implemented. A catalog server +must implement database branch/tag storage, table orchestration, retention, and forward execution. +The stateful client fixture is not a production reference-management server. ::: -## Scope and terminology - -| Term | Meaning | -| --- | --- | -| Database branch | A writable reference to a database's table membership and table versions. | -| Database tag | An immutable reference to a captured membership and versions. Deleting a tag is allowed; moving it is not. | -| Table branch/tag | The existing Paimon storage and read/write mechanism used behind a database reference. | -| Table membership | The names and identities of the tables visible in a database reference. | -| Merge base | The historical state used to distinguish source changes from target changes. It includes previous merge relationships. | - -References belong to one database, not the whole catalog. Branches and tags share a name namespace -within that database. A reference contains `type` (`BRANCH` or `TAG`) and `name`. Names match -`[A-Za-z0-9][A-Za-z0-9._-]{0,127}`. Public references have no hash or reference ID. +## Relationship to table branches and tags + +Let `D = /v1/{prefix}/databases/{database}` and `T = D/tables/{table}`. Management operations use +physical database names, without a `$branch_` or `$tag_` suffix. + +| Operation | Table REST | Database REST | Request / response | +| --- | --- | --- | --- | +| List branches | `GET T/branches` | `GET D/branches` | Shared `ListBranchesResponse`. | +| Create branch | `POST T/branches` | `POST D/branches` | Shared `CreateBranchRequest`; no response body. | +| Drop branch | `DELETE T/branches/{branch}` | `DELETE D/branches/{branch}` | No request or response body. | +| Forward | `POST T/branches/{branch}/forward` | `POST D/branches/{branch}/forward` | Shared empty `ForwardBranchRequest`; no response body. | +| List tags | `GET T/tags` | `GET D/tags` | Shared `ListTagsResponse` and pagination parameters. | +| Create tag | `POST T/tags` | `POST D/tags` | `CreateDatabaseTagRequest`; no response body. | +| Get tag | `GET T/tags/{tag}` | `GET D/tags/{tag}` | `GetDatabaseTagResponse`. | +| Delete tag | `DELETE T/tags/{tag}` | `DELETE D/tags/{tag}` | No request or response body. | + +Successful mutations return HTTP `200` with no body, following the table API. Branch listing returns +names; tag listing is paged and accepts `maxResults`, `pageToken`, and `tagNamePrefix`. There is no +combined reference list or `/trees` resource. Database merge is deferred; there is no merge endpoint, +merge mode, or three-way conflict-resolution contract in this version. + +The database tag request retains table tag field names `tagName` and `timeRetained`, and uses +`fromBranch` instead of `snapshotId`. Tables have independent snapshot IDs, so one numeric snapshot +ID cannot identify a database version. Getting a database tag returns its name, source branch and +optional creation/retention metadata. Read its table versions through the existing table APIs. + +Branch names and tag names belong to separate namespaces, as with table branches and tags. A branch +and a tag may have the same name. Both names match `[A-Za-z0-9][A-Za-z0-9._-]{0,127}`. Database tag +names are database-wide: the server stores each tag's source and backing table versions, so callers +can read `training$tag_train_v1` without remembering the source branch. Native table tags remain +branch-local; the server maps the database tag to the corresponding table pins. ### Initial server MVP Start with managed native Paimon tables and a fixed set of logical table names. Create and populate -those tables on `main` before starting the experiment. Use batch writers and pause writes during -branch creation, tag creation, and merge. Resume with freshly loaded tables after publication. +those tables on `main`. Use batch writers and pause writes during branch creation, tag creation and +forward. Invalidate cached tables and load them again after publication. -This scope can demonstrate isolated table writes, multi-table training inputs frozen under a tag, -and table-version merge. It does not require a public multi-table transaction API, public hashes, -row-level conflict resolution, or concurrent streaming publication. +Branch-local table creation, deletion and rename require versioned namespace storage and can be +deferred. Format Tables, Object Tables, external tables, views, functions and catalog permissions +are outside this initial versioned-table scope. Unsupported scoped operations return `501`. -Branch-local table creation, deletion, and rename need reference-aware namespace handling. The -merge contract covers table creation and deletion, but the first fixed-table server can defer those -operations until reference-aware namespace storage is implemented. Their scoped REST routes already -reuse the ordinary table request and response schemas; rename is deferred. Format Tables, -Object Tables, external tables, views, functions, and catalog permissions are outside this initial -versioned-table scope. +## Branch management -## Reference management API +### Create a branch without data -All paths use the configured catalog `prefix`. For brevity, the following table uses -`B = /v1/{prefix}/databases/{database}`. Encode each path segment; names in JSON remain unencoded. +```http +POST /v1/catalog/databases/training/branches +Content-Type: application/json -| Method and path | Request | Result | -| --- | --- | --- | -| `GET B/trees` | Optional `type`, `maxResults`, and `pageToken` query parameters. | One page of references. | -| `GET B/trees/{name}` | No body. | One reference. | -| `POST B/trees` | New name, type, and an existing source reference. | The created reference. | -| `POST B/trees/{name}/merge` | Source reference and optional merge modes. The path names the target branch. | The target reference after success. | -| `DELETE B/trees/{name}` | Optional expected `type` in the body. | The deleted reference. | +{"branch":"experiment"} +``` -Database merge includes fast-forward when applicable. There is no database-level `/forward` -endpoint. The existing table-level forward API is separate. +Like table `createBranch` without `fromTag`, this copies main's table membership, schemas and +properties, with no table snapshots. It does not copy main's current data. The server must preserve +empty tables explicitly, including their schema-only state. -### Create a branch or tag +### Create a branch with data -Create an experiment branch from `main`: +First capture a database tag, then create a branch from it: ```http -POST /v1/catalog/databases/training/trees +POST /v1/catalog/databases/training/tags Content-Type: application/json -{ - "name": "experiment", - "type": "BRANCH", - "source": {"type": "BRANCH", "name": "main"} -} +{"tagName":"baseline"} ``` -Freeze the experiment under a database tag: +```http +POST /v1/catalog/databases/training/branches +Content-Type: application/json -```json -{ - "name": "train_v1", - "type": "TAG", - "source": {"type": "BRANCH", "name": "experiment"} -} +{"branch":"experiment","fromTag":"baseline"} ``` -Both requests use the same path. The source must exist in the same database. A source can be a -branch or an immutable tag; the new reference can also be either type. Successful singular -operations return `DatabaseReferenceResponse`: +`fromTag` names a database tag in the same database. The server restores its table membership, +schemas, properties and snapshots. A missing tag returns `404`; an existing branch returns `409`. +There is no generic `source: {type, name}` object. -```json -{"reference": {"type": "TAG", "name": "train_v1"}} +### List and drop branches + +```http +GET /v1/catalog/databases/training/branches ``` -### Inspect and list +```json +{"branches":["main","experiment"]} +``` ```http -GET /v1/catalog/databases/training/trees/train_v1 -GET /v1/catalog/databases/training/trees?type=tag&maxResults=100 +DELETE /v1/catalog/databases/training/branches/experiment ``` -The list filter uses lowercase `branch` or `tag`; JSON reference types use uppercase enum names. -Omitting `type` includes both. A missing or zero `maxResults` uses the server default. Pass the -returned `nextPageToken` unchanged to request the next page; a missing token ends iteration. +The delete request has no body. The server protects `main` and rejects its deletion with `400`. +Deleting a branch does not authorize removing files or pins still required by a database tag or +another branch. -```json -{ - "references": [{"type": "TAG", "name": "train_v1"}], - "nextPageToken": "next-page" -} +### Forward a branch to main + +```http +POST /v1/catalog/databases/training/branches/experiment/forward +Content-Type: application/json + +{} ``` -Getting a reference returns its name and type, not the table membership, source branch, or table -version map. Pagination discovers references; it does not create a frozen view across pages. +The path names the **source branch**, following Table REST. The database operation publishes it to +`main`. A tag is not a forward source. To publish a frozen tag, first create a temporary branch from +that tag, then forward that branch. -### Merge +Forward extends table fast-forward to the database's tables. It publishes source versions on main +and can replace target changes; it does not preserve independently changed target tables using +three-way conflict resolution. The first fixed-table server requires matching membership and a +snapshot for each source table, as native table fast-forward requires a populated source. An empty +source table is a `400`; namespace changes the server cannot handle are a `501`. The server validates +all tables before starting publication. Source `main` is invalid. + +Pause both source and main writers while forwarding. Preserve retained tags, keep the two branches +independently writable afterwards, and invalidate/reload main tables before resuming work. No public +multi-table transaction or atomic read view is required for this MVP. A successful response means +all planned table operations finished. Interrupted execution needs recoverable server bookkeeping. + +## Tag management + +### Freeze a branch ```http -POST /v1/catalog/databases/training/trees/main/merge +POST /v1/catalog/databases/training/tags Content-Type: application/json +{"tagName":"train_v1","fromBranch":"experiment","timeRetained":"7d"} +``` + +Omitting `fromBranch`, or setting it to null, selects `main`. `timeRetained` uses the table tag +retention-duration syntax and is optional. Tags freeze membership, schemas, properties, snapshots, +and the empty state of tables without snapshots. They cannot be moved or updated. Expiring a tag +must respect versions still used by other references; native table pins cannot expire independently +while the database tag is valid. + +### Inspect and list tags + +```http +GET /v1/catalog/databases/training/tags/train_v1 +``` + +```json { - "source": {"type": "BRANCH", "name": "experiment"}, - "defaultMergeMode": "NORMAL", - "tableMergeModes": [ - {"table": "features", "mergeMode": "FORCE"}, - {"table": "scratch", "mergeMode": "DROP"} - ] + "tagName":"train_v1", + "fromBranch":"experiment", + "tagCreateTime":1720000000000, + "tagTimeRetained":"7d" } ``` -Only `source` is required. Omitting modes gives `NORMAL` for every table. Per-table modes override -the default, and an omitted or empty override list applies the default everywhere. Table names -are exact names within this database. Duplicate table overrides are a bad request; an override -for a table without source-side changes has no effect. +`tagCreateTime` is milliseconds since the Unix epoch. `tagCreateTime` and `tagTimeRetained` are +optional. `fromBranch` records creation provenance; the source can later be deleted without making +the tag unreadable. Table snapshots are resolved using the tag-suffixed database name. -The target is always a branch. A source tag is allowed and remains immutable. The response remains -`DatabaseReferenceResponse`; it does not include a commit hash or a detailed merge report. +```http +GET /v1/catalog/databases/training/tags?maxResults=100&tagNamePrefix=train_ +``` -### Delete +```json +{"tags":["train_v1"],"nextPageToken":"next-page"} +``` -```http -DELETE /v1/catalog/databases/training/trees/train_v1 -Content-Type: application/json +Pass the returned token unchanged to get the next page. A missing token ends iteration. An absent +or zero `maxResults` uses the server default. Pagination does not create a frozen cross-page view. + +### Delete a tag -{"type": "TAG"} +```http +DELETE /v1/catalog/databases/training/tags/train_v1 ``` -The optional type checks the reference before deletion. Omitting the body or sending `{}` omits -that check. An absent reference is an error. The MVP server should protect the default `main` -branch. Logical deletion does not authorize deleting table versions still needed by another -reference. +The request has no body and does not need an expected reference type: the resource path identifies +a tag. A missing tag returns `404`. Logical deletion and physical cleanup can be separate operations. -### Errors +## Errors + +Use the existing `ErrorResponse` and table branch/tag resource types: | Situation | HTTP behavior | | --- | --- | -| Missing database or reference | `404`; merge distinguishes the missing source or target in its error details. | -| Creating an existing reference | `409`. | -| Merge target is a tag, no merge base is available, or unresolved table conflicts remain | `409`; the target stays unchanged. | -| Invalid merge request, such as duplicate per-table modes | `400`. | -| Deleting a protected default branch or supplying the wrong expected type | `409`. | -| Server does not implement an operation | No client fallback; the server error is propagated. | - -Errors use `ErrorResponse`. The Java merge client converts `409` to `MergeConflictException` and -preserves the resource type/name, message, request ID, and cause. Resource creation still uses -`AlreadyExistsException`. See the [OpenAPI specification](/rest-catalog-open-api.yaml) for the -individual operations and their documented responses. +| Missing database, branch or tag | `404`, with `DATABASE`, `BRANCH` or `TAG` resource details. | +| Creating an existing branch or tag | `409`, with the corresponding resource type and name. | +| Invalid name, protected main mutation, or invalid forward source | `400`. | +| Missing table or snapshot during table orchestration | `404`, identifying the affected resource. | +| Authorization failure | `403`. | +| Unsupported operation or scoped DDL | `501`. | + +There is no merge-specific exception translation. Creation conflicts use the same +`AlreadyExistsException` as table branch/tag creation. Errors never cause fallback to a different +branch or to a physical database without its selector. ## Reference-scoped table API @@ -206,7 +236,7 @@ A database name can include exactly one reference selector: | Database name | Meaning | | --- | --- | -| `training` | The ordinary physical database, with its existing main-table behavior. | +| `training` | The main database branch, also addressed as `training$branch_main`. | | `training$branch_experiment` | The writable database branch `experiment`. | | `training$branch_main` | Explicit selection of the database branch `main`. | | `training$tag_train_v1` | The immutable database tag `train_v1`. | @@ -220,8 +250,8 @@ GET /v1/catalog/databases/training%24tag_train_v1/tables/features POST /v1/catalog/databases/training%24branch_experiment/tables/features/commit ``` -There are no additional table routes below `/trees/{reference}`. `/trees` remains the reference -management resource and always takes the physical database name, such as `training`. +Branch and tag management use `/branches` and `/tags` on the physical database name. +Table access uses the existing table resource paths with the selected database name. Let `D = /v1/{prefix}/databases/{database}` below, where `database` may carry a reference suffix. These are the existing operations and request/response structures: @@ -255,11 +285,11 @@ with the resolved table. `GET database` must resolve a suffixed name, because SQL engines can check namespace existence before accessing a table. The response represents the virtual database and retains its full name. -Database listing returns physical database names only; use `/trees` to discover branches and tags. +Database listing returns physical database names only; use `/branches` and `/tags` to discover their names. CREATE, DROP and ALTER DATABASE do not accept reference suffixes. In particular, dropping a -virtual database must never drop its physical database. Create, delete and merge references through -`/databases/training/trees` instead. This does not prevent ordinary create/alter/drop **table** +virtual database must never drop its physical database. Create, delete and forward branches or manage tags through +`/databases/training/branches` and `/databases/training/tags` instead. This does not prevent ordinary create/alter/drop **table** operations from modifying membership or metadata in a writable branch. The markers `$branch_` and `$tag_` are case-sensitive reserved syntax. The base database must be @@ -275,20 +305,30 @@ internally; RESTCatalog removes that internal table suffix while preserving the ### Branch and tag behavior +For a version-enabled database, both main aliases must resolve through the same mapping, including +after forward replaces its backing table branches. Writes through either alias update that mapping. + A branch resolves to its current membership and table versions. A tag resolves to the membership, schemas, options and snapshots captured when it was created, even after its source branch advances. Tag snapshot listing exposes only the pinned snapshot. `LATEST` and `EARLIEST` select that snapshot; other version selectors must resolve to it or return `404`. Schema reads may access the captured schema and older schemas retained for reading the captured data, but never later source schemas. +Freezing REST responses alone is insufficient: native readers and system tables can read metadata +directly from storage. A server can return a dedicated frozen metadata branch, with read-only +credentials and no later source snapshots/schemas, through the existing path and branch options. +Another implementation must enforce the same boundary in native reads, including time travel and +schema/system-table access. A default scan option that callers can override does not enforce it. + An empty captured table is still returned by `GET table`; snapshot lookup returns `404` with `resourceType: SNAPSHOT`. -The server rejects content changes through a tag with `409`. Read authorization remains allowed +The server rejects content changes through a tag with `403`. Read authorization remains allowed through `POST .../auth`; HTTP method alone does not determine whether an operation is a write. Tag credentials must permit reading without allowing mutation of retained metadata or data. -Missing databases, references and tables return `404`. A selector whose type does not match the -reference, such as `$branch_train_v1` for a tag, returns `409`. Malformed selectors return `400`. +Missing databases, references and tables return `404`. The selector chooses the branch or tag +namespace: `$branch_train_v1` returns `404` if only a tag with that name exists. Malformed selectors +return `400`. Reserve `409` for already-existing resources, following the table APIs. Unsupported operations on references return `501`. None of these errors permits retrying the request against the physical database without its suffix. @@ -315,8 +355,9 @@ restCatalog.getDatabase("training$tag_train_v1"); `RESTApi` uses these same identifiers with its existing table methods. `Identifier` already retains the full database name through serialization and in table loaders; no extra reference fields are stored in RESTCatalog or RESTCatalogLoader. Subsequent snapshot reads, schema changes, commits, -auth and token requests carry the same database name. Caches keyed by full table identifiers -naturally distinguish the physical database, branches and tags. +auth and token requests carry the same database name. Caches keyed by full table identifiers distinguish branches and tags. The two main aliases +(`training` and `training$branch_main`) refer to the same state; mutations and forward must invalidate +both aliases. A repeated cached getTable call is not a reload. SQL clients can pass the selector as a quoted database name, using their ordinary identifier quoting rules. For example: @@ -330,8 +371,9 @@ A REST server implementing virtual database lookup and table resolution is requi new engine catalog option or reference-switch operation. Rename, register, replace, rollback, partition/consumer endpoints, nested table branch/tag -management, views, functions and table policies do not yet accept database reference suffixes in -the Java client. Global table listing and lookup by table ID retain their physical-catalog meaning; +management, view writes, functions and table policies do not yet accept database reference suffixes +in the Java client. RESTCatalog validates the virtual database for read-only view probes, then +returns empty lists or a missing view so engine table discovery and DROP TABLE can proceed. Global table listing and lookup by table ID retain their physical-catalog meaning; they have no database selector. Extending those operations to discover or address references is additional work. Catalog-level permissions and reference management continue to use physical names. @@ -347,274 +389,123 @@ identifiers so follow-up calls stay on the same reference. Parsing the suffix does not replace reference management: listing still needs the selected membership, tags need frozen metadata, and commits must update the selected branch's recorded table state. The additional request cost is a reference/table mapping lookup, which can be cached; -this addressing scheme does not require proxying or copying table data. The storage and merge work +this addressing scheme does not require proxying or copying table data. The storage and forward work remains the server orchestration described below. ## Java management usage -Obtain tree management from an already configured `RESTCatalog`. It shares that catalog's -prefix, authentication, and HTTP configuration: +`RESTCatalog.treeManagement()` shares the catalog's prefix, authentication and HTTP configuration. +Its operations follow the existing table branch/tag method names: ```java import org.apache.paimon.PagedList; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.management.TreeManagement; -import org.apache.paimon.rest.DatabaseReference; -import org.apache.paimon.rest.DatabaseReferenceType; -import org.apache.paimon.rest.MergeMode; -import org.apache.paimon.rest.TableMergeMode; - -import java.util.Collections; +import org.apache.paimon.rest.responses.GetDatabaseTagResponse; TreeManagement trees = restCatalog.treeManagement(); -DatabaseReference main = new DatabaseReference(DatabaseReferenceType.BRANCH, "main"); -DatabaseReference experiment = trees.createReference( - "training", "experiment", DatabaseReferenceType.BRANCH, main); - -// Run batch writes on the corresponding table branches before freezing this tag. -DatabaseReference trainingTag = trees.createReference( - "training", "train_v1", DatabaseReferenceType.TAG, experiment); - -PagedList page = trees.listReferencesPaged( - "training", DatabaseReferenceType.TAG, 100, null); +trees.createTag("training", "baseline", null, null); +trees.createBranch("training", "experiment", "baseline"); -// Default three-way merge, failing on conflicting table versions. -trees.mergeBranch("training", "main", trainingTag); -``` - -When resolving a conflict, use the following call instead of the default merge to accept the -source version of `features`. Changing modes after a successful merge does not reapply that source: - -```java -trees.mergeBranch( - "training", "main", trainingTag, MergeMode.NORMAL, - Collections.singletonList(new TableMergeMode("features", MergeMode.FORCE))); -``` - -`RESTApi` exposes equivalent methods: `listDatabaseReferencesPaged`, `getDatabaseReference`, -`createDatabaseReference`, `mergeDatabaseBranch`, and `deleteDatabaseReference`. Listing is paged; -there is no non-paged database-reference helper. +// Use ordinary batch writers on these tables. +restCatalog.getTable(Identifier.create("training$branch_experiment", "features")); -These are management calls. Creating a database branch does not switch the catalog's ordinary -table operations to that branch. +// Stop experiment writes while capturing its training inputs. +trees.createTag("training", "train_v1", "experiment", "7d"); +GetDatabaseTagResponse tag = trees.getTag("training", "train_v1"); +PagedList page = trees.listTagsPaged("training", 100, null, "train_"); +restCatalog.getTable(Identifier.create("training$tag_train_v1", "features")); -## Reusing table branches and tags on the server +// Stop main and experiment writers before publishing the experiment's current state. +trees.fastForward("training", "experiment"); +// Invalidate cached main tables and reload them before resuming writers. -The server coordinates existing table-level operations and keeps database metadata around them. -An illustrative mapping is: - -```text -training / main [BRANCH] - features -> table identity A, table branch main - labels -> table identity B, table branch main - -training / experiment [BRANCH] - features -> table identity A, table branch experiment - labels -> table identity B, table branch experiment - -training / train_v1 [TAG] - features -> table identity A, experiment branch, pinned table tag train_v1 - labels -> table identity B, experiment branch, pinned table tag train_v1 +trees.dropBranch("training", "experiment"); +trees.deleteTag("training", "train_v1"); ``` -Names such as `experiment` can also name the corresponding backing table -branches, and `train_v1` can name each table tag in its source table branch. These names are owned -by the service. Reject collisions with unrelated existing table references; do not adopt them just -because the names match. Additional internal baseline tags can use private, service-generated names. - -The public database-reference name rules and native table-branch rules are not identical. For -example, the database protocol permits a purely numeric name, while native table branch creation -rejects it. A server supporting the full name contract needs an alias mapping to valid physical -branch names and must resolve the scoped logical table address through that mapping. It must not -silently narrow the database API's name rules. The examples use names valid in both layers. - -### Minimal metadata - -The server needs: - -- A database reference record: name, type, current membership, and internal ancestry/merge history. -- A mapping from logical table name to stable table identity and backing table branch or tag. -- Captured table versions at branch points, tag creation, and merges, including the schema and - snapshot state needed for comparisons and reads. - -A captured version can reuse a snapshot UUID, a pinned schema, and relevant table properties. -Empty tables need an explicit no-snapshot state. Numeric snapshot/schema IDs alone are not enough -to compare independently written branches. Table identity distinguishes a dropped-and-recreated -table from its predecessor. Copying metadata to a new physical branch does not itself constitute a -logical table change. - -These records can live in the catalog backend. Their internal identities are not public hashes and -need not introduce a new versioned storage engine. The server must update its recorded table state -when a managed branch accepts a table commit or schema change; names alone cannot support merge. -Use the server's table commit and schema operations for those writes. Uncoordinated filesystem -writes or direct edits of service-owned table references would bypass this bookkeeping. - -### Bootstrap main - -A version-enabled new database starts with an empty `main` branch. Otherwise every create-reference -request would require a source that does not yet exist. For an existing database, the server can -initialize `main` from its current tables while writers are stopped. Automatic online conversion of -an actively written database is outside the first MVP. - -This is a server lifecycle rule, not an additional REST endpoint. Reference creation always keeps -its existing `source` field. - -### Create a database branch - -1. Capture the source membership and each selected table version while writes are paused. -2. For a populated table, pin the selected source snapshot with a service-owned table tag and create - the destination table branch from that tag. -3. For an empty table, create a schema-only table branch. Preserve the selected schema and properties. -4. Record the common baseline and publish the database branch after all table branches are ready. - -The existing `FileSystemBranchManager.createBranch(name)` creates an empty branch by copying -schemas. It does not clone the source data. `createBranch(name, tagName)` copies the selected -snapshot and its schemas. If the captured current schema is newer than the snapshot's schema, -the server must also preserve that schema-only change; snapshot cloning alone is insufficient. - -No data files need to be copied merely to create a branch. In the single-process MVP, table-level -setup can run sequentially; do not expose an incomplete database reference as successfully created. -Failures can leave private work to clean up or resume. - -### Create and retain a database tag - -Capture the table membership and pin a table tag for each populated table. Persist the source table -branch with each pin: native Paimon tags belong to a table branch, not a database-wide directory. -An empty table has no snapshot to tag, so its frozen entry must retain the schema and empty state; -the server cannot blindly call `createTag` on every table. - -A database tag never follows subsequent writes to its source. For an empty tagged table, reads must -remain empty even if the source later receives its first snapshot. The source's later schema -changes must also leave the tagged schema unchanged. A demonstration server that has not implemented -empty-table reads must restrict tagging to populated tables explicitly. - -Service-owned pins must not expire through ordinary automatic tag-retention settings or be replaced -through user table-tag operations. Table branch deletion removes its metadata directory, including -the tags in that directory. Keep a backing branch while a database tag or merge baseline still needs -it, or relocate the retained metadata before deleting it. - -The first MVP can defer physical deletion and cleanup. Removing a logical database reference need -not immediately drop its underlying table branches or files. Enable physical cleanup only when it -accounts for all retained database references and merge baselines. - -## Merge semantics and execution - -Merge operates on complete table versions, including schema, properties, and snapshot state. It -also defines how table presence or absence is combined once branch-aware DDL is available. - -Let `B`, `S`, and `T` be a table's base, source, and target version, with absence represented as a -state. First determine whether the source changed relative to `B`: - -| Condition or mode | Result | -| --- | --- | -| `S = B` | Keep `T`, including target-only changes. | -| Source changed, mode `DROP` | Keep `T`, even when there would be no conflict. | -| Source changed, mode `FORCE` | Use `S`, including source-side deletion. | -| Source changed, mode `NORMAL`, and `T = B` | Use `S`. | -| Source changed, mode `NORMAL`, and `S = T` | Accept the identical result. | -| Source changed, mode `NORMAL`, and both sides changed differently | Fail the merge with `409`. | - -Different tables can therefore change independently and merge successfully. Different versions of -the same table conflict under `NORMAL`, even when an application might know how to combine their -rows. `FORCE` selects a complete source version; `DROP` skips all source changes to the selected -table, not just conflicting changes. - -### Publication - -1. Resolve the source and target and find their merge base, including earlier merges. -2. Compare table versions and compute the complete result using the selected modes. -3. If any unresolved conflict exists, return `409` before changing target tables. -4. Prepare the selected target table versions with table-level snapshot/schema mechanisms and publish - the database result. Record the source as merged; never modify the source reference. - -When the target is an ancestor of the source, fast-forward is possible only if the chosen modes -produce exactly the source state. Already-merged sources and identical reference states succeed -without changing the target. Divergent histories use three-way merge. - -The existing table `mergeBranch` implementation merges append-only data-file changes. It is not an -implementation of this whole-table-version algorithm. Existing table `fastForward` also has its own -replacement semantics and can remove target metadata and tags. A server needs an adapter that -checks the database result first and preserves retained references; looping over either operation -without that adapter is insufficient. - -Preparing fresh backing branches and publishing a new mapping is one possible server implementation. -The server can update the reference mapping to those prepared versions while retaining any -physical branches needed by tags or merge baselines. The client-visible table address must continue -to resolve correctly. Source and target must remain independently writable: pointing both at the -same mutable table branch would make future source writes modify the target as well. - -### Repeated merge - -After a successful merge, the server records the integrated source version even if `DROP` preserved -all target table contents. Repeating a merge of that same source state must not bring skipped -changes back. A later source write can participate in a subsequent merge using the updated history. - -Do not identify prior merges only by the source branch name; that branch can continue to advance. -Internal ancestry is required even though the public API has no hash. The first MVP can serialize -these operations and pause writers instead of introducing public concurrency tokens or multi-table -transactions. Reads during a multi-table publication need not provide an atomic database view in -this restricted MVP. Partial backend execution still needs a recoverable server operation record; -an HTTP success must mean that the planned result is installed. - -## Exercise the fixed-table MVP - -The following workflow requires a server that implements the orchestration above. The client tests -exercise suffix-based HTTP addressing and table loaders; they do not implement database reference -storage or the database merge algorithm. - -1. Create database `training` and two populated managed tables, `features` and `labels`, on `main`. - Stop writes and create database branch `experiment` from `main` using tree management. -2. List and load `features` and `labels` from database `training$branch_experiment`, then write - experiment data with the usual batch write API. Their metadata reads and commits use the - existing table paths with the complete suffixed database name. -3. Stop experiment writes and create database tag `train_v1` from `experiment`. Use the same - catalog to access database `training$tag_train_v1`. Load the same logical table names for training; - the service resolves the pinned table versions without a source-branch hint. -4. Advance the experiment tables, then reload and read them through the tag-suffixed database name. - The tagged data and schemas must remain unchanged. Verify that writes through the tag are rejected. -5. With main and experiment writers stopped, merge `train_v1` into `main`. This publishes the - evaluated source version. Merging the live `experiment` branch would instead include its newer - state. If both sides changed a table, choose a per-table merge mode when appropriate. -6. Reload main tables and verify the published state. Resume writes separately on `main` and - `experiment` and verify that neither changes the other. Merge the same source state again to - check no-op behavior. Delete unused database references through tree management. - -## Beyond the fixed-table MVP - -The database name suffix now identifies the selected database view for database lookup, table listing, -reads, commits and the ordinary create/alter/drop endpoints. A complete server namespace still -needs branch-local membership changes, stable identities across rename, and new identities for -drop-and-recreate. The fixed-table server may return `501` for unsupported scoped DDL. - -Global table IDs, global listings, rename and the other deferred endpoints need explicit reference -semantics before they can be extended. Engine integrations must preserve the full database name in -identifiers and perform database existence checks through the catalog. These additions do not -require callers to construct per-table branch names. - -## Validation and implementation sequence - -The reference tests validate HTTP paths, request bodies, authentication/configuration, pagination, -JSON compatibility, exception propagation and suffix preservation through serialized tables and -catalog loaders. Tests cover virtual database lookup, mutation guards and malformed or mixed -selectors. The OpenAPI validator checks that reference access uses the ordinary table paths. A stateful test fixture also uses real Paimon data files -to exercise batch writes on separate branches, frozen tag reads after source writes, and tag write -rejection. This validates client integration with a resolving server; production reference -lifecycle, snapshot retention and database merge still require server integration tests. - -Implement and verify in this order: - -1. **Reference records and bootstrap:** create `main`, list/get/create/delete references, and protect - managed table-reference names. -2. **Table orchestration:** clone populated and empty tables correctly; record baselines; route - scoped logical table names through existing Paimon readers and writers. -3. **Frozen training inputs:** pin table tags and schemas, validate repeated reads after source - writes, and retain dependencies after logical reference deletion. Add empty-table coverage when - that case is enabled. -4. **Merge:** verify automatic fast-forward, independent changes to different tables, same-table - conflicts leaving the target unchanged, all three modes, repeated merge including `DROP`, and - continued independent writes after merge. -5. **Complete database views:** implement branch-local DDL storage and verify membership changes, - then extend the scoped protocol to the deferred operations as needed. - -A useful acceptance test uses real Paimon snapshots for two tables and exercises the workflow above -against a stateful server. Passing that test establishes the fixed-table MVP; a full database-view -MVP additionally requires the final namespace step. +`RESTApi` exposes `listDatabaseBranches`, `createDatabaseBranch`, `dropDatabaseBranch`, +`fastForwardDatabase`, `createDatabaseTag`, `getDatabaseTag`, `listDatabaseTagsPaged`, and +`deleteDatabaseTag`. All management methods take the physical database name. Table access keeps +using the suffix in the ordinary `Identifier`. + +## Server implementation using table capabilities + +### Metadata and bootstrap + +Keep a database branch record with membership and logical-table-to-backing-branch mappings. Keep a +separate database tag record with its source branch, retention metadata, frozen membership and each +table's identity, schema, properties and snapshot (or explicit no-snapshot state). + +A version-enabled database starts with `main`. An existing database can be initialized while writers +are stopped. Normal database access and `$branch_main` must use the same record. Accepted table +commits and schema changes update its table state. Direct filesystem writes and unmanaged edits of +service-owned table references bypass this bookkeeping and are outside the MVP. + +Backing reference names are owned by the service. Reject collisions with unrelated table branches +or tags. Database and native table naming rules differ: a purely numeric database branch name needs +a valid physical alias because native table branches reject it. Name matching alone is not a safe +way to locate a backing version. + +### Create branches + +Without `fromTag`, call the schema-only table branch operation for each table on main. With +`fromTag`, resolve each frozen table entry and create its backing branch from the corresponding +native table tag. Paimon's `createBranch(name, tagName)` copies the selected snapshot and its schemas; +`createBranch(name)` copies schemas without data. + +Preserve captured schema-only changes newer than the snapshot schema. Empty captured tables need +schema-only branches. Publish the database branch only after all table entries are ready. No data +files need to be copied just to create a branch. Failed setup can leave private work to resume or +clean up. + +### Capture tags and retain data + +Pause source writers, capture table membership and versions, and pin each populated table snapshot. +Native table tags belong to physical table branches, so store that backing branch with each pin. +Empty tables need frozen schema and no-snapshot metadata because native createTag needs a snapshot. +A minimal server can explicitly reject empty-table tagging until that behavior is implemented. + +A database tag must remain readable after source writes, schema evolution, source deletion, and +forward. Provide frozen metadata to native table readers, including system tables and explicit +snapshot reads, as described above. Native automatic retention must not delete service-owned pins +before the database tag expires or is deleted. + +Physical cleanup must account for all database references. Native dropBranch deletes its metadata +directory, including its tags. Defer physical cleanup in the first MVP, or relocate retained metadata +before deleting a backing branch. Keep the data files referenced by every retained snapshot. + +### Execute forward + +1. Resolve the source branch and main. Validate the whole fixed-table membership and source snapshots. +2. Resolve each source table version and prepare the corresponding main table state using native + table snapshot/schema mechanisms. +3. Preserve database tags before applying native fastForward: that operation can remove target + metadata and tags. An adapter can instead prepare fresh backing branches and publish their mapping. +4. Publish all target entries, keeping source and main independently writable. Do not point both at + one mutable table branch. Both main aliases resolve the published mapping. +5. Return success after the planned work completes. Keep an operation record for recovery from a + partial backend failure, and invalidate/reload client caches before writers resume. + +This uses table forward semantics, including replacement of target state. It needs no public hash, +reference ID, multi-table transaction endpoint, merge base or merge-mode API. More advanced merge +semantics will be designed separately. + +## Validate the fixed-table MVP + +Use two populated tables, `features` and `labels`: + +1. Capture main as `baseline`, then create `experiment` from that tag. Verify both tables contain + baseline data. Separately create a branch without `fromTag` and verify the schemas exist with no data. +2. Write both experiment tables through `$branch_experiment`; main must remain unchanged. +3. Capture `train_v1`, advance the source data and schema, and verify `$tag_train_v1` still reads the + captured versions, including native time travel and system-table boundaries. Tag writes must fail. +4. Stop writers and forward experiment to main. Invalidate/reload both main aliases and verify the + published tables. Resume independent writes on main and experiment and verify isolation. +5. Delete the experiment branch and verify the retained tag still reads correctly. Delete unused tags + and verify that cleanup preserves any versions retained by other branches or tags. + +The client tests cover shared Table REST payloads, paths, empty mutation responses, pagination, +authentication, error propagation, serialization and ordinary table reads/writes through suffixes. +Production branch/tag lifecycle, retention and forward still require integration tests against an +implementing catalog server. Branch-local namespace changes and additional table kinds are later work. diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index c099dff39691..30a3e9afa1ee 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -53,7 +53,7 @@ payloads, and error responses are defined in the OpenAPI specification. | Data access | Request storage credentials and authorize a query. | Table-scoped `token` and `auth`. | | Partitions | List, create, drop, and mark partitions done. | Table-scoped `partitions`. | | Table branches and tags | Manage named histories and retained snapshots. | Table-scoped `branches` and `tags`. | -| Database branches and tags | List, get, create, delete, and merge references. | Database-scoped `trees` and `trees/{name}/merge`. | +| Database branches and tags | Create, list, inspect, delete, and forward. | Database-scoped `branches`, `tags`, and `branches/{branch}/forward`. | | Consumers | List and reset streaming consumer progress. | Table-scoped `consumers`. | | Views and functions | Manage reusable SQL and function definitions. | Database- and catalog-scoped `views` and `functions`. | @@ -61,8 +61,8 @@ In this table, **table-scoped** means `/v1/{prefix}/databases/{database}/tables/{table}`. Catalog-wide listing and detail-listing endpoints are described in the specification alongside their database-scoped counterparts. -See [Database Branches and Tags](./database-versioning) for reference-management examples, merge -modes, and the server MVP design. Supported table operations select a reference with a database +See [Database Branches and Tags](./database-versioning) for table-aligned branch/tag examples, forward +semantics, and the server MVP design. Supported table operations select a reference with a database name such as `training$branch_experiment` or `training$tag_train_v1`. The existing table paths, request/response structures and Java methods carry the full database name. diff --git a/docs/scripts/validate-rest-openapi.js b/docs/scripts/validate-rest-openapi.js index a7bc8a1029b4..bf3d7d22b5b8 100644 --- a/docs/scripts/validate-rest-openapi.js +++ b/docs/scripts/validate-rest-openapi.js @@ -226,9 +226,36 @@ function requireExactEnum(contract, schemaName, expectedValues) { function validateCatalogOpenApi() { const contract = validateCommon('rest-catalog-open-api.yaml'); contract.checkSpec( - !Object.keys(contract.spec.paths).some((path) => /\/trees\/\{[^}]+\}\/(tables|table-details)/.test(path)), - 'Database reference access must reuse ordinary table paths', + !Object.keys(contract.spec.paths).some((path) => /\/trees(?:\/|$)/.test(path)), + 'Database branches and tags must use table-aligned paths; trees is not supported', ); + [ + ['createDatabaseBranch', 'CreateBranchRequest'], + ['forwardDatabaseBranch', 'ForwardBranchRequest'], + ['createDatabaseTag', 'CreateDatabaseTagRequest'], + ].forEach(([operationId, schemaName]) => { + const operation = contract.requireOperation(operationId); + contract.checkSpec( + operation.requestBody.content['application/json'].schema.$ref === `#/components/schemas/${schemaName}`, + `${operationId} must use ${schemaName}`, + ); + contract.checkSpec(!operation.responses['200'].content, `${operationId} must return no body`); + }); + [['listDatabaseBranches', 'ListBranchesResponse'], ['listDatabaseTagsPaged', 'ListTagsResponse'], + ['getDatabaseTag', 'GetDatabaseTagResponse']].forEach(([operationId, schemaName]) => { + contract.checkSpec( + contract.requireOperation(operationId).responses['200'].content['application/json'].schema.$ref === `#/components/schemas/${schemaName}`, + `${operationId} must use ${schemaName}`, + ); + }); + ['dropDatabaseBranch', 'deleteDatabaseTag'].forEach((operationId) => { + const operation = contract.requireOperation(operationId); + contract.checkSpec(!operation.requestBody && !operation.responses['200'].content, + `${operationId} must have no request or response body`); + }); + contract.checkSpec(!Object.keys(contract.spec.paths).some((path) => + /\/databases\/\{database\}\/branches\/[^/]+\/merge$/.test(path)), + 'Database merge is deferred'); const databaseParameter = contract.spec.components.parameters.Database; contract.checkSpec( databaseParameter.examples.branch.value === 'training$branch_experiment' && diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 0bf557b2eba1..9a43dbee8d0a 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -91,7 +91,7 @@ paths: $ref: "#/components/responses/ServerErrorResponse" description: >- List physical databases. Branch and tag access names are not additional database entries; - discover references through /databases/{database}/trees. + discover database branches and tags through /databases/{database}/branches and /databases/{database}/tags. post: tags: - database @@ -121,7 +121,7 @@ paths: $ref: "#/components/responses/ServerErrorResponse" description: >- Create a physical database. The reserved $branch_ and $tag_ suffix markers are not allowed; - create database references through /trees. + create database branches and tags through /branches and /tags. /v1/{prefix}/databases/{database}: get: tags: @@ -157,8 +157,8 @@ paths: description: >- A database name with $branch_ or $tag_ selects an existing reference. Return metadata for the virtual database, retaining the full requested name. This lookup supports - engine namespace existence checks. Missing databases or references return 404; a reference - type mismatch returns 409. Never resolve a missing reference to the base database. + engine namespace existence checks. Missing databases or references return 404; a missing branch or tag of the selected + type also returns 404. Never resolve a missing reference to the base database. delete: tags: - database @@ -187,7 +187,7 @@ paths: "500": $ref: "#/components/responses/ServerErrorResponse" description: >- - Database reference suffixes are not allowed for database mutation. Use the /trees management + Database reference suffixes are not allowed for database mutation. Use the /branches and /tags management endpoints with the physical database name to manage references. post: tags: @@ -226,14 +226,14 @@ paths: "500": $ref: "#/components/responses/ServerErrorResponse" description: >- - Database reference suffixes are not allowed for database mutation. Use the /trees management + Database reference suffixes are not allowed for database mutation. Use the /branches and /tags management endpoints with the physical database name to manage references. - /v1/{prefix}/databases/{database}/trees: + /v1/{prefix}/databases/{database}/branches: get: tags: - - database-reference - summary: List database references - operationId: listDatabaseReferencesPaged + - branch + summary: List database branches + operationId: listDatabaseBranches parameters: - name: prefix in: path @@ -245,30 +245,13 @@ paths: required: true schema: type: string - - name: type - in: query - required: false - schema: - type: string - enum: [ "branch", "tag" ] - - name: maxResults - in: query - required: false - schema: - type: integer - format: int32 - - name: pageToken - in: query - required: false - schema: - type: string responses: "200": - description: Database branches and immutable tags. + description: OK content: application/json: schema: - $ref: '#/components/schemas/ListDatabaseReferencesResponse' + $ref: '#/components/schemas/ListBranchesResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -277,9 +260,14 @@ paths: $ref: '#/components/responses/ServerErrorResponse' post: tags: - - database-reference - summary: Create database reference - operationId: createDatabaseReference + - branch + summary: Create database branch + description: >- + Uses the table CreateBranchRequest. Without fromTag, create a branch with main's table + membership and schemas but no data. With fromTag, restore the membership and versions + captured by that database tag. Database must be a physical name without a selector. + Branch and tag names have separate namespaces. The server protects main. + operationId: createDatabaseBranch parameters: - name: prefix in: path @@ -292,36 +280,49 @@ paths: schema: type: string requestBody: - required: true content: application/json: schema: - $ref: '#/components/schemas/CreateDatabaseReferenceRequest' + $ref: '#/components/schemas/CreateBranchRequest' responses: "200": - description: Created branch or immutable tag. - content: - application/json: - schema: - $ref: '#/components/schemas/DatabaseReferenceResponse' + description: Success, no content + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' - "409": - description: Reference already exists. + description: + Not Found + - DatabaseNotExistException, database does not exist + - TagNotExistException, the requested tag does not exist content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + examples: + DatabaseNotExist: + value: + code: 404 + resourceType: DATABASE + resourceName: training + message: Database does not exist + TagNotExist: + $ref: '#/components/examples/TagNotExistError' + "409": + $ref: '#/components/responses/BranchAlreadyExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/trees/{name}: - get: + /v1/{prefix}/databases/{database}/branches/{branch}: + delete: tags: - - database-reference - summary: Get database reference - operationId: getDatabaseReference + - branch + summary: Drop database branch + description: >- + Delete a database branch by name, with no request or response body. Deleting main is a + bad request. Retain any backing metadata and data still needed by database tags or other + branches. Database must be a physical name without a selector. + operationId: dropDatabaseBranch parameters: - name: prefix in: path @@ -333,33 +334,38 @@ paths: required: true schema: type: string - - name: name + - name: branch in: path required: true schema: type: string responses: "200": - description: Named branch or immutable tag. - content: - application/json: - schema: - $ref: '#/components/schemas/DatabaseReferenceResponse' + description: Success, no content + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - description: Database or reference does not exist. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' + $ref: '#/components/responses/BranchNotExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - delete: + /v1/{prefix}/databases/{database}/branches/{branch}/forward: + post: tags: - - database-reference - summary: Delete database reference - operationId: deleteDatabaseReference + - branch + summary: Forward main to a database branch + description: >- + Extends the table forward operation to the database's tables. The path names the source + branch; the physical database selects main as the target. Send an empty ForwardBranchRequest + and return success without a body. A tag is not a valid source. This publishes source table + versions on main; it does not combine target-only writes or perform three-way conflict + resolution. Source must be a non-main branch. Pause writers and reload tables after success. + For the fixed-table MVP, validate the same table membership and a snapshot for every source + table before forwarding. Unsupported namespace changes return 501. Preserve retained tags + and keep source and main independently writable. No public multi-table transaction or atomic + read view is promised; a success means all planned table updates completed. + operationId: forwardDatabaseBranch parameters: - name: prefix in: path @@ -371,65 +377,84 @@ paths: required: true schema: type: string - - name: name + - name: branch in: path required: true schema: type: string requestBody: - required: false content: application/json: schema: - $ref: '#/components/schemas/DeleteDatabaseReferenceRequest' + $ref: '#/components/schemas/ForwardBranchRequest' responses: "200": - description: Deleted branch or immutable tag. - content: - application/json: - schema: - $ref: '#/components/schemas/DatabaseReferenceResponse' + description: Success, no content + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - description: Database or reference does not exist. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - "409": - description: Reference type does not match or the default branch is protected. + $ref: '#/components/responses/BranchNotExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + "501": + $ref: '#/components/responses/ReferenceTableNotImplementedErrorResponse' + /v1/{prefix}/databases/{database}/tags: + get: + tags: + - tag + summary: List database tags + operationId: listDatabaseTagsPaged + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: maxResults + in: query + schema: + type: integer + format: int32 + - name: pageToken + in: query + schema: + type: string + - name: tagNamePrefix + description: A prefix for tag names. All tags will be returned if not set or empty. + in: query + schema: + type: string + responses: + "200": + description: OK content: application/json: schema: - $ref: '#/components/schemas/ErrorResponse' + $ref: '#/components/schemas/ListTagsResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/DatabaseNotExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/trees/{name}/merge: post: tags: - - database-reference - summary: Merge a branch or tag into a database branch + - tag + summary: Create database tag description: >- - The target must be a branch. The source branch or tag is resolved in the same database - when the request is processed. Merge compares complete table versions with their merge - base, including schemas, properties, snapshots, and table creation or deletion; table row - data is not merged. Changes made only on the target are preserved. Source-side changes - use defaultMergeMode (NORMAL when omitted), overridden by tableMergeModes for individual - table names. NORMAL accepts one-sided or identical changes and rejects different changes - to the same table. FORCE accepts the source-side change even on conflict, including a - deletion. DROP skips all source-side changes to that table, even without a conflict, and - preserves its target state. Modes do not replace target-only changes with unchanged source - versions. The server checks for unresolved conflicts before publishing the result; a - conflict leaves the target unchanged, and the source reference is never modified. - Identical references or an already-merged source succeed without modifying the target. - When the target is an ancestor of the source, the server may fast-forward only if the - selected modes produce exactly the source state. Divergent histories use three-way merge; - no available merge base is a conflict. A successful merge must record the source as merged, - including changes skipped by DROP, even when the table contents remain unchanged. Repeating - a merge of the same source state does not reapply skipped changes. The server retains - ancestry and merge relationships to resolve subsequent merges. - operationId: mergeDatabaseBranch + Capture each table version from fromBranch (main when absent or null), using the table + tagName and timeRetained conventions. Each table has its own snapshot; there is no single + database snapshotId. Tags have database-wide names and freeze membership, schema, properties, + and data. Database must be a physical name without a selector. Pause writes while capturing. + Expiring a database tag must not remove versions still needed by a branch or another tag. + operationId: createDatabaseTag parameters: - name: prefix in: path @@ -441,40 +466,111 @@ paths: required: true schema: type: string - - name: name - in: path - required: true - schema: - type: string requestBody: - required: true content: application/json: schema: - $ref: '#/components/schemas/MergeDatabaseBranchRequest' + $ref: '#/components/schemas/CreateDatabaseTagRequest' responses: "200": - description: Target branch after the merge, including when no changes were needed. - content: - application/json: - schema: - $ref: '#/components/schemas/DatabaseReferenceResponse' + description: Success, no content "400": $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - description: Database, target branch, or source reference does not exist. + description: + Not Found + - DatabaseNotExistException, database does not exist + - BranchNotExistException, the requested source branch does not exist content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' + examples: + DatabaseNotExist: + value: + code: 404 + resourceType: DATABASE + resourceName: training + message: Database does not exist + BranchNotExist: + value: + code: 404 + resourceType: BRANCH + resourceName: experiment + message: Source branch does not exist "409": - description: Target is not a branch, no merge base is available, or table conflicts remain unresolved. + $ref: '#/components/responses/TagAlreadyExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tags/{tag}: + get: + tags: + - tag + summary: Get database tag + description: >- + Get database tag metadata. fromBranch records its source but is not needed to read it: + load tables through database$tag_tagName using the ordinary table endpoints. Database + must be a physical name without a selector. Tag lookup remains valid after source deletion. + operationId: getDatabaseTag + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: tag + in: path + required: true + schema: + type: string + responses: + "200": + description: OK content: application/json: schema: - $ref: '#/components/schemas/ErrorResponse' + $ref: '#/components/schemas/GetDatabaseTagResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/TagNotExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + delete: + tags: + - tag + summary: Delete database tag + operationId: deleteDatabaseTag + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: tag + in: path + required: true + schema: + type: string + responses: + "200": + description: Success, no content + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/TagNotExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' /v1/{prefix}/databases/{database}/register: @@ -563,8 +659,7 @@ paths: description: >- The database path segment may select a database branch or immutable tag using its reserved suffix. Resolve the table through that reference. Tags expose captured membership and - metadata, never newer source state. Missing references never fall back to the base database. A - suffix whose type does not match the reference returns 409. + metadata, never newer source state. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. post: tags: - table @@ -601,10 +696,9 @@ paths: $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" description: >- The database path segment may select a database branch or immutable tag using its reserved - suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + suffix. Apply this operation to the selected branch; a tag returns 403. Identifiers in the body must retain the full database name including its suffix and agree with the path. Table - IDs must match the resolved table. Missing references never fall back to the base database. A - suffix whose type does not match the reference returns 409. + IDs must match the resolved table. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/databases/{database}/table-details: get: tags: @@ -661,8 +755,7 @@ paths: description: >- The database path segment may select a database branch or immutable tag using its reserved suffix. Resolve the table through that reference. Tags expose captured membership and - metadata, never newer source state. Missing references never fall back to the base database. A - suffix whose type does not match the reference returns 409. + metadata, never newer source state. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/tables: get: tags: @@ -781,8 +874,7 @@ paths: suffix. Resolve the table through that reference. Tags expose captured membership and metadata, never newer source state. Return the requested database name including its suffix, the logical table name, and resolved schema, path and storage options. Internal branch aliases - may be carried in schema options. Missing references never fall back to the base database. A - suffix whose type does not match the reference returns 409. + may be carried in schema options. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. post: tags: - table @@ -824,10 +916,9 @@ paths: $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" description: >- The database path segment may select a database branch or immutable tag using its reserved - suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + suffix. Apply this operation to the selected branch; a tag returns 403. Identifiers in the body must retain the full database name including its suffix and agree with the path. Table - IDs must match the resolved table. Missing references never fall back to the base database. A - suffix whose type does not match the reference returns 409. + IDs must match the resolved table. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. delete: tags: - table @@ -864,10 +955,9 @@ paths: $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" description: >- The database path segment may select a database branch or immutable tag using its reserved - suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + suffix. Apply this operation to the selected branch; a tag returns 403. Identifiers in the body must retain the full database name including its suffix and agree with the path. Table - IDs must match the resolved table. Missing references never fall back to the base database. A - suffix whose type does not match the reference returns 409. + IDs must match the resolved table. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/tables/rename: post: tags: @@ -944,10 +1034,9 @@ paths: $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" description: >- The database path segment may select a database branch or immutable tag using its reserved - suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + suffix. Apply this operation to the selected branch; a tag returns 403. Identifiers in the body must retain the full database name including its suffix and agree with the path. Table - IDs must match the resolved table. Missing references never fall back to the base database. A - suffix whose type does not match the reference returns 409. + IDs must match the resolved table. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/databases/{database}/tables/{table}/rollback: post: tags: @@ -1079,7 +1168,7 @@ paths: suffix. Resolve the table through that reference. Tags expose captured membership and metadata, never newer source state. Tag credentials must allow reading without permitting mutation of retained metadata or data. Missing references never fall back to the base - database. A suffix whose type does not match the reference returns 409. + database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/databases/{database}/tables/{table}/auth: post: tags: @@ -1127,8 +1216,7 @@ paths: description: >- The database path segment may select a database branch or immutable tag using its reserved suffix. Resolve the table through that reference. Tags expose captured membership and - metadata, never newer source state. Missing references never fall back to the base database. A - suffix whose type does not match the reference returns 409. + metadata, never newer source state. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/databases/{database}/tables/{table}/snapshot: get: tags: @@ -1172,8 +1260,7 @@ paths: The database path segment may select a database branch or immutable tag using its reserved suffix. Resolve the table through that reference. Tags expose captured membership and metadata, never newer source state. An existing empty table returns 404 with resourceType - SNAPSHOT. Missing references never fall back to the base database. A suffix whose type does - not match the reference returns 409. + SNAPSHOT. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/databases/{database}/tables/{table}/snapshots/{version}: get: tags: @@ -1223,8 +1310,7 @@ paths: suffix. Resolve the table through that reference. Tags expose captured membership and metadata, never newer source state. A database tag exposes only its pinned snapshot: LATEST and EARLIEST select it; other versions must resolve to it or return 404. Missing references - never fall back to the base database. A suffix whose type does not match the reference returns - 409. + never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/databases/{database}/tables/{table}/snapshots: get: tags: @@ -1278,8 +1364,7 @@ paths: suffix. Resolve the table through that reference. Tags expose captured membership and metadata, never newer source state. A database tag exposes only its pinned snapshot: LATEST and EARLIEST select it; other versions must resolve to it or return 404. Missing references - never fall back to the base database. A suffix whose type does not match the reference returns - 409. + never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/databases/{database}/tables/{table}/schemas: parameters: - $ref: "#/components/parameters/Prefix" @@ -1326,8 +1411,7 @@ paths: suffix. Resolve the table through that reference. Tags expose captured membership and metadata, never newer source state. For a database tag, LATEST selects the captured schema. History is limited to schemas retained for its captured data, excluding newer source schemas. - Missing references never fall back to the base database. A suffix whose type does not match - the reference returns 409. + Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/databases/{database}/tables/{table}/schemas/{version}: parameters: - $ref: "#/components/parameters/Prefix" @@ -1365,8 +1449,7 @@ paths: suffix. Resolve the table through that reference. Tags expose captured membership and metadata, never newer source state. For a database tag, LATEST selects the captured schema. History is limited to schemas retained for its captured data, excluding newer source schemas. - Missing references never fall back to the base database. A suffix whose type does not match - the reference returns 409. + Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. /v1/{prefix}/databases/{database}/tables/{table}/partitions: get: tags: @@ -2895,8 +2978,8 @@ components: $ref: "#/components/schemas/ErrorResponse" ReferenceTableConflictErrorResponse: description: >- - The reference type does not match the suffix, the target is an immutable tag, the table already - exists, or the operation conflicts with the selected table state. + The table or column already exists within the selected branch. Tag writes return 403; + a missing branch or tag of the selected type returns 404. content: application/json: schema: @@ -4283,96 +4366,37 @@ components: $ref: '#/components/schemas/Identifier' nextPageToken: type: string - DatabaseReference: - type: object - required: - - type - - name - properties: - type: - type: string - enum: [ "BRANCH", "TAG" ] - name: - type: string - pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" - CreateDatabaseReferenceRequest: + CreateDatabaseTagRequest: type: object required: - - name - - type - - source + - tagName properties: - name: + tagName: type: string pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" - type: - type: string - enum: [ "BRANCH", "TAG" ] - source: - $ref: '#/components/schemas/DatabaseReference' - MergeMode: - type: string - enum: [ "NORMAL", "FORCE", "DROP" ] - description: >- - NORMAL performs three-way conflict detection. FORCE accepts source-side table changes - even on conflict. DROP skips all source-side changes to the table and preserves its - target state. Each mode operates on complete table versions, not individual rows. - TableMergeMode: - type: object - required: - - table - - mergeMode - properties: - table: - type: string - description: Exact table name within the database being merged. - mergeMode: - $ref: '#/components/schemas/MergeMode' - MergeDatabaseBranchRequest: + fromBranch: + type: [ string, "null" ] + description: Source database branch; absent or null selects main. + timeRetained: + type: [ string, "null" ] + description: Optional retention duration, using the table tag duration syntax. + GetDatabaseTagResponse: type: object required: - - source - properties: - source: - $ref: '#/components/schemas/DatabaseReference' - defaultMergeMode: - description: Mode for tables without a per-table override. Defaults to NORMAL. - default: NORMAL - allOf: - - $ref: '#/components/schemas/MergeMode' - tableMergeModes: - type: array - description: >- - Per-table modes override defaultMergeMode. Omit or use an empty array to apply the - default to every table. Each table name may appear at most once; duplicates are a - bad request. Names without source-side changes have no effect. - items: - $ref: '#/components/schemas/TableMergeMode' - DeleteDatabaseReferenceRequest: - type: object + - tagName + - fromBranch properties: - type: + tagName: type: string - description: Expected reference type. Omit to delete without checking the type. - enum: [ "BRANCH", "TAG" ] - DatabaseReferenceResponse: - type: object - required: - - reference - properties: - reference: - $ref: '#/components/schemas/DatabaseReference' - ListDatabaseReferencesResponse: - type: object - required: - - references - properties: - references: - type: array - items: - $ref: '#/components/schemas/DatabaseReference' - nextPageToken: + fromBranch: type: string + description: Source branch recorded at creation, even if subsequently deleted. + tagCreateTime: + type: [ integer, "null" ] + format: int64 + description: Creation time as milliseconds since the Unix epoch. + tagTimeRetained: + type: [ string, "null" ] ConfigResponse: type: object properties: @@ -4407,9 +4431,7 @@ components: type: string ForwardBranchRequest: type: object - properties: - branch: - type: string + description: Empty action request, shared by table and database forward operations. ListBranchesResponse: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java index 8f864017676a..87cf377d4554 100644 --- a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -20,80 +20,55 @@ import org.apache.paimon.PagedList; import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.rest.DatabaseReference; -import org.apache.paimon.rest.DatabaseReferenceType; -import org.apache.paimon.rest.MergeMode; -import org.apache.paimon.rest.TableMergeMode; +import org.apache.paimon.rest.responses.GetDatabaseTagResponse; import javax.annotation.Nullable; import java.util.List; -/** Control-plane contract for database-level writable branches and immutable tags. */ +/** Database-level extensions of Paimon's table branch and tag operations. */ @Experimental public interface TreeManagement { + /** Lists branch names, using the same response as table branch listing. */ + List listBranches(String databaseName); + /** - * Lists one page of references. - * - * @param type reference type to include; null includes branches and tags - * @param maxResults maximum page size; null or zero uses the server default - * @param pageToken opaque continuation token; null for the first page + * Creates a branch. Without fromTag, copies main's table schemas without data. With fromTag, + * copies the membership and table versions captured by that database tag. */ - PagedList listReferencesPaged( - String databaseName, - @Nullable DatabaseReferenceType type, - @Nullable Integer maxResults, - @Nullable String pageToken); + void createBranch(String databaseName, String branch, @Nullable String fromTag); - /** Gets a named branch or tag. A missing reference is an error. */ - DatabaseReference getReference(String databaseName, String referenceName); - - /** Creates a branch or immutable tag from an existing reference in the same database. */ - DatabaseReference createReference( - String databaseName, - String referenceName, - DatabaseReferenceType type, - DatabaseReference source); + /** Drops a database branch. The default main branch is protected. */ + void dropBranch(String databaseName, String branch); /** - * Merges a branch or immutable tag into a target branch in the same database. - * - *

Table entries are merged relative to a common ancestor. Conflicting changes fail the merge - * without modifying the target; the source reference is never modified. A merge with no changes - * succeeds. The server automatically fast-forwards when possible. + * Forwards main to the named branch, extending the table fast-forward operation to the + * database's tables. The path names the source branch. This replaces target table state; it + * does not perform conflict resolution. Pause writers and reload tables after publication. */ - default DatabaseReference mergeBranch( - String databaseName, String targetBranch, DatabaseReference source) { - return mergeBranch(databaseName, targetBranch, source, null, null); - } + void fastForward(String databaseName, String branch); /** - * Merges a branch or immutable tag using default and per-table merge modes. - * - *

Modes apply to source-side changes to complete table versions, including creation and - * deletion; table row data is not merged. Per-table modes override the default. Unresolved - * conflicts leave the target unchanged, and the source is never modified. A successful merge - * records the source as merged, including changes skipped by {@link MergeMode#DROP}. - * - * @param defaultMergeMode mode for tables without an override; null means {@link - * MergeMode#NORMAL} - * @param tableMergeModes per-table overrides; null or empty uses the default for every table + * Captures an immutable database tag from a branch. Null fromBranch selects main. There is no + * database-wide snapshot ID; each table contributes its own captured version. */ - DatabaseReference mergeBranch( + void createTag( String databaseName, - String targetBranch, - DatabaseReference source, - @Nullable MergeMode defaultMergeMode, - @Nullable List tableMergeModes); + String tagName, + @Nullable String fromBranch, + @Nullable String timeRetained); - /** - * Deletes and returns a named reference. A missing reference is an error. - * - * @param expectedType required type of the reference to delete; null omits the type check - */ - DatabaseReference deleteReference( + /** Gets database tag metadata. Table versions are read through the tag-suffixed database. */ + GetDatabaseTagResponse getTag(String databaseName, String tagName); + + /** Lists tag names with the same pagination and prefix filter as table tag listing. */ + PagedList listTagsPaged( String databaseName, - String referenceName, - @Nullable DatabaseReferenceType expectedType); + @Nullable Integer maxResults, + @Nullable String pageToken, + @Nullable String tagNamePrefix); + + /** Deletes a database tag without deleting versions retained by another reference. */ + void deleteTag(String databaseName, String tagName); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java index 55b3e7813a62..6d13566bcb5d 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java @@ -20,16 +20,9 @@ import org.apache.paimon.annotation.Experimental; -import java.util.Locale; - /** Types of database-level references supported by the REST catalog. */ @Experimental public enum DatabaseReferenceType { BRANCH, - TAG; - - /** Lowercase form used by the trees query parameters. */ - public String queryValue() { - return name().toLowerCase(Locale.ROOT); - } + TAG } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java index 925809504a56..8205dfe21295 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -115,20 +115,12 @@ public T post( @Override public T delete(String path, RESTAuthFunction restAuthFunction) { - return delete(path, null, null, restAuthFunction); + return delete(path, null, restAuthFunction); } @Override public T delete( String path, RESTRequest body, RESTAuthFunction restAuthFunction) { - return delete(path, body, null, restAuthFunction); - } - - public T delete( - String path, - RESTRequest body, - Class responseType, - RESTAuthFunction restAuthFunction) { HttpDelete httpDelete = HttpClientUtils.newHttpDelete(getRequestUrl(path, null)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { @@ -136,7 +128,7 @@ public T delete( } Header[] authHeaders = getHeaders(path, "DELETE", encodedBody, restAuthFunction); httpDelete.setHeaders(authHeaders); - return exec(httpDelete, responseType); + return exec(httpDelete, null); } @VisibleForTesting diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java b/paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java deleted file mode 100644 index 8b18529e9a30..000000000000 --- a/paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest; - -import org.apache.paimon.annotation.Experimental; - -/** How source-side table changes are handled when merging database references. */ -@Experimental -public enum MergeMode { - /** Merge changes relative to the common ancestor, failing on conflicting table versions. */ - NORMAL, - - /** Accept source-side changes even when they conflict with the target table version. */ - FORCE, - - /** Skip all source-side changes to the table, preserving its target state. */ - DROP -} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index e71f674266a7..14f4545b22a4 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -39,7 +39,6 @@ import org.apache.paimon.rest.auth.RESTAuthFunction; import org.apache.paimon.rest.exceptions.AlreadyExistsException; import org.apache.paimon.rest.exceptions.ForbiddenException; -import org.apache.paimon.rest.exceptions.MergeConflictException; import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterFunctionRequest; @@ -48,14 +47,13 @@ import org.apache.paimon.rest.requests.AuthTableQueryRequest; import org.apache.paimon.rest.requests.CommitTableRequest; import org.apache.paimon.rest.requests.CreateBranchRequest; -import org.apache.paimon.rest.requests.CreateDatabaseReferenceRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.requests.CreateDatabaseTagRequest; import org.apache.paimon.rest.requests.CreateFunctionRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; -import org.apache.paimon.rest.requests.DeleteDatabaseReferenceRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.DropPolicyRequest; import org.apache.paimon.rest.requests.ForwardBranchRequest; @@ -63,7 +61,6 @@ import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; -import org.apache.paimon.rest.requests.MergeDatabaseBranchRequest; import org.apache.paimon.rest.requests.PolicyRequest; import org.apache.paimon.rest.requests.RegisterTableRequest; import org.apache.paimon.rest.requests.RenameTableRequest; @@ -79,10 +76,10 @@ import org.apache.paimon.rest.responses.CommitTableResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreatePartitionsResponse; -import org.apache.paimon.rest.responses.DatabaseReferenceResponse; import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; +import org.apache.paimon.rest.responses.GetDatabaseTagResponse; import org.apache.paimon.rest.responses.GetFunctionResponse; import org.apache.paimon.rest.responses.GetLabelResponse; import org.apache.paimon.rest.responses.GetSchemaResponse; @@ -95,7 +92,6 @@ import org.apache.paimon.rest.responses.GetViewResponse; import org.apache.paimon.rest.responses.ListBranchesResponse; import org.apache.paimon.rest.responses.ListConsumersResponse; -import org.apache.paimon.rest.responses.ListDatabaseReferencesResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListFunctionDetailsResponse; import org.apache.paimon.rest.responses.ListFunctionsGloballyResponse; @@ -200,8 +196,6 @@ public class RESTApi { public static final String PARTITION_NAME_PATTERN = "partitionNamePattern"; public static final String TAG_NAME_PREFIX = "tagNamePrefix"; - private static final String REFERENCE_TYPE = "type"; - public static final long TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000L; public static final ObjectMapper OBJECT_MAPPER = JsonSerdeUtil.OBJECT_MAPPER_INSTANCE; @@ -371,99 +365,86 @@ public void alterDatabase(String name, List removals, Map listDatabaseReferencesPaged( - String databaseName, - @Nullable DatabaseReferenceType type, - @Nullable Integer maxResults, - @Nullable String pageToken) { - Map queryParams = buildPagedQueryParams(maxResults, pageToken); - if (type != null) { - queryParams.put(REFERENCE_TYPE, type.queryValue()); - } - ListDatabaseReferencesResponse response = + public List listDatabaseBranches(String databaseName) { + ListBranchesResponse response = client.get( - resourcePaths.databaseTrees(databaseName), - queryParams, - ListDatabaseReferencesResponse.class, + resourcePaths.databaseBranches(databaseName), + ListBranchesResponse.class, restAuthFunction); - List references = response.getReferences(); - return new PagedList<>( - references == null ? emptyList() : references, response.getNextPageToken()); + return response.branches() == null ? emptyList() : response.branches(); } - /** Get one database-level branch or immutable tag. */ + /** Creates a schema-only branch, or restores the versions captured by fromTag. */ @Experimental - public DatabaseReference getDatabaseReference(String databaseName, String referenceName) { - DatabaseReferenceResponse response = - client.get( - resourcePaths.databaseTree(databaseName, referenceName), - DatabaseReferenceResponse.class, - restAuthFunction); - return checkNotNull(response.getReference(), "Reference response must contain reference"); + public void createDatabaseBranch(String databaseName, String branch, @Nullable String fromTag) { + client.post( + resourcePaths.databaseBranches(databaseName), + new CreateBranchRequest(branch, fromTag), + restAuthFunction); } - /** Create a database-level branch or immutable tag from an existing reference. */ + /** Drops a database branch. */ @Experimental - public DatabaseReference createDatabaseReference( - String databaseName, - String referenceName, - DatabaseReferenceType type, - DatabaseReference source) { - DatabaseReferenceResponse response = - client.post( - resourcePaths.databaseTrees(databaseName), - new CreateDatabaseReferenceRequest(referenceName, type, source), - DatabaseReferenceResponse.class, - restAuthFunction); - return checkNotNull(response.getReference(), "Reference response must contain reference"); + public void dropDatabaseBranch(String databaseName, String branch) { + client.delete(resourcePaths.databaseBranch(databaseName, branch), restAuthFunction); } - /** Merge a branch or immutable tag into a database-level branch, failing on conflicts. */ + /** Forwards main to the source branch, using the table forward request. */ @Experimental - public DatabaseReference mergeDatabaseBranch( - String databaseName, String targetBranch, DatabaseReference source) { - return mergeDatabaseBranch(databaseName, targetBranch, source, null, null); + public void fastForwardDatabase(String databaseName, String branch) { + client.post( + resourcePaths.forwardDatabaseBranch(databaseName, branch), + new ForwardBranchRequest(), + restAuthFunction); } - /** Merge a branch or immutable tag using default and per-table merge modes. */ + /** Captures the selected database branch; null fromBranch selects main. */ @Experimental - public DatabaseReference mergeDatabaseBranch( + public void createDatabaseTag( String databaseName, - String targetBranch, - DatabaseReference source, - @Nullable MergeMode defaultMergeMode, - @Nullable List tableMergeModes) { - try { - DatabaseReferenceResponse response = - client.post( - resourcePaths.mergeDatabaseBranch(databaseName, targetBranch), - new MergeDatabaseBranchRequest( - source, defaultMergeMode, tableMergeModes), - DatabaseReferenceResponse.class, - restAuthFunction); - return checkNotNull( - response.getReference(), "Reference response must contain reference"); - } catch (AlreadyExistsException e) { - throw new MergeConflictException( - e, e.resourceType(), e.resourceName(), "%s", e.getMessage()); - } + String tagName, + @Nullable String fromBranch, + @Nullable String timeRetained) { + client.post( + resourcePaths.databaseTags(databaseName), + new CreateDatabaseTagRequest(tagName, fromBranch, timeRetained), + restAuthFunction); + } + + /** Gets database tag metadata, without a fictitious database-wide snapshot ID. */ + @Experimental + public GetDatabaseTagResponse getDatabaseTag(String databaseName, String tagName) { + return client.get( + resourcePaths.databaseTag(databaseName, tagName), + GetDatabaseTagResponse.class, + restAuthFunction); } - /** Delete one database-level branch or immutable tag. */ + /** Lists database tag names with table tag pagination and prefix filtering. */ @Experimental - public DatabaseReference deleteDatabaseReference( + public PagedList listDatabaseTagsPaged( String databaseName, - String referenceName, - @Nullable DatabaseReferenceType expectedType) { - DatabaseReferenceResponse response = - client.delete( - resourcePaths.databaseTree(databaseName, referenceName), - new DeleteDatabaseReferenceRequest(expectedType), - DatabaseReferenceResponse.class, + @Nullable Integer maxResults, + @Nullable String pageToken, + @Nullable String tagNamePrefix) { + ListTagsResponse response = + client.get( + resourcePaths.databaseTags(databaseName), + buildPagedQueryParams( + maxResults, pageToken, Pair.of(TAG_NAME_PREFIX, tagNamePrefix)), + ListTagsResponse.class, restAuthFunction); - return checkNotNull(response.getReference(), "Reference response must contain reference"); + return new PagedList<>( + response.tags() == null ? emptyList() : response.tags(), + response.getNextPageToken()); + } + + /** Deletes a database tag. */ + @Experimental + public void deleteDatabaseTag(String databaseName, String tagName) { + client.delete(resourcePaths.databaseTag(databaseName, tagName), restAuthFunction); } /** diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java index f5cf357b877b..358c1e889f76 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java @@ -21,12 +21,13 @@ import org.apache.paimon.PagedList; import org.apache.paimon.annotation.Experimental; import org.apache.paimon.management.TreeManagement; +import org.apache.paimon.rest.responses.GetDatabaseTagResponse; import javax.annotation.Nullable; import java.util.List; -/** REST implementation of tree management, bound to the configured REST catalog prefix. */ +/** Database branch and tag management using the REST catalog's configuration. */ @Experimental public class RESTTreeManagement implements TreeManagement { @@ -37,44 +38,50 @@ public RESTTreeManagement(RESTApi api) { } @Override - public PagedList listReferencesPaged( - String databaseName, - @Nullable DatabaseReferenceType type, - @Nullable Integer maxResults, - @Nullable String pageToken) { - return api.listDatabaseReferencesPaged(databaseName, type, maxResults, pageToken); + public List listBranches(String databaseName) { + return api.listDatabaseBranches(databaseName); } @Override - public DatabaseReference getReference(String databaseName, String referenceName) { - return api.getDatabaseReference(databaseName, referenceName); + public void createBranch(String databaseName, String branch, @Nullable String fromTag) { + api.createDatabaseBranch(databaseName, branch, fromTag); } @Override - public DatabaseReference createReference( - String databaseName, - String referenceName, - DatabaseReferenceType type, - DatabaseReference source) { - return api.createDatabaseReference(databaseName, referenceName, type, source); + public void dropBranch(String databaseName, String branch) { + api.dropDatabaseBranch(databaseName, branch); } @Override - public DatabaseReference mergeBranch( + public void fastForward(String databaseName, String branch) { + api.fastForwardDatabase(databaseName, branch); + } + + @Override + public void createTag( String databaseName, - String targetBranch, - DatabaseReference source, - @Nullable MergeMode defaultMergeMode, - @Nullable List tableMergeModes) { - return api.mergeDatabaseBranch( - databaseName, targetBranch, source, defaultMergeMode, tableMergeModes); + String tagName, + @Nullable String fromBranch, + @Nullable String timeRetained) { + api.createDatabaseTag(databaseName, tagName, fromBranch, timeRetained); } @Override - public DatabaseReference deleteReference( + public GetDatabaseTagResponse getTag(String databaseName, String tagName) { + return api.getDatabaseTag(databaseName, tagName); + } + + @Override + public PagedList listTagsPaged( String databaseName, - String referenceName, - @Nullable DatabaseReferenceType expectedType) { - return api.deleteDatabaseReference(databaseName, referenceName, expectedType); + @Nullable Integer maxResults, + @Nullable String pageToken, + @Nullable String tagNamePrefix) { + return api.listDatabaseTagsPaged(databaseName, maxResults, pageToken, tagNamePrefix); + } + + @Override + public void deleteTag(String databaseName, String tagName) { + api.deleteDatabaseTag(databaseName, tagName); } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 22d6ce38d81f..5477a27314d6 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -36,7 +36,6 @@ public class ResourcePaths { protected static final String PARTITIONS = "partitions"; protected static final String BRANCHES = "branches"; protected static final String TAGS = "tags"; - protected static final String TREES = "trees"; protected static final String SNAPSHOTS = "snapshots"; protected static final String CONSUMERS = "consumers"; protected static final String SCHEMAS = "schemas"; @@ -148,23 +147,34 @@ public String database(String databaseName) { return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName)); } - /** Database-level branches and immutable tags. */ + /** Database-level extension of table branch management. */ @Experimental - public String databaseTrees(String databaseName) { - DatabaseIdentifier.checkNoReference(databaseName, "tree management"); - return SLASH.join(database(databaseName), TREES); + public String databaseBranches(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "database branch management"); + return SLASH.join(database(databaseName), BRANCHES); } - /** One named database-level branch or immutable tag. */ @Experimental - public String databaseTree(String databaseName, String referenceName) { - return SLASH.join(databaseTrees(databaseName), encodeString(referenceName)); + public String databaseBranch(String databaseName, String branch) { + return SLASH.join(databaseBranches(databaseName), encodeString(branch)); } - /** Action endpoint for merging a branch or tag into a database-level branch. */ + /** The path names the source branch; main is the target. */ @Experimental - public String mergeDatabaseBranch(String databaseName, String branch) { - return SLASH.join(databaseTree(databaseName, branch), "merge"); + public String forwardDatabaseBranch(String databaseName, String branch) { + return SLASH.join(databaseBranch(databaseName, branch), "forward"); + } + + /** Database-level extension of table tag management. */ + @Experimental + public String databaseTags(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "database tag management"); + return SLASH.join(database(databaseName), TAGS); + } + + @Experimental + public String databaseTag(String databaseName, String tagName) { + return SLASH.join(databaseTags(databaseName), encodeString(tagName)); } public String tables(String databaseName) { diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java b/paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java deleted file mode 100644 index b7eab7f34ed0..000000000000 --- a/paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest; - -import org.apache.paimon.annotation.Experimental; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; - -import java.beans.ConstructorProperties; - -/** Overrides the default merge mode for one table name within the database being merged. */ -@Experimental -@JsonIgnoreProperties(ignoreUnknown = true) -public class TableMergeMode { - - private static final String FIELD_TABLE = "table"; - private static final String FIELD_MERGE_MODE = "mergeMode"; - - private final String table; - private final MergeMode mergeMode; - - @JsonCreator - @ConstructorProperties({FIELD_TABLE, FIELD_MERGE_MODE}) - public TableMergeMode( - @JsonProperty(FIELD_TABLE) String table, - @JsonProperty(FIELD_MERGE_MODE) MergeMode mergeMode) { - this.table = table; - this.mergeMode = mergeMode; - } - - @JsonGetter(FIELD_TABLE) - public String getTable() { - return table; - } - - @JsonGetter(FIELD_MERGE_MODE) - public MergeMode getMergeMode() { - return mergeMode; - } -} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java b/paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java deleted file mode 100644 index ba8433bbe754..000000000000 --- a/paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.exceptions; - -import org.apache.paimon.annotation.Experimental; - -/** Exception thrown when an HTTP 409 prevents a database branch merge. */ -@Experimental -public class MergeConflictException extends RESTException { - - private final String resourceType; - private final String resourceName; - - public MergeConflictException( - Throwable cause, - String resourceType, - String resourceName, - String message, - Object... args) { - super(cause, message, args); - this.resourceType = resourceType; - this.resourceName = resourceName; - } - - public String resourceType() { - return resourceType; - } - - public String resourceName() { - return resourceName; - } -} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java deleted file mode 100644 index cb70e45dc471..000000000000 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.requests; - -import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.rest.DatabaseReference; -import org.apache.paimon.rest.DatabaseReferenceType; -import org.apache.paimon.rest.RESTRequest; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; - -import java.beans.ConstructorProperties; - -/** Request for creating a database branch or immutable tag from an existing reference. */ -@Experimental -@JsonIgnoreProperties(ignoreUnknown = true) -public class CreateDatabaseReferenceRequest implements RESTRequest { - - private static final String FIELD_NAME = "name"; - private static final String FIELD_TYPE = "type"; - private static final String FIELD_SOURCE = "source"; - - private final String name; - private final DatabaseReferenceType type; - private final DatabaseReference source; - - @JsonCreator - @ConstructorProperties({FIELD_NAME, FIELD_TYPE, FIELD_SOURCE}) - public CreateDatabaseReferenceRequest( - @JsonProperty(FIELD_NAME) String name, - @JsonProperty(FIELD_TYPE) DatabaseReferenceType type, - @JsonProperty(FIELD_SOURCE) DatabaseReference source) { - this.name = name; - this.type = type; - this.source = source; - } - - @JsonGetter(FIELD_NAME) - public String getName() { - return name; - } - - @JsonGetter(FIELD_TYPE) - public DatabaseReferenceType getType() { - return type; - } - - @JsonGetter(FIELD_SOURCE) - public DatabaseReference getSource() { - return source; - } -} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseTagRequest.java similarity index 53% rename from paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java rename to paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseTagRequest.java index a7193b9b608e..a454143cef7d 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseTagRequest.java @@ -19,39 +19,56 @@ package org.apache.paimon.rest.requests; import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.rest.DatabaseReferenceType; import org.apache.paimon.rest.RESTRequest; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import javax.annotation.Nullable; import java.beans.ConstructorProperties; -/** Request for deleting a database reference, optionally checking its type. */ +/** Database extension of tag creation: capture a branch instead of one table snapshot ID. */ @Experimental @JsonIgnoreProperties(ignoreUnknown = true) -public class DeleteDatabaseReferenceRequest implements RESTRequest { +public class CreateDatabaseTagRequest implements RESTRequest { - private static final String FIELD_TYPE = "type"; + private static final String FIELD_TAG_NAME = "tagName"; + private static final String FIELD_FROM_BRANCH = "fromBranch"; + private static final String FIELD_TIME_RETAINED = "timeRetained"; - @Nullable private final DatabaseReferenceType type; + private final String tagName; + @Nullable private final String fromBranch; + @Nullable private final String timeRetained; @JsonCreator - @ConstructorProperties({FIELD_TYPE}) - public DeleteDatabaseReferenceRequest( - @Nullable @JsonProperty(FIELD_TYPE) DatabaseReferenceType type) { - this.type = type; + @ConstructorProperties({FIELD_TAG_NAME, FIELD_FROM_BRANCH, FIELD_TIME_RETAINED}) + public CreateDatabaseTagRequest( + @JsonProperty(FIELD_TAG_NAME) String tagName, + @Nullable @JsonProperty(FIELD_FROM_BRANCH) String fromBranch, + @Nullable @JsonProperty(FIELD_TIME_RETAINED) String timeRetained) { + this.tagName = tagName; + this.fromBranch = fromBranch; + this.timeRetained = timeRetained; + } + + @JsonGetter(FIELD_TAG_NAME) + public String tagName() { + return tagName; + } + + /** Null selects the database's main branch. */ + @Nullable + @JsonGetter(FIELD_FROM_BRANCH) + public String fromBranch() { + return fromBranch; } @Nullable - @JsonGetter(FIELD_TYPE) - @JsonInclude(JsonInclude.Include.NON_NULL) - public DatabaseReferenceType getType() { - return type; + @JsonGetter(FIELD_TIME_RETAINED) + public String timeRetained() { + return timeRetained; } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java deleted file mode 100644 index 5beb2e7973ec..000000000000 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.requests; - -import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.rest.DatabaseReference; -import org.apache.paimon.rest.MergeMode; -import org.apache.paimon.rest.RESTRequest; -import org.apache.paimon.rest.TableMergeMode; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; - -import javax.annotation.Nullable; - -import java.beans.ConstructorProperties; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** Request for merging a branch or immutable tag into a database branch. */ -@Experimental -@JsonIgnoreProperties(ignoreUnknown = true) -public class MergeDatabaseBranchRequest implements RESTRequest { - - private static final String FIELD_SOURCE = "source"; - private static final String FIELD_DEFAULT_MERGE_MODE = "defaultMergeMode"; - private static final String FIELD_TABLE_MERGE_MODES = "tableMergeModes"; - - private final DatabaseReference source; - @Nullable private final MergeMode defaultMergeMode; - @Nullable private final List tableMergeModes; - - public MergeDatabaseBranchRequest(DatabaseReference source) { - this(source, null, null); - } - - @JsonCreator - @ConstructorProperties({FIELD_SOURCE, FIELD_DEFAULT_MERGE_MODE, FIELD_TABLE_MERGE_MODES}) - public MergeDatabaseBranchRequest( - @JsonProperty(FIELD_SOURCE) DatabaseReference source, - @Nullable @JsonProperty(FIELD_DEFAULT_MERGE_MODE) MergeMode defaultMergeMode, - @Nullable @JsonProperty(FIELD_TABLE_MERGE_MODES) List tableMergeModes) { - this.source = source; - this.defaultMergeMode = defaultMergeMode; - this.tableMergeModes = - tableMergeModes == null - ? null - : Collections.unmodifiableList(new ArrayList<>(tableMergeModes)); - } - - @JsonGetter(FIELD_SOURCE) - public DatabaseReference getSource() { - return source; - } - - /** Null uses the server default, {@link MergeMode#NORMAL}. */ - @Nullable - @JsonGetter(FIELD_DEFAULT_MERGE_MODE) - @JsonInclude(JsonInclude.Include.NON_NULL) - public MergeMode getDefaultMergeMode() { - return defaultMergeMode; - } - - /** Per-table modes override the default; null or empty supplies no overrides. */ - @Nullable - @JsonGetter(FIELD_TABLE_MERGE_MODES) - @JsonInclude(JsonInclude.Include.NON_NULL) - public List getTableMergeModes() { - return tableMergeModes; - } -} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java deleted file mode 100644 index d8570559fdd7..000000000000 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.responses; - -import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.rest.DatabaseReference; -import org.apache.paimon.rest.RESTResponse; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; - -import java.beans.ConstructorProperties; - -/** Response containing one database-level reference. */ -@Experimental -public class DatabaseReferenceResponse implements RESTResponse { - - private static final String FIELD_REFERENCE = "reference"; - - @JsonProperty(FIELD_REFERENCE) - private final DatabaseReference reference; - - @JsonCreator - @ConstructorProperties({FIELD_REFERENCE}) - public DatabaseReferenceResponse(@JsonProperty(FIELD_REFERENCE) DatabaseReference reference) { - this.reference = reference; - } - - @JsonGetter(FIELD_REFERENCE) - public DatabaseReference getReference() { - return reference; - } -} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetDatabaseTagResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetDatabaseTagResponse.java new file mode 100644 index 000000000000..9a5e88b53c59 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetDatabaseTagResponse.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +/** + * Database tag metadata. Its captured table versions are accessed through the database selector. + */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class GetDatabaseTagResponse implements RESTResponse { + + private static final String FIELD_TAG_NAME = "tagName"; + private static final String FIELD_FROM_BRANCH = "fromBranch"; + private static final String FIELD_TAG_CREATE_TIME = "tagCreateTime"; + private static final String FIELD_TAG_TIME_RETAINED = "tagTimeRetained"; + + private final String tagName; + private final String fromBranch; + @Nullable private final Long tagCreateTime; + @Nullable private final String tagTimeRetained; + + @JsonCreator + public GetDatabaseTagResponse( + @JsonProperty(FIELD_TAG_NAME) String tagName, + @JsonProperty(FIELD_FROM_BRANCH) String fromBranch, + @Nullable @JsonProperty(FIELD_TAG_CREATE_TIME) Long tagCreateTime, + @Nullable @JsonProperty(FIELD_TAG_TIME_RETAINED) String tagTimeRetained) { + this.tagName = tagName; + this.fromBranch = fromBranch; + this.tagCreateTime = tagCreateTime; + this.tagTimeRetained = tagTimeRetained; + } + + @JsonGetter(FIELD_TAG_NAME) + public String tagName() { + return tagName; + } + + @JsonGetter(FIELD_FROM_BRANCH) + public String fromBranch() { + return fromBranch; + } + + @Nullable + @JsonGetter(FIELD_TAG_CREATE_TIME) + public Long tagCreateTime() { + return tagCreateTime; + } + + @Nullable + @JsonGetter(FIELD_TAG_TIME_RETAINED) + public String tagTimeRetained() { + return tagTimeRetained; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java deleted file mode 100644 index 3fbbc18e7887..000000000000 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest.responses; - -import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.rest.DatabaseReference; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; - -import javax.annotation.Nullable; - -import java.beans.ConstructorProperties; -import java.util.List; - -/** Paged response for database-level branches and tags. */ -@Experimental -public class ListDatabaseReferencesResponse implements PagedResponse { - - private static final String FIELD_REFERENCES = "references"; - private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken"; - - @JsonProperty(FIELD_REFERENCES) - private final List references; - - @Nullable - @JsonProperty(FIELD_NEXT_PAGE_TOKEN) - private final String nextPageToken; - - @JsonCreator - @ConstructorProperties({FIELD_REFERENCES, FIELD_NEXT_PAGE_TOKEN}) - public ListDatabaseReferencesResponse( - @JsonProperty(FIELD_REFERENCES) List references, - @Nullable @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) { - this.references = references; - this.nextPageToken = nextPageToken; - } - - @JsonGetter(FIELD_REFERENCES) - public List getReferences() { - return references; - } - - @Nullable - @JsonGetter(FIELD_NEXT_PAGE_TOKEN) - @Override - public String getNextPageToken() { - return nextPageToken; - } - - @Override - public List data() { - return references; - } -} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseBranchTagTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseBranchTagTest.java new file mode 100644 index 000000000000..867ab09a147f --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseBranchTagTest.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.responses.GetDatabaseTagResponse; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.apache.paimon.rest.RESTCatalogInternalOptions.PREFIX; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.URI; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** HTTP contract tests for database-level branches and immutable tags. */ +class RESTApiDatabaseBranchTagTest { + + private static final String DATABASE_PATH = "/v1/catalog%2Fid/databases/training+db"; + + private final Queue replies = new ConcurrentLinkedQueue<>(); + private final List requests = new CopyOnWriteArrayList<>(); + + private HttpServer server; + private RESTApi api; + + @BeforeEach + void setUp() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/v1/", + exchange -> { + requests.add(new Request(exchange)); + Reply reply = replies.poll(); + if (reply == null) { + reply = new Reply(500, "{\"code\":500,\"message\":\"unexpected request\"}"); + } + byte[] data = reply.body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(reply.code, data.length == 0 ? -1 : data.length); + if (data.length > 0) { + try (OutputStream output = exchange.getResponseBody()) { + output.write(data); + } + } + exchange.close(); + }); + server.start(); + + Options options = new Options(); + options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); + options.set(PREFIX, "catalog/id"); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "test-token"); + api = new RESTApi(options, false); + } + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(0); + } + } + + @Test + void testBranchRequestsMatchTableRequests() throws Exception { + Identifier table = Identifier.create("training db", "features"); + enqueue(200, ""); + api.createDatabaseBranch("training db", "experiment", "baseline"); + enqueue(200, ""); + api.createBranch(table, "experiment", "baseline"); + assertRequest(0, "POST", DATABASE_PATH + "/branches"); + assertRequest(1, "POST", DATABASE_PATH + "/tables/features/branches"); + assertBody(requests.get(0), "{\"branch\":\"experiment\",\"fromTag\":\"baseline\"}"); + assertThat(requests.get(0).body).isEqualTo(requests.get(1).body); + + enqueue(200, ""); + api.createDatabaseBranch("training db", "empty", null); + assertBody(requests.get(2), "{\"branch\":\"empty\",\"fromTag\":null}"); + + enqueue(200, ""); + api.fastForwardDatabase("training db", "experiment"); + enqueue(200, ""); + api.fastForward(table, "experiment"); + assertRequest(3, "POST", DATABASE_PATH + "/branches/experiment/forward"); + assertRequest(4, "POST", DATABASE_PATH + "/tables/features/branches/experiment/forward"); + assertBody(requests.get(3), "{}"); + assertThat(requests.get(3).body).isEqualTo(requests.get(4).body); + + enqueue(200, ""); + api.dropDatabaseBranch("training db", "experiment"); + assertRequest(5, "DELETE", DATABASE_PATH + "/branches/experiment"); + assertThat(requests.get(5).body).isEmpty(); + assertThat(requests.get(5).query).isNull(); + assertThat(requests).hasSize(6); + } + + @Test + void testListBranchesUsesTableResponse() { + enqueue(200, "{\"branches\":[\"main\",\"experiment\"]}"); + assertThat(api.listDatabaseBranches("training db")).containsExactly("main", "experiment"); + assertRequest(0, "GET", DATABASE_PATH + "/branches"); + assertThat(requests.get(0).query).isNull(); + enqueue(200, "{}"); + assertThat(api.listDatabaseBranches("training db")).isEmpty(); + } + + @Test + void testDatabaseTagMetadataAndDefaults() throws Exception { + enqueue(200, ""); + api.createDatabaseTag("training db", "train-v1", "experiment", "7d"); + assertRequest(0, "POST", DATABASE_PATH + "/tags"); + assertBody( + requests.get(0), + "{\"tagName\":\"train-v1\",\"fromBranch\":\"experiment\",\"timeRetained\":\"7d\"}"); + + enqueue(200, ""); + api.createDatabaseTag("training db", "baseline", null, null); + assertBody( + requests.get(1), + "{\"tagName\":\"baseline\",\"fromBranch\":null,\"timeRetained\":null}"); + + enqueue( + 200, + "{\"tagName\":\"train-v1\",\"fromBranch\":\"experiment\",\"tagCreateTime\":1720000000000,\"tagTimeRetained\":\"7d\",\"futureField\":true}"); + GetDatabaseTagResponse tag = api.getDatabaseTag("training db", "train-v1"); + assertThat(tag.tagName()).isEqualTo("train-v1"); + assertThat(tag.fromBranch()).isEqualTo("experiment"); + assertThat(tag.tagCreateTime()).isEqualTo(1720000000000L); + assertThat(tag.tagTimeRetained()).isEqualTo("7d"); + assertRequest(2, "GET", DATABASE_PATH + "/tags/train-v1"); + + enqueue(200, ""); + api.deleteDatabaseTag("training db", "train-v1"); + assertRequest(3, "DELETE", DATABASE_PATH + "/tags/train-v1"); + assertThat(requests.get(3).body).isEmpty(); + assertThat(requests.get(3).query).isNull(); + assertThat(requests).hasSize(4); + } + + @Test + void testTagPagesUseTableResponseAndPreserveTokens() { + enqueue(200, "{\"tags\":[\"train-v1\"],\"nextPageToken\":\"next +/%?&\"}"); + PagedList first = + api.listDatabaseTagsPaged("training db", 10, "start +/%", "train-"); + assertThat(first.getElements()).containsExactly("train-v1"); + assertThat(first.getNextPageToken()).isEqualTo("next +/%?&"); + assertRequest(0, "GET", DATABASE_PATH + "/tags"); + Map query = queryParameters(requests.get(0).query); + assertThat(query) + .hasSize(3) + .containsEntry("maxResults", "10") + .containsEntry("pageToken", "start +/%") + .containsEntry("tagNamePrefix", "train-"); + + enqueue(200, "{\"tags\":[\"train-v2\"]}"); + PagedList second = + api.listDatabaseTagsPaged("training db", null, first.getNextPageToken(), "train-"); + assertThat(second.getElements()).containsExactly("train-v2"); + assertThat(second.getNextPageToken()).isNull(); + assertThat(queryParameters(requests.get(1).query)) + .hasSize(2) + .containsEntry("pageToken", "next +/%?&") + .containsEntry("tagNamePrefix", "train-"); + + enqueue(200, "{\"tags\":null,\"nextPageToken\":\"continue\"}"); + PagedList empty = api.listDatabaseTagsPaged("training db", null, null, null); + assertThat(empty.getElements()).isEmpty(); + assertThat(empty.getNextPageToken()).isEqualTo("continue"); + assertThat(requests.get(2).query).isNull(); + } + + @Test + void testManagementRejectsVirtualDatabaseNamesBeforeSendingRequests() { + for (String database : + new String[] {"training db$branch_experiment", "training db$tag_train-v1"}) { + assertThatThrownBy(() -> api.listDatabaseBranches(database)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.createDatabaseBranch(database, "new", null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.dropDatabaseBranch(database, "experiment")) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.fastForwardDatabase(database, "experiment")) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.createDatabaseTag(database, "new", null, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.getDatabaseTag(database, "train-v1")) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.listDatabaseTagsPaged(database, null, null, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.deleteDatabaseTag(database, "train-v1")) + .isInstanceOf(UnsupportedOperationException.class); + } + assertThat(requests).isEmpty(); + } + + private void enqueue(int code, String body) { + replies.add(new Reply(code, body)); + } + + private void assertRequest(int index, String method, String path) { + Request request = requests.get(index); + assertThat(request.method).isEqualTo(method); + assertThat(request.path).isEqualTo(path); + assertThat(request.authorization).isEqualTo("Bearer test-token"); + } + + private static void assertBody(Request request, String expectedJson) throws Exception { + assertThat(request.query).isNull(); + assertThat(RESTApi.fromJson(request.body, Map.class)) + .isEqualTo(RESTApi.fromJson(expectedJson, Map.class)); + } + + private static Map queryParameters(String query) { + Map values = new LinkedHashMap<>(); + if (query == null || query.isEmpty()) { + return values; + } + for (String parameter : query.split("&")) { + String[] pair = parameter.split("=", 2); + values.put(decode(pair[0]), decode(pair[1])); + } + return values; + } + + private static String decode(String value) { + try { + return URLDecoder.decode(value, "UTF-8"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static class Reply { + private final int code; + private final String body; + + private Reply(int code, String body) { + this.code = code; + this.body = body; + } + } + + private static class Request { + private final String method; + private final String path; + private final String query; + private final String body; + private final String authorization; + + private Request(HttpExchange exchange) throws IOException { + method = exchange.getRequestMethod(); + path = exchange.getRequestURI().getRawPath(); + query = exchange.getRequestURI().getRawQuery(); + body = read(exchange.getRequestBody()); + authorization = exchange.getRequestHeaders().getFirst("Authorization"); + } + + private static String read(InputStream input) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = input.read(buffer)) >= 0) { + output.write(buffer, 0, length); + } + return new String(output.toByteArray(), StandardCharsets.UTF_8); + } + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java deleted file mode 100644 index d8991249eac4..000000000000 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.rest; - -import org.apache.paimon.PagedList; -import org.apache.paimon.options.Options; - -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpServer; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.CopyOnWriteArrayList; - -import static org.apache.paimon.rest.RESTCatalogInternalOptions.PREFIX; -import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; -import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; -import static org.apache.paimon.rest.RESTCatalogOptions.URI; -import static org.assertj.core.api.Assertions.assertThat; - -/** HTTP contract tests for database-level branches and immutable tags. */ -class RESTApiDatabaseReferenceTest { - - private static final String TREES_PATH = "/v1/catalog%2Fid/databases/training+db/trees"; - - private final Queue replies = new ConcurrentLinkedQueue<>(); - private final List requests = new CopyOnWriteArrayList<>(); - - private HttpServer server; - private RESTApi api; - - @BeforeEach - void setUp() throws IOException { - server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); - server.createContext( - "/v1/", - exchange -> { - requests.add(new Request(exchange)); - Reply reply = replies.poll(); - if (reply == null) { - reply = new Reply(500, "{\"code\":500,\"message\":\"unexpected request\"}"); - } - byte[] data = reply.body.getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders().set("Content-Type", "application/json"); - exchange.sendResponseHeaders(reply.code, data.length); - try (OutputStream output = exchange.getResponseBody()) { - output.write(data); - } finally { - exchange.close(); - } - }); - server.start(); - - Options options = new Options(); - options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); - options.set(PREFIX, "catalog/id"); - options.set(TOKEN_PROVIDER, "bear"); - options.set(TOKEN, "test-token"); - api = new RESTApi(options, false); - } - - @AfterEach - void tearDown() { - if (server != null) { - server.stop(0); - } - } - - @ParameterizedTest - @EnumSource(DatabaseReferenceType.class) - void testBranchAndImmutableTagHappyPath(DatabaseReferenceType sourceType) throws Exception { - enqueue( - 200, - "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}," - + "{\"type\":\"TAG\",\"name\":\"train-v1\"}]," - + "\"nextPageToken\":\"next\"}"); - PagedList page = - api.listDatabaseReferencesPaged( - "training db", DatabaseReferenceType.TAG, 100, "start token"); - assertThat(page.getElements()) - .containsExactly( - new DatabaseReference(DatabaseReferenceType.BRANCH, "main"), - new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); - assertThat(page.getNextPageToken()).isEqualTo("next"); - assertRequest(0, "GET", TREES_PATH); - assertThat(queryParameters(requests.get(0).query)) - .containsEntry("type", "tag") - .containsEntry("maxResults", "100") - .containsEntry("pageToken", "start token"); - - enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); - assertThat(api.getDatabaseReference("training db", "main")) - .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - assertRequest(1, "GET", TREES_PATH + "/main"); - - enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); - DatabaseReference branch = - api.createDatabaseReference( - "training db", - "exp-1", - DatabaseReferenceType.BRANCH, - new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - assertThat(branch).isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); - assertRequest(2, "POST", TREES_PATH); - assertBody( - requests.get(2), - "{\"name\":\"exp-1\",\"type\":\"BRANCH\"," - + "\"source\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); - - enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); - DatabaseReference tag = - api.createDatabaseReference( - "training db", - "train-v1", - DatabaseReferenceType.TAG, - new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); - assertThat(tag).isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); - assertRequest(3, "POST", TREES_PATH); - assertBody( - requests.get(3), - "{\"name\":\"train-v1\",\"type\":\"TAG\"," - + "\"source\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); - - enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); - DatabaseReference source = sourceType == DatabaseReferenceType.BRANCH ? branch : tag; - assertThat(api.mergeDatabaseBranch("training db", "main", source)) - .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - assertRequest(4, "POST", TREES_PATH + "/main/merge"); - assertBody( - requests.get(4), - sourceType == DatabaseReferenceType.BRANCH - ? "{\"source\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}" - : "{\"source\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); - - enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); - assertThat( - api.deleteDatabaseReference( - "training db", "exp-1", DatabaseReferenceType.BRANCH)) - .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); - assertRequest(5, "DELETE", TREES_PATH + "/exp-1"); - assertBody(requests.get(5), "{\"type\":\"BRANCH\"}"); - } - - @Test - void testListReferencesPaged() { - enqueue( - 200, - "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}]," - + "\"nextPageToken\":\"p2\"}"); - enqueue(200, "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"exp-1\"}]}"); - - PagedList first = - api.listDatabaseReferencesPaged( - "training db", DatabaseReferenceType.BRANCH, 1, null); - assertThat(first.getElements()) - .containsExactly(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - assertThat(first.getNextPageToken()).isEqualTo("p2"); - assertThat(requests).hasSize(1); - - PagedList second = - api.listDatabaseReferencesPaged( - "training db", DatabaseReferenceType.BRANCH, 1, first.getNextPageToken()); - assertThat(second.getElements()) - .containsExactly(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); - assertThat(second.getNextPageToken()).isNull(); - assertThat(requests).hasSize(2); - assertThat(queryParameters(requests.get(0).query)).containsEntry("type", "branch"); - assertThat(queryParameters(requests.get(1).query)) - .containsEntry("type", "branch") - .containsEntry("pageToken", "p2"); - } - - @ParameterizedTest - @EnumSource(DatabaseReferenceType.class) - void testMergeBranchOrTag(DatabaseReferenceType sourceType) throws Exception { - enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); - DatabaseReference source = new DatabaseReference(sourceType, "experiment"); - - assertThat(api.mergeDatabaseBranch("training db", "main", source)) - .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - - assertRequest(0, "POST", TREES_PATH + "/main/merge"); - assertBody( - requests.get(0), - sourceType == DatabaseReferenceType.BRANCH - ? "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}}" - : "{\"source\":{\"type\":\"TAG\",\"name\":\"experiment\"}}"); - assertThat(requests).hasSize(1); - } - - @ParameterizedTest - @EnumSource(MergeMode.class) - void testMergeModesAreSentInBody(MergeMode defaultMergeMode) throws Exception { - enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); - DatabaseReference source = - new DatabaseReference(DatabaseReferenceType.BRANCH, "experiment"); - - assertThat( - api.mergeDatabaseBranch( - "training db", - "main", - source, - defaultMergeMode, - Arrays.asList( - new TableMergeMode("features.v2", MergeMode.FORCE), - new TableMergeMode("scratch", MergeMode.DROP), - new TableMergeMode("labels", MergeMode.NORMAL)))) - .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - - assertRequest(0, "POST", TREES_PATH + "/main/merge"); - assertBody( - requests.get(0), - "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}," - + "\"defaultMergeMode\":\"" - + defaultMergeMode.name() - + "\"," - + "\"tableMergeModes\":[{\"table\":\"features.v2\",\"mergeMode\":\"FORCE\"}," - + "{\"table\":\"scratch\",\"mergeMode\":\"DROP\"}," - + "{\"table\":\"labels\",\"mergeMode\":\"NORMAL\"}]}"); - assertThat(requests).hasSize(1); - } - - @Test - void testDeleteReferenceWithoutType() throws Exception { - enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); - assertThat(api.deleteDatabaseReference("training db", "train-v1", null)) - .isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); - assertRequest(0, "DELETE", TREES_PATH + "/train-v1"); - assertBody(requests.get(0), "{}"); - } - - private void enqueue(int code, String body) { - replies.add(new Reply(code, body)); - } - - private void assertRequest(int index, String method, String path) { - Request request = requests.get(index); - assertThat(request.method).isEqualTo(method); - assertThat(request.path).isEqualTo(path); - assertThat(request.authorization).isEqualTo("Bearer test-token"); - } - - private static void assertBody(Request request, String expectedJson) throws Exception { - assertThat(request.query).isNull(); - assertThat(RESTApi.fromJson(request.body, Map.class)) - .isEqualTo(RESTApi.fromJson(expectedJson, Map.class)); - } - - private static Map queryParameters(String query) { - Map values = new LinkedHashMap<>(); - if (query == null || query.isEmpty()) { - return values; - } - for (String parameter : query.split("&")) { - String[] pair = parameter.split("=", 2); - values.put(decode(pair[0]), decode(pair[1])); - } - return values; - } - - private static String decode(String value) { - try { - return URLDecoder.decode(value, "UTF-8"); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private static class Reply { - private final int code; - private final String body; - - private Reply(int code, String body) { - this.code = code; - this.body = body; - } - } - - private static class Request { - private final String method; - private final String path; - private final String query; - private final String body; - private final String authorization; - - private Request(HttpExchange exchange) throws IOException { - method = exchange.getRequestMethod(); - path = exchange.getRequestURI().getRawPath(); - query = exchange.getRequestURI().getRawQuery(); - body = read(exchange.getRequestBody()); - authorization = exchange.getRequestHeaders().getFirst("Authorization"); - } - - private static String read(InputStream input) throws IOException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - byte[] buffer = new byte[1024]; - int length; - while ((length = input.read(buffer)) >= 0) { - output.write(buffer, 0, length); - } - return new String(output.toByteArray(), StandardCharsets.UTF_8); - } - } -} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 9e67468e6324..83ad1da80192 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -18,20 +18,14 @@ package org.apache.paimon.rest.requests; -import org.apache.paimon.rest.DatabaseReference; -import org.apache.paimon.rest.DatabaseReferenceType; -import org.apache.paimon.rest.MergeMode; import org.apache.paimon.rest.RESTApi; import org.apache.paimon.rest.RESTRequest; -import org.apache.paimon.rest.TableMergeMode; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; import java.beans.ConstructorProperties; import java.lang.reflect.Constructor; @@ -164,6 +158,17 @@ public class RequestJacksonCompatibilityTest { "{\"schemaId\":44}", request -> assertThat(request.getSchemaId()).isEqualTo(44L), "schemaId"), + requestCase( + CreateDatabaseTagRequest.class, + "{\"tagName\":\"train-v1\",\"fromBranch\":\"experiment\",\"timeRetained\":\"7d\"}", + request -> { + assertThat(request.tagName()).isEqualTo("train-v1"); + assertThat(request.fromBranch()).isEqualTo("experiment"); + assertThat(request.timeRetained()).isEqualTo("7d"); + }, + "tagName", + "fromBranch", + "timeRetained"), requestCase( UpsertLabelRequest.class, "{\"value\":\"identifier\"}", @@ -177,15 +182,12 @@ public class RequestJacksonCompatibilityTest { AlterTableRequest.class, AlterViewRequest.class, CommitTableRequest.class, - CreateDatabaseReferenceRequest.class, CreateFunctionRequest.class, CreatePartitionsRequest.class, CreateTableRequest.class, CreateViewRequest.class, - DeleteDatabaseReferenceRequest.class, DropPolicyRequest.class, GrantPermissionRequest.class, - MergeDatabaseBranchRequest.class, PolicyRequest.class, RegisterTableRequest.class, RenameTableRequest.class, @@ -221,108 +223,15 @@ void testConstructorPropertyNamesAndOrder(RequestCase requestCase) { } @Test - void testCreateDatabaseReferenceRequestRoundTrips() throws Exception { - String json = - "{\"name\":\"exp-1\",\"type\":\"BRANCH\"," - + "\"source\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"; - CreateDatabaseReferenceRequest request = - EXTERNAL_MAPPER.readValue(json, CreateDatabaseReferenceRequest.class); - CreateDatabaseReferenceRequest roundTrip = - RESTApi.fromJson(RESTApi.toJson(request), CreateDatabaseReferenceRequest.class); - assertThat(roundTrip.getName()).isEqualTo("exp-1"); - assertThat(roundTrip.getType()).isEqualTo(DatabaseReferenceType.BRANCH); - assertThat(roundTrip.getSource()) - .isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); - } - - @Test - void testDeleteDatabaseReferenceRequestRoundTrips() throws Exception { - DeleteDatabaseReferenceRequest request = + void testDatabaseTagRequestDefaultsRoundTrip() throws Exception { + CreateDatabaseTagRequest request = EXTERNAL_MAPPER.readValue( - "{\"type\":\"TAG\"}", DeleteDatabaseReferenceRequest.class); - assertThat( - RESTApi.fromJson( - RESTApi.toJson(request), - DeleteDatabaseReferenceRequest.class) - .getType()) - .isEqualTo(DatabaseReferenceType.TAG); - - DeleteDatabaseReferenceRequest withoutType = - EXTERNAL_MAPPER.readValue("{}", DeleteDatabaseReferenceRequest.class); - assertThat(RESTApi.toJson(withoutType)).isEqualTo("{}"); - assertThat(RESTApi.fromJson("{}", DeleteDatabaseReferenceRequest.class).getType()).isNull(); - } - - @ParameterizedTest - @EnumSource(DatabaseReferenceType.class) - void testMergeDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) - throws Exception { - String json = - "{\"source\":{\"type\":\"" + sourceType.name() + "\",\"name\":\"experiment\"}}"; - MergeDatabaseBranchRequest request = - EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class); - MergeDatabaseBranchRequest roundTrip = - RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); - assertThat(roundTrip.getSource()) - .isEqualTo(new DatabaseReference(sourceType, "experiment")); - assertThat(roundTrip.getDefaultMergeMode()).isNull(); - assertThat(roundTrip.getTableMergeModes()).isNull(); - assertThat(RESTApi.fromJson(RESTApi.toJson(roundTrip), Map.class)) - .isEqualTo(RESTApi.fromJson(json, Map.class)); - } - - @ParameterizedTest - @EnumSource(MergeMode.class) - void testMergeModesRoundTrip(MergeMode mode) throws Exception { - String json = - "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}," - + "\"defaultMergeMode\":\"" - + mode.name() - + "\"," - + "\"tableMergeModes\":[{\"table\":\"features.v2\",\"mergeMode\":\"FORCE\"}," - + "{\"table\":\"scratch\",\"mergeMode\":\"DROP\"}," - + "{\"table\":\"labels\",\"mergeMode\":\"NORMAL\"}]}"; - MergeDatabaseBranchRequest request = - EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class); - MergeDatabaseBranchRequest roundTrip = - RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); - assertThat(roundTrip.getDefaultMergeMode()).isEqualTo(mode); - assertThat(roundTrip.getTableMergeModes()) - .extracting(TableMergeMode::getTable) - .containsExactly("features.v2", "scratch", "labels"); - assertThat(roundTrip.getTableMergeModes()) - .extracting(TableMergeMode::getMergeMode) - .containsExactly(MergeMode.FORCE, MergeMode.DROP, MergeMode.NORMAL); - assertThat(RESTApi.fromJson(RESTApi.toJson(roundTrip), Map.class)) - .isEqualTo(RESTApi.fromJson(json, Map.class)); - } - - @Test - void testMergeWithEmptyOverrides() throws Exception { - String json = - "{\"source\":{\"type\":\"TAG\",\"name\":\"train-v1\"},\"tableMergeModes\":[]}"; - MergeDatabaseBranchRequest request = - EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class); - MergeDatabaseBranchRequest roundTrip = - RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); - assertThat(roundTrip.getDefaultMergeMode()).isNull(); - assertThat(roundTrip.getTableMergeModes()).isEmpty(); - assertThat(RESTApi.fromJson(RESTApi.toJson(roundTrip), Map.class)) - .isEqualTo(RESTApi.fromJson(json, Map.class)); - } - - @ParameterizedTest - @ValueSource( - strings = { - "\"defaultMergeMode\":\"UNKNOWN\"", - "\"tableMergeModes\":[{\"table\":\"features\",\"mergeMode\":\"UNKNOWN\"}]" - }) - void testUnknownMergeModesAreRejected(String modes) { - String json = "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}," + modes + "}"; - assertThatThrownBy(() -> EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class)) - .hasMessageContaining("UNKNOWN"); - assertThatThrownBy(() -> RESTApi.fromJson(json, MergeDatabaseBranchRequest.class)) - .hasMessageContaining("UNKNOWN"); + "{\"tagName\":\"train-v1\"}", CreateDatabaseTagRequest.class); + CreateDatabaseTagRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), CreateDatabaseTagRequest.class); + assertThat(roundTrip.tagName()).isEqualTo("train-v1"); + assertThat(roundTrip.fromBranch()).isNull(); + assertThat(roundTrip.timeRetained()).isNull(); } @Test diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index acf362fc95e9..8c8abea95c87 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -658,6 +658,10 @@ public void createTable(Identifier identifier, Schema schema, boolean ignoreIfEx Schema newSchema = inferSchemaIfExternalPaimonTable(schema); api.createTable(identifier, newSchema); } catch (AlreadyExistsException e) { + if (DatabaseIdentifier.parse(identifier.getDatabaseName()).getReference() != null + && !StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_TABLE)) { + throw e; + } if (!ignoreIfExists) { throw new TableAlreadyExistException(identifier); } @@ -704,6 +708,10 @@ public void alterTable( try { api.alterTable(identifier, changes); } catch (NoSuchResourceException e) { + if (!StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_TABLE) + && !StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_COLUMN)) { + throw e; + } if (!ignoreIfNotExists) { if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_TABLE)) { throw new TableNotExistException(identifier); @@ -713,6 +721,10 @@ public void alterTable( } } } catch (AlreadyExistsException e) { + if (DatabaseIdentifier.parse(identifier.getDatabaseName()).getReference() != null + && !StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_COLUMN)) { + throw e; + } throw new ColumnAlreadyExistException(identifier, e.resourceName()); } catch (ForbiddenException e) { throw new TableNoPermissionException(identifier, e); @@ -1231,6 +1243,10 @@ public PagedList listFunctionDetailsPaged( @Override public View getView(Identifier identifier) throws ViewNotExistException { try { + if (DatabaseIdentifier.parse(identifier.getDatabaseName()).getReference() != null) { + api.getDatabase(identifier.getDatabaseName()); + throw new ViewNotExistException(identifier); + } GetViewResponse response = api.getView(identifier); return toView(identifier.getDatabaseName(), response); } catch (NoSuchResourceException e) { @@ -1282,6 +1298,10 @@ public void createView(Identifier identifier, View view, boolean ignoreIfExists) @Override public List listViews(String databaseName) throws DatabaseNotExistException { try { + if (DatabaseIdentifier.parse(databaseName).getReference() != null) { + api.getDatabase(databaseName); + return Collections.emptyList(); + } return CatalogUtils.isSystemDatabase(databaseName) ? Collections.emptyList() : api.listViews(databaseName); @@ -1300,6 +1320,10 @@ public PagedList listViewsPaged( @Nullable String viewNamePattern) throws DatabaseNotExistException { try { + if (DatabaseIdentifier.parse(databaseName).getReference() != null) { + api.getDatabase(databaseName); + return new PagedList<>(Collections.emptyList(), null); + } return api.listViewsPaged(databaseName, maxResults, pageToken, viewNamePattern); } catch (NoSuchResourceException e) { throw new DatabaseNotExistException(databaseName); @@ -1316,6 +1340,10 @@ public PagedList listViewDetailsPaged( @Nullable String viewNamePattern) throws DatabaseNotExistException { try { + if (DatabaseIdentifier.parse(db).getReference() != null) { + api.getDatabase(db); + return new PagedList<>(Collections.emptyList(), null); + } PagedList views = api.listViewDetailsPaged(db, maxResults, pageToken, viewNamePattern); return new PagedList<>( diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java index b71d8c93a681..de67c179da57 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java @@ -275,7 +275,7 @@ void testErrorsDoNotFallBackToDefaultBranch() throws Exception { assertThatThrownBy(() -> catalog.getTable(selected)) .isInstanceOf(Catalog.TableNotExistException.class); takeRequest("GET", DATABASE_PATH + "%24tag_train_v1/tables/features"); - enqueue(409, "{\"message\":\"tag is immutable\",\"code\":409}"); + enqueue(403, "{\"message\":\"tag is immutable\",\"code\":403}"); assertThatThrownBy( () -> catalog.commitSnapshot( @@ -284,12 +284,64 @@ void testErrorsDoNotFallBackToDefaultBranch() throws Exception { null, Snapshot.fromJson(SNAPSHOT_JSON), emptyList())) - .isInstanceOf(AlreadyExistsException.class) + .isInstanceOf(Catalog.TableNoPermissionException.class) .hasMessageContaining("tag is immutable"); takeRequest("POST", DATABASE_PATH + "%24tag_train_v1/tables/features/commit"); assertThat(server.getRequestCount()).isEqualTo(3); } + @Test + void testReferenceErrorsAreNotIgnoredByTableDdl() throws Exception { + Identifier selected = Identifier.create(DATABASE + "$branch_experiment", "features"); + enqueue( + 409, + "{\"code\":409,\"resourceType\":\"BRANCH\",\"message\":\"branch is not writable\"}"); + assertThatThrownBy(() -> catalog.createTable(selected, schema("main"), true)) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageContaining("not writable"); + takeRequest("POST", DATABASE_PATH + "%24branch_experiment/tables"); + + enqueue(409, "{\"code\":409,\"resourceType\":\"TABLE\",\"message\":\"table exists\"}"); + catalog.createTable(selected, schema("main"), true); + takeRequest("POST", DATABASE_PATH + "%24branch_experiment/tables"); + + for (boolean ignore : new boolean[] {false, true}) { + enqueue( + 404, + "{\"code\":404,\"resourceType\":\"BRANCH\",\"message\":\"branch missing\"}"); + assertThatThrownBy( + () -> + catalog.alterTable( + selected, + singletonList(SchemaChange.setOption("key", "value")), + ignore)) + .isInstanceOf(org.apache.paimon.rest.exceptions.NoSuchResourceException.class) + .hasMessageContaining("branch missing"); + takeRequest("POST", DATABASE_PATH + "%24branch_experiment/tables/features"); + } + } + + @Test + void testViewProbesAllowTableOnlyReferenceNamespaces() throws Exception { + String database = DATABASE + "$branch_experiment"; + String databasePath = DATABASE_PATH + "%24branch_experiment"; + for (int i = 0; i < 4; i++) { + enqueue(200, "{\"name\":\"training db$branch_experiment\",\"options\":{}}"); + } + assertThat(catalog.listViews(database)).isEmpty(); + assertThat(catalog.listViewsPaged(database, 10, null, null).getElements()).isEmpty(); + assertThat(catalog.listViewDetailsPaged(database, 10, null, null).getElements()).isEmpty(); + assertThatThrownBy(() -> catalog.getView(Identifier.create(database, "features"))) + .isInstanceOf(Catalog.ViewNotExistException.class); + for (int i = 0; i < 4; i++) { + takeRequest("GET", databasePath); + } + enqueue(404, "{\"code\":404,\"resourceType\":\"BRANCH\",\"message\":\"branch missing\"}"); + assertThatThrownBy(() -> catalog.listViews(database)) + .isInstanceOf(Catalog.DatabaseNotExistException.class); + takeRequest("GET", databasePath); + } + @Test void testUnsupportedDatabaseOperationsAndMixedSelectorsDoNotSendRequests() { RESTApi api = catalog.api(); @@ -320,7 +372,7 @@ database, emptyList(), java.util.Collections.emptyMap())) .isInstanceOf(UnsupportedOperationException.class); assertThatThrownBy(() -> catalog.dropDatabase(database, true, true)) .isInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> catalog.treeManagement().getReference(database, "main")) + assertThatThrownBy(() -> catalog.treeManagement().listBranches(database)) .isInstanceOf(UnsupportedOperationException.class); assertThat(server.getRequestCount()).isEqualTo(1); } @@ -347,7 +399,7 @@ void testBatchReadWriteAndPinnedTagWithRealDataFiles() throws Exception { org.apache.paimon.fs.Path location = new org.apache.paimon.fs.Path(tempDir.resolve("features").toUri()); LocalFileIO fileIO = LocalFileIO.create(); - for (String branch : new String[] {"main", "physical-experiment"}) { + for (String branch : new String[] {"main", "physical-experiment", "frozen-train-v1"}) { new FileSystemSchemaManager(fileIO, location, branch).createTable(schema(branch)); } Map snapshots = new ConcurrentHashMap<>(); @@ -381,7 +433,11 @@ public MockResponse dispatch(RecordedRequest request) { return response(500, "{}"); } String branch = - reference.equals("main") ? "main" : "physical-experiment"; + reference.equals("main") + ? "main" + : reference.equals("train_v1") + ? "frozen-train-v1" + : "physical-experiment"; if (request.getMethod().equals("GET") && parts.length == 3) { return response( 200, @@ -406,7 +462,7 @@ public MockResponse dispatch(RecordedRequest request) { && parts[3].equals("commit")) { if (reference.equals("train_v1")) { return response( - 409, "{\"code\":409,\"message\":\"tag is immutable\"}"); + 403, "{\"code\":403,\"message\":\"tag is immutable\"}"); } CommitTableRequest commit = RESTApi.fromJson( @@ -434,6 +490,11 @@ public MockResponse dispatch(RecordedRequest request) { writeRows(main, 10); writeRows(experiment, 20); snapshots.put("train_v1", snapshots.get("experiment")); + // REST latest alone cannot constrain native metadata reads. Return frozen backing metadata. + fileIO.overwriteFileUtf8( + new SnapshotManager(fileIO, location, "frozen-train-v1", null, null) + .snapshotPath(snapshots.get("train_v1").id()), + snapshots.get("train_v1").toJson()); Identifier tag = Identifier.create(DATABASE + "$tag_train_v1", "features"); assertThat(readRows(tag)).containsExactly(20); @@ -446,6 +507,20 @@ public MockResponse dispatch(RecordedRequest request) { assertThat(readRows(tag)).containsExactly(20); assertThat(snapshots.get("train_v1").id()).isEqualTo(1); assertThat(snapshots.get("experiment").id()).isEqualTo(2); + new FileSystemSchemaManager(fileIO, location, "physical-experiment") + .commitChanges(SchemaChange.addColumn("later", DataTypes.INT())); + FileStoreTable frozen = (FileStoreTable) catalog.getTable(tag); + assertThat(frozen.copyWithLatestSchema().schema().id()).isZero(); + assertThat(frozen.schemaManager().listAll()).hasSize(1); + assertThatThrownBy( + () -> + frozen.copy( + java.util.Collections.singletonMap( + "scan.snapshot-id", "2")) + .newReadBuilder() + .newScan() + .plan()) + .isInstanceOf(IllegalArgumentException.class); assertThat(unexpected).isEmpty(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index e4cb58ebc7a9..9c4ab362392c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -24,7 +24,6 @@ import org.apache.paimon.options.Options; import org.apache.paimon.rest.exceptions.AlreadyExistsException; import org.apache.paimon.rest.exceptions.BadRequestException; -import org.apache.paimon.rest.exceptions.MergeConflictException; import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.exceptions.NotImplementedException; @@ -34,16 +33,11 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; import static org.apache.paimon.options.CatalogOptions.WAREHOUSE; -import static org.apache.paimon.rest.DatabaseReferenceType.BRANCH; -import static org.apache.paimon.rest.DatabaseReferenceType.TAG; import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; import static org.apache.paimon.rest.RESTCatalogOptions.URI; @@ -54,10 +48,7 @@ class RESTCatalogTreeManagementTest { private static final String DATABASE = "training db"; - private static final String TREES_PATH = "/v1/catalog%2Fid/databases/training+db/trees"; - private static final String MAIN_JSON = "{\"type\":\"BRANCH\",\"name\":\"main\"}"; - private static final String BRANCH_JSON = "{\"type\":\"BRANCH\",\"name\":\"exp-1\"}"; - private static final String TAG_JSON = "{\"type\":\"TAG\",\"name\":\"train-v1\"}"; + private static final String DATABASE_PATH = "/v1/catalog%2Fid/databases/training+db"; private MockWebServer server; private RESTCatalog catalog; @@ -97,228 +88,118 @@ void tearDown() throws Exception { } } - @ParameterizedTest - @EnumSource(DatabaseReferenceType.class) - void testBranchAndTagOperationsUseCatalogConfiguration(DatabaseReferenceType sourceType) - throws Exception { - DatabaseReference main = new DatabaseReference(BRANCH, "main"); - DatabaseReference branch = new DatabaseReference(BRANCH, "exp-1"); - DatabaseReference tag = new DatabaseReference(TAG, "train-v1"); - - enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); - assertThat(trees.getReference(DATABASE, "main")).isEqualTo(main); - takeRequest("GET", TREES_PATH + "/main"); + @Test + void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { + enqueue(200, "{\"branches\":[\"main\",\"experiment\"]}"); + assertThat(trees.listBranches(DATABASE)).containsExactly("main", "experiment"); + takeRequest("GET", DATABASE_PATH + "/branches"); - enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); - assertThat(trees.createReference(DATABASE, "exp-1", BRANCH, main)).isEqualTo(branch); - RecordedRequest createBranch = takeRequest("POST", TREES_PATH); + enqueue(200, ""); + trees.createTag(DATABASE, "baseline", null, null); assertBody( - createBranch, - "{\"name\":\"exp-1\",\"type\":\"BRANCH\",\"source\":" + MAIN_JSON + "}"); + takeRequest("POST", DATABASE_PATH + "/tags"), + "{\"tagName\":\"baseline\",\"fromBranch\":null,\"timeRetained\":null}"); - enqueue(200, "{\"reference\":" + TAG_JSON + "}"); - assertThat(trees.createReference(DATABASE, "train-v1", TAG, branch)).isEqualTo(tag); - RecordedRequest createTag = takeRequest("POST", TREES_PATH); + enqueue(200, ""); + trees.createBranch(DATABASE, "experiment", "baseline"); assertBody( - createTag, - "{\"name\":\"train-v1\",\"type\":\"TAG\",\"source\":" + BRANCH_JSON + "}"); - - enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); - DatabaseReference source = sourceType == BRANCH ? branch : tag; - assertThat(trees.mergeBranch(DATABASE, "main", source)).isEqualTo(main); - RecordedRequest merge = takeRequest("POST", TREES_PATH + "/main/merge"); - assertBody(merge, "{\"source\":" + (sourceType == BRANCH ? BRANCH_JSON : TAG_JSON) + "}"); - - enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); - assertThat(trees.deleteReference(DATABASE, "exp-1", BRANCH)).isEqualTo(branch); - RecordedRequest deleteBranch = takeRequest("DELETE", TREES_PATH + "/exp-1"); - assertBody(deleteBranch, "{\"type\":\"BRANCH\"}"); + takeRequest("POST", DATABASE_PATH + "/branches"), + "{\"branch\":\"experiment\",\"fromTag\":\"baseline\"}"); - enqueue(200, "{\"reference\":" + TAG_JSON + "}"); - assertThat(trees.deleteReference(DATABASE, "train-v1", null)).isEqualTo(tag); - assertBody(takeRequest("DELETE", TREES_PATH + "/train-v1"), "{}"); - assertThat(server.getRequestCount()).isEqualTo(7); + enqueue(200, ""); + trees.createTag(DATABASE, "train-v1", "experiment", "7d"); + assertBody( + takeRequest("POST", DATABASE_PATH + "/tags"), + "{\"tagName\":\"train-v1\",\"fromBranch\":\"experiment\",\"timeRetained\":\"7d\"}"); + + enqueue(200, "{\"tagName\":\"train-v1\",\"fromBranch\":\"experiment\"}"); + assertThat(trees.getTag(DATABASE, "train-v1").fromBranch()).isEqualTo("experiment"); + takeRequest("GET", DATABASE_PATH + "/tags/train-v1"); + + enqueue(200, ""); + trees.fastForward(DATABASE, "experiment"); + assertBody(takeRequest("POST", DATABASE_PATH + "/branches/experiment/forward"), "{}"); + + enqueue(200, ""); + trees.dropBranch(DATABASE, "experiment"); + assertThat(takeRequest("DELETE", DATABASE_PATH + "/branches/experiment").getBodySize()) + .isZero(); + + enqueue(200, ""); + trees.deleteTag(DATABASE, "train-v1"); + assertThat(takeRequest("DELETE", DATABASE_PATH + "/tags/train-v1").getBodySize()).isZero(); + assertThat(server.getRequestCount()).isEqualTo(9); } @Test - void testListPagesPreserveFilterAndTokens() throws Exception { - enqueue(200, "{\"references\":[" + TAG_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); - PagedList page = - trees.listReferencesPaged(DATABASE, TAG, 10, "start +/%"); - assertThat(page.getElements()).containsExactly(new DatabaseReference(TAG, "train-v1")); + void testTagPagesPreserveFilterAndTokens() throws Exception { + enqueue(200, "{\"tags\":[\"train-v1\"],\"nextPageToken\":\"next +/%?&\"}"); + PagedList page = trees.listTagsPaged(DATABASE, 10, "start +/%", "train-"); + assertThat(page.getElements()).containsExactly("train-v1"); assertThat(page.getNextPageToken()).isEqualTo("next +/%?&"); - RecordedRequest paged = takeRequest("GET", TREES_PATH); - assertThat(paged.getRequestUrl().queryParameter("type")).isEqualTo("tag"); - assertThat(paged.getRequestUrl().queryParameter("maxResults")).isEqualTo("10"); - assertThat(paged.getRequestUrl().queryParameter("pageToken")).isEqualTo("start +/%"); - - enqueue(200, "{\"references\":[" + MAIN_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); - enqueue(200, "{\"references\":[" + BRANCH_JSON + "]}"); - PagedList firstPage = - trees.listReferencesPaged(DATABASE, BRANCH, null, null); - assertThat(firstPage.getElements()).containsExactly(new DatabaseReference(BRANCH, "main")); - assertThat(firstPage.getNextPageToken()).isEqualTo("next +/%?&"); - RecordedRequest first = takeRequest("GET", TREES_PATH); - assertThat(first.getRequestUrl().queryParameter("type")).isEqualTo("branch"); - assertThat(first.getRequestUrl().queryParameter("pageToken")).isNull(); - PagedList secondPage = - trees.listReferencesPaged(DATABASE, BRANCH, null, firstPage.getNextPageToken()); - assertThat(secondPage.getElements()) - .containsExactly(new DatabaseReference(BRANCH, "exp-1")); - assertThat(secondPage.getNextPageToken()).isNull(); - RecordedRequest second = takeRequest("GET", TREES_PATH); - assertThat(second.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + RecordedRequest first = takeRequest("GET", DATABASE_PATH + "/tags"); + assertThat(first.getRequestUrl().queryParameter("maxResults")).isEqualTo("10"); + assertThat(first.getRequestUrl().queryParameter("pageToken")).isEqualTo("start +/%"); + assertThat(first.getRequestUrl().queryParameter("tagNamePrefix")).isEqualTo("train-"); + + enqueue(200, "{\"tags\":[]}"); + PagedList last = + trees.listTagsPaged(DATABASE, null, page.getNextPageToken(), "train-"); + assertThat(last.getElements()).isEmpty(); + assertThat(last.getNextPageToken()).isNull(); + RecordedRequest second = takeRequest("GET", DATABASE_PATH + "/tags"); assertThat(second.getRequestUrl().queryParameter("pageToken")).isEqualTo("next +/%?&"); + assertThat(second.getRequestUrl().queryParameter("tagNamePrefix")).isEqualTo("train-"); assertThat(second.getRequestUrl().queryParameter("maxResults")).isNull(); } - @ParameterizedTest - @EnumSource(DatabaseReferenceType.class) - void testMergeUsesCatalogConfiguration(DatabaseReferenceType sourceType) throws Exception { - enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); - DatabaseReference source = new DatabaseReference(sourceType, "experiment"); - - assertThat(trees.mergeBranch(DATABASE, "main", source)) - .isEqualTo(new DatabaseReference(BRANCH, "main")); - - RecordedRequest merge = takeRequest("POST", TREES_PATH + "/main/merge"); - assertBody( - merge, - sourceType == BRANCH - ? "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}}" - : "{\"source\":{\"type\":\"TAG\",\"name\":\"experiment\"}}"); - assertThat(server.getRequestCount()).isEqualTo(2); - } - - @ParameterizedTest - @EnumSource(DatabaseReferenceType.class) - void testMergeModesUseCatalogConfiguration(DatabaseReferenceType sourceType) throws Exception { - enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); - DatabaseReference source = new DatabaseReference(sourceType, "experiment"); - - assertThat( - trees.mergeBranch( - DATABASE, - "main", - source, - MergeMode.NORMAL, - Arrays.asList( - new TableMergeMode("features", MergeMode.FORCE), - new TableMergeMode("scratch", MergeMode.DROP)))) - .isEqualTo(new DatabaseReference(BRANCH, "main")); - - assertBody( - takeRequest("POST", TREES_PATH + "/main/merge"), - "{\"source\":{\"type\":\"" - + sourceType.name() - + "\",\"name\":\"experiment\"}," - + "\"defaultMergeMode\":\"NORMAL\",\"tableMergeModes\":[" - + "{\"table\":\"features\",\"mergeMode\":\"FORCE\"}," - + "{\"table\":\"scratch\",\"mergeMode\":\"DROP\"}]}"); - assertThat(server.getRequestCount()).isEqualTo(2); - } - @Test - void testMergeErrorsPreserveDetails() throws Exception { - DatabaseReference source = new DatabaseReference(BRANCH, "experiment"); + void testErrorsKeepTableBranchAndTagConventions() throws Exception { server.enqueue( new MockResponse() .setResponseCode(409) .setHeader("Content-Type", "application/json") - .setHeader("x-request-id", "merge-request") + .setHeader("x-request-id", "branch-request") .setBody( - "{\"message\":\"Conflicting changes to table features (100%)\"," - + "\"resourceType\":\"TABLE\",\"resourceName\":\"training db.features\"}")); - assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) - .isInstanceOfSatisfying( - MergeConflictException.class, - conflict -> { - assertThat(conflict.resourceType()).isEqualTo("TABLE"); - assertThat(conflict.resourceName()).isEqualTo("training db.features"); - assertThat(conflict.getCause()) - .isInstanceOf(AlreadyExistsException.class) - .hasMessage(conflict.getMessage()); - }) - .hasMessage("Conflicting changes to table features (100%) requestId:merge-request"); - takeRequest("POST", TREES_PATH + "/main/merge"); - - enqueue(409, "{\"code\":409,\"message\":\"reference already exists\"}"); - assertThatThrownBy(() -> trees.createReference(DATABASE, "existing", BRANCH, source)) - .isExactlyInstanceOf(AlreadyExistsException.class); - takeRequest("POST", TREES_PATH); + "{\"resourceType\":\"BRANCH\",\"resourceName\":\"experiment\",\"message\":\"branch exists\"}")); + assertThatThrownBy(() -> trees.createBranch(DATABASE, "experiment", null)) + .isExactlyInstanceOf(AlreadyExistsException.class) + .hasMessageContaining("branch exists") + .hasMessageContaining("branch-request"); + takeRequest("POST", DATABASE_PATH + "/branches"); - enqueue(400, "{\"code\":400,\"message\":\"duplicate table merge mode\"}"); - assertThatThrownBy( - () -> - trees.mergeBranch( - DATABASE, - "main", - source, - MergeMode.NORMAL, - Arrays.asList( - new TableMergeMode("features", MergeMode.FORCE), - new TableMergeMode("features", MergeMode.DROP)))) + enqueue( + 404, + "{\"code\":404,\"resourceType\":\"TAG\",\"resourceName\":\"baseline\",\"message\":\"tag missing\"}"); + assertThatThrownBy(() -> trees.createBranch(DATABASE, "experiment", "baseline")) + .isInstanceOfSatisfying( + NoSuchResourceException.class, + e -> { + assertThat(e.resourceType()).isEqualTo("TAG"); + assertThat(e.resourceName()).isEqualTo("baseline"); + }); + takeRequest("POST", DATABASE_PATH + "/branches"); + + enqueue(400, "{\"code\":400,\"message\":\"source table has no snapshot\"}"); + assertThatThrownBy(() -> trees.fastForward(DATABASE, "empty")) .isInstanceOf(BadRequestException.class) - .hasMessageContaining("duplicate table merge mode"); - takeRequest("POST", TREES_PATH + "/main/merge"); + .hasMessageContaining("no snapshot"); + takeRequest("POST", DATABASE_PATH + "/branches/empty/forward"); - enqueue(404, "{\"code\":404,\"message\":\"source reference missing\"}"); - assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) + enqueue(404, "{\"code\":404,\"message\":\"branch missing\"}"); + assertThatThrownBy(() -> trees.fastForward(DATABASE, "missing")) .isInstanceOf(NoSuchResourceException.class) - .hasMessageContaining("source reference missing"); - takeRequest("POST", TREES_PATH + "/main/merge"); + .hasMessageContaining("branch missing"); + takeRequest("POST", DATABASE_PATH + "/branches/missing/forward"); - enqueue(501, "{\"code\":501,\"message\":\"merge unsupported\"}"); - assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) + enqueue(501, "{\"code\":501,\"message\":\"forward unsupported\"}"); + assertThatThrownBy(() -> trees.fastForward(DATABASE, "experiment")) .isInstanceOf(NotImplementedException.class) - .hasMessageContaining("merge unsupported"); - takeRequest("POST", TREES_PATH + "/main/merge"); + .hasMessageContaining("forward unsupported"); + takeRequest("POST", DATABASE_PATH + "/branches/experiment/forward"); assertThat(server.getRequestCount()).isEqualTo(6); } - @Test - void testListAllTypesAndEmptyReferences() throws Exception { - enqueue(200, "{\"references\":[" + MAIN_JSON + "," + TAG_JSON + "]}"); - assertThat(trees.listReferencesPaged(DATABASE, null, null, null).getElements()) - .containsExactly( - new DatabaseReference(BRANCH, "main"), - new DatabaseReference(TAG, "train-v1")); - assertThat(takeRequest("GET", TREES_PATH).getRequestUrl().query()).isNull(); - - enqueue(200, "{\"references\":[]}"); - PagedList emptyPage = - trees.listReferencesPaged(DATABASE, null, null, null); - assertThat(emptyPage.getElements()).isEmpty(); - assertThat(emptyPage.getNextPageToken()).isNull(); - takeRequest("GET", TREES_PATH); - assertThat(server.getRequestCount()).isEqualTo(3); - } - - @Test - void testErrorsPropagate() { - enqueue(404, "{\"code\":404,\"message\":\"reference missing\"}"); - assertThatThrownBy(() -> trees.getReference(DATABASE, "missing")) - .isInstanceOf(NoSuchResourceException.class) - .hasMessageContaining("reference missing"); - - enqueue(409, "{\"code\":409,\"message\":\"reference already exists\"}"); - assertThatThrownBy( - () -> - trees.createReference( - DATABASE, - "exp-1", - BRANCH, - new DatabaseReference(BRANCH, "main"))) - .isInstanceOf(AlreadyExistsException.class) - .hasMessageContaining("reference already exists"); - - enqueue(501, "{\"code\":501,\"message\":\"trees unsupported\"}"); - assertThatThrownBy(() -> trees.listReferencesPaged(DATABASE, null, null, null)) - .isInstanceOf(NotImplementedException.class) - .hasMessageContaining("trees unsupported"); - assertThat(server.getRequestCount()).isEqualTo(4); - } - private void enqueue(int status, String body) { server.enqueue( new MockResponse() From cc5ebd7de2ce77f078cfc187d0679fdeb2b01469 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 20 Sep 2026 11:25:57 +0800 Subject: [PATCH 12/12] [rest] Fix database main routing and cache invalidation --- .../docs/concepts/rest/database-versioning.md | 47 +++++- docs/static/rest-catalog-open-api.yaml | 15 +- .../apache/paimon/catalog/CachingCatalog.java | 23 +++ .../paimon/catalog/CatalogSnapshotCommit.java | 18 ++- .../apache/paimon/catalog/CatalogUtils.java | 3 +- .../org/apache/paimon/rest/RESTCatalog.java | 11 -- .../paimon/table/CatalogEnvironment.java | 34 ++++- .../paimon/catalog/CachingCatalogTest.java | 19 +++ .../catalog/CatalogSnapshotCommitTest.java | 34 +++++ .../paimon/rest/RESTCatalogReferenceTest.java | 139 +++++++++++++++++- 10 files changed, 312 insertions(+), 31 deletions(-) diff --git a/docs/docs/concepts/rest/database-versioning.md b/docs/docs/concepts/rest/database-versioning.md index c5569cdbc4bc..7057acf1bdd1 100644 --- a/docs/docs/concepts/rest/database-versioning.md +++ b/docs/docs/concepts/rest/database-versioning.md @@ -76,6 +76,8 @@ forward. Invalidate cached tables and load them again after publication. Branch-local table creation, deletion and rename require versioned namespace storage and can be deferred. Format Tables, Object Tables, external tables, views, functions and catalog permissions are outside this initial versioned-table scope. Unsupported scoped operations return `501`. +These restrictions also apply to operations on main when they would affect retained references: +the absence of a database suffix does not permit deleting storage used by a branch or tag. ## Branch management @@ -147,8 +149,9 @@ that tag, then forward that branch. Forward extends table fast-forward to the database's tables. It publishes source versions on main and can replace target changes; it does not preserve independently changed target tables using -three-way conflict resolution. The first fixed-table server requires matching membership and a -snapshot for each source table, as native table fast-forward requires a populated source. An empty +three-way conflict resolution. The first fixed-table server requires matching membership, including +both logical names and table identities, and a snapshot for each source table, as native table +fast-forward requires a populated source. An empty source table is a `400`; namespace changes the server cannot handle are a `501`. The server validates all tables before starting publication. Source `main` is invalid. @@ -292,6 +295,32 @@ virtual database must never drop its physical database. Create, delete and forwa `/databases/training/branches` and `/databases/training/tags` instead. This does not prevent ordinary create/alter/drop **table** operations from modifying membership or metadata in a writable branch. +The initial server rejects physical `DROP DATABASE` with `400` while any non-main database branch +or database tag exists, even if main has no tables. Remove those references before dropping the +database. The existing database deletion endpoint does not implicitly cascade through references. + +### Table creation, alteration and deletion + +Table operations keep their existing request and response structures. Namespace changes require +server-side versioned membership; accepting a suffix in the client does not imply that the server +implements them. A fixed-table server returns `501` for unsupported namespace changes. + +| Operation | Required server behavior | +| --- | --- | +| Create a table on a branch | Allocate a new table identity and storage, then add its logical name only to that branch after metadata is ready. Do not expose it on main as a side effect of physical creation. | +| Create a table on main after branching | Add it only to main. Existing branches and tags retain their own membership; a later forward can fail with `501` because the table sets differ. | +| Alter a table's schema or properties | Update the selected branch's backing table and recorded state. Other branches and existing tags retain their own definitions. | +| Drop a table on any branch, including main | Remove only that branch's membership entry. Keep metadata and data required by other branches or tags; do not recursively delete the shared table path. Return `501` if the server cannot preserve those references. | +| Recreate a dropped table with the same name | Allocate a new table identity. A same-name table retained on another branch is a different table and does not satisfy fixed-table forward validation. | +| Create, alter or drop through a tag | Return `403`; tag membership and table definitions are immutable. | + +For example, if main and experiment initially contain `features` and `labels`, creating +`training$branch_experiment.samples` adds `samples` only to experiment. Creating `training.metrics` +later adds `metrics` only to main. The fixed-table forward operation cannot publish these different +table sets; it rejects the operation before changing any target table. + +### Selector validation + The markers `$branch_` and `$tag_` are case-sensitive reserved syntax. The base database must be nonblank, and the reference follows the name rules above. Missing names, multiple selectors, or invalid reference names are rejected rather than interpreted as literal database names. Other @@ -300,8 +329,9 @@ any pre-existing physical database names containing the reserved markers before lookup must not switch between literal and reference meanings based on which object exists. Caller-supplied table branch suffixes cannot be combined with a database selector. For example, -`training$branch_a.features$branch_b` is rejected. Storage commits can supply a physical table branch -internally; RESTCatalog removes that internal table suffix while preserving the database selector. +`training$branch_a.features$branch_b` is rejected. REST storage commits preserve the original logical +table identifier, including when bare main is mapped to a different physical backing branch after +forward. Explicit Table branch identifiers on an unsuffixed database retain their table selector. ### Branch and tag behavior @@ -356,8 +386,10 @@ restCatalog.getDatabase("training$tag_train_v1"); the full database name through serialization and in table loaders; no extra reference fields are stored in RESTCatalog or RESTCatalogLoader. Subsequent snapshot reads, schema changes, commits, auth and token requests carry the same database name. Caches keyed by full table identifiers distinguish branches and tags. The two main aliases -(`training` and `training$branch_main`) refer to the same state; mutations and forward must invalidate -both aliases. A repeated cached getTable call is not a reload. +(`training` and `training$branch_main`) refer to the same state. The REST catalog cache invalidates +both aliases when a table is altered, dropped or explicitly invalidated through either name. +After forward, invalidate each affected main table in every client cache before loading it again; +invalidating either main alias clears both. A repeated cached getTable call is not a reload. SQL clients can pass the selector as a quoted database name, using their ordinary identifier quoting rules. For example: @@ -477,7 +509,8 @@ before deleting a backing branch. Keep the data files referenced by every retain ### Execute forward -1. Resolve the source branch and main. Validate the whole fixed-table membership and source snapshots. +1. Resolve the source branch and main. Validate the whole fixed-table membership, including logical + names and table identities, and all source snapshots before changing target state. 2. Resolve each source table version and prepare the corresponding main table state using native table snapshot/schema mechanisms. 3. Preserve database tags before applying native fastForward: that operation can remove target diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 9a43dbee8d0a..223a07eb8459 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -188,7 +188,9 @@ paths: $ref: "#/components/responses/ServerErrorResponse" description: >- Database reference suffixes are not allowed for database mutation. Use the /branches and /tags management - endpoints with the physical database name to manage references. + endpoints with the physical database name to manage references. The initial server rejects physical + database deletion with 400 while any non-main database branch or database tag exists, even + when main has no tables. Deletion does not implicitly cascade through references. post: tags: - database @@ -361,8 +363,9 @@ paths: and return success without a body. A tag is not a valid source. This publishes source table versions on main; it does not combine target-only writes or perform three-way conflict resolution. Source must be a non-main branch. Pause writers and reload tables after success. - For the fixed-table MVP, validate the same table membership and a snapshot for every source - table before forwarding. Unsupported namespace changes return 501. Preserve retained tags + For the fixed-table MVP, validate matching logical table names and table identities, and a + snapshot for every source table, before changing target state. Same-name tables recreated + with different identities do not match. Unsupported namespace changes return 501. Preserve retained tags and keep source and main independently writable. No public multi-table transaction or atomic read view is promised; a success means all planned table updates completed. operationId: forwardDatabaseBranch @@ -699,6 +702,9 @@ paths: suffix. Apply this operation to the selected branch; a tag returns 403. Identifiers in the body must retain the full database name including its suffix and agree with the path. Table IDs must match the resolved table. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. + Create a new table identity and add its name only to the selected branch. New main tables + do not appear in existing branches or tags. Recreating a dropped name allocates a new + table identity. A fixed-table server returns 501 for unsupported namespace changes. /v1/{prefix}/databases/{database}/table-details: get: tags: @@ -958,6 +964,9 @@ paths: suffix. Apply this operation to the selected branch; a tag returns 403. Identifiers in the body must retain the full database name including its suffix and agree with the path. Table IDs must match the resolved table. Missing references never fall back to the base database. A missing branch or tag of the selected type returns 404. + Remove the table only from the selected branch membership, including when the database + has no suffix and selects main. Preserve storage and metadata referenced by any other + branch or tag. Return 501 if the server cannot preserve those references. /v1/{prefix}/tables/rename: post: tags: diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index 7fce5edf0ef4..b57c9532c894 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -25,6 +25,10 @@ import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionStatistics; +import org.apache.paimon.rest.DatabaseIdentifier; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.RESTCatalog; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.FileStoreTable; @@ -46,6 +50,7 @@ import java.util.Map; import java.util.Optional; +import static org.apache.paimon.catalog.Identifier.DEFAULT_MAIN_BRANCH; import static org.apache.paimon.options.CatalogOptions.CACHE_DV_MAX_NUM; import static org.apache.paimon.options.CatalogOptions.CACHE_ENABLED; import static org.apache.paimon.options.CatalogOptions.CACHE_EXPIRE_AFTER_ACCESS; @@ -385,6 +390,24 @@ public void alterPartitions(Identifier identifier, List par @Override public void invalidateTable(Identifier identifier) { + invalidateTableCache(identifier); + if (DelegateCatalog.rootCatalog(wrapped) instanceof RESTCatalog) { + DatabaseIdentifier database = DatabaseIdentifier.parse(identifier.getDatabaseName()); + DatabaseReference reference = database.getReference(); + String alias = null; + if (reference == null) { + alias = database.getDatabaseName() + "$branch_" + DEFAULT_MAIN_BRANCH; + } else if (reference.getType() == DatabaseReferenceType.BRANCH + && DEFAULT_MAIN_BRANCH.equals(reference.getName())) { + alias = database.getDatabaseName(); + } + if (alias != null) { + invalidateTableCache(Identifier.create(alias, identifier.getObjectName())); + } + } + } + + private void invalidateTableCache(Identifier identifier) { tableCache.invalidate(identifier); if (partitionCache != null) { partitionCache.invalidate(identifier); diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogSnapshotCommit.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogSnapshotCommit.java index fc50e33b24a0..f6ce3220003d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogSnapshotCommit.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogSnapshotCommit.java @@ -31,11 +31,21 @@ public class CatalogSnapshotCommit implements SnapshotCommit { private final Catalog catalog; private final Identifier identifier; @Nullable private final String uuid; + @Nullable private final String storageBranch; public CatalogSnapshotCommit(Catalog catalog, Identifier identifier, @Nullable String uuid) { + this(catalog, identifier, uuid, null); + } + + public CatalogSnapshotCommit( + Catalog catalog, + Identifier identifier, + @Nullable String uuid, + @Nullable String storageBranch) { this.catalog = catalog; this.identifier = identifier; this.uuid = uuid; + this.storageBranch = storageBranch; } @Override @@ -45,8 +55,14 @@ public boolean commit( String branch, List statistics) throws Exception { + // REST resolves the original logical identifier to its physical storage branch. Keep + // that identifier even when main is backed by a different branch after publication. + // An explicit switch away from the loaded storage branch still selects a table branch. Identifier newIdentifier = - new Identifier(identifier.getDatabaseName(), identifier.getTableName(), branch); + storageBranch != null && storageBranch.equals(branch) + ? identifier + : new Identifier( + identifier.getDatabaseName(), identifier.getTableName(), branch); return catalog.commitSnapshot(newIdentifier, uuid, baseSnapshotUuid, snapshot, statistics); } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index 916f63350c7b..a3746e8dd938 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -394,7 +394,8 @@ public static Table loadTable( isRestCatalog ? null : lockContext, catalogContext, catalog.supportsVersionManagement(), - catalog.supportsPartitionModification()); + catalog.supportsPartitionModification(), + isRestCatalog ? options.branch() : null); Path path = new Path(schema.options().get(PATH.key())); FileStoreTable table = FileStoreTableFactory.create(dataFileIO.apply(path), path, schema, catalogEnv); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 8c8abea95c87..4f683864a38d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -534,17 +534,6 @@ public boolean commitSnapshot( Snapshot snapshot, List statistics) throws TableNotExistException { - // CatalogSnapshotCommit supplies the physical storage branch. The database suffix - // already selects the write target; keep the logical table name on the wire. - if (DatabaseIdentifier.parse(identifier.getDatabaseName()).getReference() != null - && identifier.getBranchName() != null) { - identifier = - new Identifier( - identifier.getDatabaseName(), - identifier.getTableName(), - null, - identifier.getSystemTableName()); - } try { return api.commitSnapshot( identifier, tableUuid, baseSnapshotUuid, snapshot, statistics); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java index 66979ba811f2..fd9e062cc653 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java @@ -44,6 +44,7 @@ import javax.annotation.Nullable; import java.io.Serializable; +import java.util.Objects; import java.util.Optional; import java.util.function.LongConsumer; @@ -63,6 +64,9 @@ public class CatalogEnvironment implements Serializable { @Nullable private final CatalogContext catalogContext; private final boolean supportsVersionManagement; private final boolean supportsPartitionModification; + // Physical branch resolved when loading a REST table; retained across option + // copies/serialization. + @Nullable private final String storageBranch; public CatalogEnvironment( @Nullable Identifier identifier, @@ -73,6 +77,28 @@ public CatalogEnvironment( @Nullable CatalogContext catalogContext, boolean supportsVersionManagement, boolean supportsPartitionModification) { + this( + identifier, + uuid, + catalogLoader, + lockFactory, + lockContext, + catalogContext, + supportsVersionManagement, + supportsPartitionModification, + null); + } + + public CatalogEnvironment( + @Nullable Identifier identifier, + @Nullable String uuid, + @Nullable CatalogLoader catalogLoader, + @Nullable CatalogLockFactory lockFactory, + @Nullable CatalogLockContext lockContext, + @Nullable CatalogContext catalogContext, + boolean supportsVersionManagement, + boolean supportsPartitionModification, + @Nullable String storageBranch) { this.identifier = identifier; this.uuid = uuid; this.catalogLoader = catalogLoader; @@ -81,6 +107,7 @@ public CatalogEnvironment( this.catalogContext = catalogContext; this.supportsVersionManagement = supportsVersionManagement; this.supportsPartitionModification = supportsPartitionModification; + this.storageBranch = storageBranch; } public static CatalogEnvironment empty() { @@ -135,7 +162,9 @@ public SchemaModification schemaModification() { public SnapshotCommit snapshotCommit(SnapshotManager snapshotManager) { SnapshotCommit snapshotCommit; if (catalogLoader != null && supportsVersionManagement) { - snapshotCommit = new CatalogSnapshotCommit(catalogLoader.load(), identifier, uuid); + snapshotCommit = + new CatalogSnapshotCommit( + catalogLoader.load(), identifier, uuid, storageBranch); } else { Lock lock = Optional.ofNullable(lockFactory) @@ -253,7 +282,8 @@ public CatalogEnvironment copy(Identifier identifier) { lockContext, catalogContext, supportsVersionManagement, - supportsPartitionModification); + supportsPartitionModification, + Objects.equals(this.identifier, identifier) ? storageBranch : null); } public TableQueryAuth tableQueryAuth(CoreOptions options) { diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java index 3de7bbf73f16..fbea25196b71 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java @@ -103,6 +103,25 @@ public void testListDatabasesWhenNoDatabases() { assertThat(databases).contains("db"); } + @Test + public void testReferenceSuffixIsLiteralOutsideRestCatalog() throws Exception { + CachingCatalog cached = new CachingCatalog(catalog, new Options()); + String literalDatabase = "db$branch_main"; + cached.createDatabase(literalDatabase, false); + Identifier main = Identifier.create("db", "features"); + Identifier literal = Identifier.create(literalDatabase, "features"); + Schema schema = Schema.newBuilder().column("id", DataTypes.INT()).build(); + cached.createTable(main, schema, false); + cached.createTable(literal, schema, false); + Table literalTable = cached.getTable(literal); + + cached.alterTable(main, SchemaChange.addColumn("added", DataTypes.STRING()), false); + + assertThat(cached.getTable(main).rowType().getFieldNames()).containsExactly("id", "added"); + assertThat(cached.getTable(literal)).isSameAs(literalTable); + assertThat(literalTable.rowType().getFieldNames()).containsExactly("id"); + } + @Test public void testInvalidateWhenDatabaseIsAltered() throws Exception { Catalog mockcatalog = Mockito.mock(Catalog.class); diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogSnapshotCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogSnapshotCommitTest.java index 1b5cfeb1cceb..b63b6a6f6a38 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogSnapshotCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogSnapshotCommitTest.java @@ -19,6 +19,8 @@ package org.apache.paimon.catalog; import org.apache.paimon.Snapshot; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTCatalog; import org.apache.paimon.utils.SnapshotManagerTest; import org.junit.jupiter.api.Test; @@ -59,4 +61,36 @@ public void testCommitForwardsBaseSnapshotUuid() throws Exception { snapshot, Collections.emptyList()); } + + @Test + public void testWrappedRestCatalogKeepsLogicalIdentifier() throws Exception { + RESTCatalog rest = mock(RESTCatalog.class); + Catalog catalog = new CachingCatalog(rest, new Options()); + Identifier identifier = Identifier.create("database", "table"); + Snapshot snapshot = SnapshotManagerTest.createSnapshotWithMillis(2L, 1000L); + when(rest.commitSnapshot( + identifier, + "table-uuid", + "base-snapshot-uuid", + snapshot, + Collections.emptyList())) + .thenReturn(true); + + CatalogSnapshotCommit commit = + new CatalogSnapshotCommit(catalog, identifier, "table-uuid", "physical-main"); + assertThat( + commit.commit( + "base-snapshot-uuid", + snapshot, + "physical-main", + Collections.emptyList())) + .isTrue(); + verify(rest) + .commitSnapshot( + identifier, + "table-uuid", + "base-snapshot-uuid", + snapshot, + Collections.emptyList()); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java index de67c179da57..3aec2ce1d7f7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest; import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.CachingCatalog; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.Identifier; @@ -152,12 +153,14 @@ void testTableAndSerializedLoaderKeepReference(String reference) throws Exceptio assertThat(server.getRequestCount()).isEqualTo(6); } - @Test - void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { - Identifier selected = Identifier.create(DATABASE + "$branch_experiment", "features"); + @ParameterizedTest + @ValueSource(strings = {"", "$branch_main", "$branch_experiment"}) + void testStorageCommitUsesLogicalTableAndExistingBody(String reference) throws Exception { + Identifier selected = Identifier.create(DATABASE + reference, "features"); + String scope = DATABASE_PATH + reference.replace("$", "%24"); enqueue(200, tableResponse(selected.getDatabaseName(), "physical-experiment", 2)); FileStoreTable table = InstantiationUtil.clone((FileStoreTable) catalog.getTable(selected)); - takeRequest("GET", DATABASE_PATH + "%24branch_experiment/tables/features"); + takeRequest("GET", scope + "/tables/features"); Snapshot snapshot = Snapshot.fromJson(SNAPSHOT_JSON); enqueue(200, "{\"success\":true}"); @@ -171,8 +174,7 @@ void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { emptyList())) .isTrue(); } - RecordedRequest request = - takeRequest("POST", DATABASE_PATH + "%24branch_experiment/tables/features/commit"); + RecordedRequest request = takeRequest("POST", scope + "/tables/features/commit"); CommitTableRequest body = RESTApi.fromJson(request.getBody().readUtf8(), CommitTableRequest.class); assertThat(body.getTableId()).isEqualTo("table-id"); @@ -181,6 +183,131 @@ void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { assertThat(body.getStatistics()).isEmpty(); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testStorageCommitPreservesExplicitTableBranch(boolean dynamicBranch) throws Exception { + Identifier selected = + Identifier.create(DATABASE, dynamicBranch ? "features" : "features$branch_dev"); + enqueue(200, tableResponse(DATABASE, dynamicBranch ? "physical-main" : "dev", 2)); + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) catalog.getTable(selected)); + takeRequest( + "GET", + DATABASE_PATH + "/tables/" + RESTUtil.encodeString(selected.getObjectName())); + if (dynamicBranch) { + table = + InstantiationUtil.clone( + table.copy(java.util.Collections.singletonMap(BRANCH.key(), "dev"))); + } + + enqueue(200, "{\"success\":true}"); + try (SnapshotCommit commit = + table.catalogEnvironment().snapshotCommit(table.snapshotManager())) { + assertThat( + commit.commit( + "snapshot-6", + Snapshot.fromJson(SNAPSHOT_JSON), + table.snapshotManager().branch(), + emptyList())) + .isTrue(); + } + takeRequest("POST", DATABASE_PATH + "/tables/features%24branch_dev/commit"); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testMainAliasesInvalidateTogether(boolean explicitMain) throws Exception { + CachingCatalog cached = new CachingCatalog(catalog, new Options()); + Identifier selected = + Identifier.create(DATABASE + (explicitMain ? "$branch_main" : ""), "features"); + Identifier other = + Identifier.create(DATABASE + (explicitMain ? "" : "$branch_main"), "features"); + String selectedPath = + DATABASE_PATH + (explicitMain ? "%24branch_main" : "") + "/tables/features"; + for (Identifier identifier : new Identifier[] {selected, other}) { + enqueue(200, tableResponse(identifier.getDatabaseName(), "physical-main", 2)); + cached.getTable(identifier); + takeRequest( + "GET", + new ResourcePaths("catalog/id") + .table(identifier.getDatabaseName(), identifier.getObjectName())); + } + + Identifier dev = Identifier.create(DATABASE + "$branch_dev", "features"); + Identifier tag = Identifier.create(DATABASE + "$tag_baseline", "features"); + enqueue(200, tableResponse(dev.getDatabaseName(), "physical-dev", 2)); + FileStoreTable devTable = (FileStoreTable) cached.getTable(dev); + takeRequest("GET", DATABASE_PATH + "%24branch_dev/tables/features"); + enqueue(200, tableResponse(tag.getDatabaseName(), "frozen-baseline", 2)); + FileStoreTable tagTable = (FileStoreTable) cached.getTable(tag); + takeRequest("GET", DATABASE_PATH + "%24tag_baseline/tables/features"); + + enqueue(200, ""); + cached.alterTable( + selected, + java.util.Collections.singletonList(SchemaChange.setOption("comment", "updated")), + false); + takeRequest("POST", selectedPath); + assertMainAliasesReload(cached, selected, other, 3); + + // Forward is followed by explicit invalidation, which must refresh both main aliases. + cached.invalidateTable(selected); + assertMainAliasesReload(cached, selected, other, 4); + + enqueue(200, ""); + cached.dropTable(selected, false); + takeRequest("DELETE", selectedPath); + for (Identifier identifier : new Identifier[] {selected, other}) { + enqueue(404, "{\"code\":404,\"resourceType\":\"TABLE\",\"message\":\"missing\"}"); + assertThatThrownBy(() -> cached.getTable(identifier)) + .isInstanceOf(Catalog.TableNotExistException.class); + takeRequest( + "GET", + new ResourcePaths("catalog/id") + .table(identifier.getDatabaseName(), identifier.getObjectName())); + } + assertThat(cached.getTable(dev)).isSameAs(devTable); + assertThat(cached.getTable(tag)).isSameAs(tagTable); + } + + @Test + void testStorageCommitAfterSwitchingBranchAndCopyingBack() throws Exception { + enqueue(200, tableResponse("main")); + FileStoreTable main = (FileStoreTable) catalog.getTable(TABLE); + takeRequest("GET", DATABASE_PATH + "/tables/features"); + main.schemaManager().copyWithBranch("dev").createTable(schema("dev")); + FileStoreTable copied = + InstantiationUtil.clone( + main.switchToBranch("dev") + .copy(java.util.Collections.singletonMap(BRANCH.key(), "main"))); + + enqueue(200, "{\"success\":true}"); + try (SnapshotCommit commit = + copied.catalogEnvironment().snapshotCommit(copied.snapshotManager())) { + assertThat( + commit.commit( + "snapshot-6", + Snapshot.fromJson(SNAPSHOT_JSON), + copied.snapshotManager().branch(), + emptyList())) + .isTrue(); + } + takeRequest("POST", DATABASE_PATH + "/tables/features/commit"); + } + + private void assertMainAliasesReload( + CachingCatalog cached, Identifier selected, Identifier other, long schemaId) + throws Exception { + for (Identifier identifier : new Identifier[] {selected, other}) { + enqueue(200, tableResponse(identifier.getDatabaseName(), "physical-main", schemaId)); + assertThat(((FileStoreTable) cached.getTable(identifier)).schema().id()) + .isEqualTo(schemaId); + takeRequest( + "GET", + new ResourcePaths("catalog/id") + .table(identifier.getDatabaseName(), identifier.getObjectName())); + } + } + @Test void testReadFollowUpsAndPaginationReuseProtocol() throws Exception { RESTApi api = catalog.api();