diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml
index 124001ca7b5..cfb6e2ca31b 100644
--- a/fluss-kafka/pom.xml
+++ b/fluss-kafka/pom.xml
@@ -64,6 +64,13 @@
+
+ org.apache.curator
+ curator-test
+ ${curator.version}
+ test
+
+
org.apache.fluss
fluss-test-utils
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java
index 5e7551a9af7..29bdc745ca9 100644
--- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java
@@ -31,17 +31,20 @@
public class KafkaChannelInitializer extends NettyChannelInitializer {
private final RequestChannel[] requestChannels;
+ private final String listenerName;
private final int maxRequestSize;
private final LengthFieldPrepender prepender = new LengthFieldPrepender(4);
private final boolean preferHeap;
public KafkaChannelInitializer(
RequestChannel[] requestChannels,
+ String listenerName,
long maxIdleTimeSeconds,
int maxRequestSize,
boolean preferHeap) {
super(maxIdleTimeSeconds);
this.requestChannels = requestChannels;
+ this.listenerName = listenerName;
this.maxRequestSize = maxRequestSize;
this.preferHeap = preferHeap;
}
@@ -53,6 +56,6 @@ protected void initChannel(SocketChannel ch) throws Exception {
ch.pipeline().addLast(prepender);
addFrameDecoder(ch, maxRequestSize, 4, preferHeap);
ch.pipeline().addLast("flowController", new FlowControlHandler());
- ch.pipeline().addLast(new KafkaCommandDecoder(requestChannels));
+ ch.pipeline().addLast(new KafkaCommandDecoder(requestChannels, listenerName));
}
}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java
index 43a0533b2d3..637a1aaa1fe 100644
--- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java
@@ -27,6 +27,7 @@
import org.apache.fluss.utils.MathUtils;
import org.apache.kafka.common.errors.LeaderNotAvailableException;
+import org.apache.kafka.common.message.ApiVersionsRequestData;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.AbstractResponse;
@@ -55,6 +56,7 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler {
private final RequestChannel[] requestChannels;
private final int numChannels;
+ private final String listenerName;
// Need to use a Queue to store the inflight responses, because Kafka clients require the
// responses to be sent in order.
@@ -65,18 +67,18 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler {
protected volatile ChannelHandlerContext ctx;
protected SocketAddress remoteAddress;
- public KafkaCommandDecoder(RequestChannel[] requestChannels) {
+ public KafkaCommandDecoder(RequestChannel[] requestChannels, String listenerName) {
super(false);
this.requestChannels = requestChannels;
this.numChannels = requestChannels.length;
+ this.listenerName = listenerName;
}
@Override
public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Exception {
CompletableFuture future = new CompletableFuture<>();
- boolean needRelease = false;
try {
- KafkaRequest request = parseRequest(ctx, future, buffer);
+ KafkaRequest request = parseRequest(ctx, future, buffer, listenerName);
inflightResponses.addLast(request);
future.whenCompleteAsync((r, t) -> sendResponse(ctx), ctx.executor());
int channelIndex =
@@ -86,16 +88,15 @@ public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Excep
if (!isActive.get()) {
LOG.warn("Received a request on an inactive channel: {}", remoteAddress);
request.fail(new LeaderNotAvailableException("Channel is inactive"));
- needRelease = true;
}
} catch (Throwable t) {
- needRelease = true;
LOG.error("Error handling request", t);
future.completeExceptionally(t);
} finally {
- if (needRelease) {
- ReferenceCountUtil.release(buffer);
- }
+ // KafkaRequest retains the buffer to transfer ownership to request processing. Release
+ // the decoder's ownership on every path. KafkaRequest.releaseBuffer() is idempotent
+ // because worker cleanup and response completion can both release that ownership.
+ ReferenceCountUtil.release(buffer);
}
}
@@ -184,19 +185,39 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E
}
private static KafkaRequest parseRequest(
- ChannelHandlerContext ctx, CompletableFuture future, ByteBuf buffer) {
+ ChannelHandlerContext ctx,
+ CompletableFuture future,
+ ByteBuf buffer,
+ String listenerName) {
ByteBuffer nioBuffer = buffer.nioBuffer();
RequestHeader header = RequestHeader.parse(nioBuffer);
if (isUnsupportedApiVersionRequest(header)) {
ApiVersionsRequest request =
- new ApiVersionsRequest.Builder(header.apiVersion()).build();
+ new ApiVersionsRequest(
+ new ApiVersionsRequestData(),
+ API_VERSIONS.oldestVersion(),
+ header.apiVersion());
return new KafkaRequest(
- API_VERSIONS, header.apiVersion(), header, request, buffer, ctx, future);
+ API_VERSIONS,
+ header.apiVersion(),
+ header,
+ request,
+ listenerName,
+ buffer,
+ ctx,
+ future);
}
RequestAndSize request =
AbstractRequest.parseRequest(header.apiKey(), header.apiVersion(), nioBuffer);
return new KafkaRequest(
- header.apiKey(), header.apiVersion(), header, request.request, buffer, ctx, future);
+ header.apiKey(),
+ header.apiVersion(),
+ header,
+ request.request,
+ listenerName,
+ buffer,
+ ctx,
+ future);
}
private static boolean isUnsupportedApiVersionRequest(RequestHeader header) {
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java
index d92ba5e68fc..59ce1b2a0ed 100644
--- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java
@@ -53,6 +53,7 @@ public ChannelHandler createChannelHandler(
RequestChannel[] requestChannels, String listenerName) {
return new KafkaChannelInitializer(
requestChannels,
+ listenerName,
conf.get(ConfigOptions.KAFKA_CONNECTION_MAX_IDLE_TIME).getSeconds(),
(int) conf.get(ConfigOptions.NETTY_SERVER_MAX_REQUEST_SIZE).getBytes(),
conf.getBoolean(ConfigOptions.NETTY_CLIENT_ALLOCATOR_HEAP_BUFFER_FIRST));
@@ -66,6 +67,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) {
+ service.getClass().getSimpleName());
}
TabletServerGateway gateway = (TabletServerGateway) service;
- return new KafkaRequestHandler(gateway);
+ return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE));
}
}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java
index 25e409a7455..0d2799a7a18 100644
--- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java
@@ -35,6 +35,7 @@
import java.nio.ByteBuffer;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
/** Represents a request received from Kafka protocol channel. */
@@ -46,10 +47,12 @@ public class KafkaRequest implements RpcRequest {
private final long requestId = ID_GENERATOR.getAndIncrement();
private final RequestHeader header;
private final AbstractRequest request;
+ private final String listenerName;
private final ByteBuf buffer;
private final ChannelHandlerContext ctx;
private final long startTimeMs;
private final CompletableFuture future;
+ private final AtomicBoolean bufferReleased = new AtomicBoolean();
private volatile boolean cancelled = false;
protected KafkaRequest(
@@ -60,10 +63,23 @@ protected KafkaRequest(
ByteBuf buffer,
ChannelHandlerContext ctx,
CompletableFuture future) {
+ this(apiKey, apiVersion, header, request, "UNKNOWN", buffer, ctx, future);
+ }
+
+ protected KafkaRequest(
+ ApiKeys apiKey,
+ short apiVersion,
+ RequestHeader header,
+ AbstractRequest request,
+ String listenerName,
+ ByteBuf buffer,
+ ChannelHandlerContext ctx,
+ CompletableFuture future) {
this.apiKey = apiKey;
this.apiVersion = apiVersion;
this.header = header;
this.request = request;
+ this.listenerName = listenerName;
this.buffer = buffer.retain();
this.ctx = ctx;
this.startTimeMs = System.currentTimeMillis();
@@ -77,7 +93,9 @@ public RequestType getRequestType() {
@Override
public void releaseBuffer() {
- ReferenceCountUtil.safeRelease(buffer);
+ if (bufferReleased.compareAndSet(false, true)) {
+ ReferenceCountUtil.safeRelease(buffer);
+ }
}
public ApiKeys apiKey() {
@@ -100,6 +118,10 @@ public T request() {
return (T) request;
}
+ public String listenerName() {
+ return listenerName;
+ }
+
public ChannelHandlerContext ctx() {
return ctx;
}
@@ -149,12 +171,17 @@ private ByteBuf serialize(AbstractResponse response) {
int headerSize = headerData.size(cache, headerVersion);
ApiMessage apiMessage = response.data();
int messageSize = apiMessage.size(cache, apiVersion);
- final ByteBuf buffer = ctx.alloc().buffer(headerSize + messageSize);
- buffer.writerIndex(headerSize + messageSize);
- final ByteBuffer nioBuffer = buffer.nioBuffer();
- final ByteBufferAccessor writable = new ByteBufferAccessor(nioBuffer);
- headerData.write(writable, cache, headerVersion);
- apiMessage.write(writable, cache, apiVersion);
- return buffer;
+ final ByteBuf responseBuffer = ctx.alloc().buffer(headerSize + messageSize);
+ try {
+ responseBuffer.writerIndex(headerSize + messageSize);
+ final ByteBuffer nioBuffer = responseBuffer.nioBuffer();
+ final ByteBufferAccessor writable = new ByteBufferAccessor(nioBuffer);
+ headerData.write(writable, cache, headerVersion);
+ apiMessage.write(writable, cache, apiVersion);
+ return responseBuffer;
+ } catch (Throwable t) {
+ ReferenceCountUtil.safeRelease(responseBuffer);
+ throw t;
+ }
}
}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java
new file mode 100644
index 00000000000..e75a20babcc
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.shaded.netty4.io.netty.channel.Channel;
+
+import org.apache.kafka.common.protocol.ApiKeys;
+
+import java.net.SocketAddress;
+
+/** Immutable wire-level context made available to Kafka API handlers. */
+@Internal
+public final class KafkaRequestContext {
+
+ private final int correlationId;
+ private final String clientId;
+ private final ApiKeys apiKey;
+ private final short apiVersion;
+ private final String listenerName;
+ private final SocketAddress localAddress;
+ private final SocketAddress remoteAddress;
+ private final long receivedTimeMs;
+
+ private KafkaRequestContext(KafkaRequest request) {
+ this.correlationId = request.header().correlationId();
+ this.clientId = request.header().clientId();
+ this.apiKey = request.apiKey();
+ this.apiVersion = request.apiVersion();
+ this.listenerName = request.listenerName();
+ Channel channel = request.ctx().channel();
+ this.localAddress = channel == null ? null : channel.localAddress();
+ this.remoteAddress = channel == null ? null : channel.remoteAddress();
+ this.receivedTimeMs = request.startTimeMs();
+ }
+
+ /** Creates a context from a network request. */
+ public static KafkaRequestContext fromRequest(KafkaRequest request) {
+ return new KafkaRequestContext(request);
+ }
+
+ /** Returns the request correlation ID. */
+ public int correlationId() {
+ return correlationId;
+ }
+
+ /** Returns the client ID, or {@code null} when the request did not provide one. */
+ public String clientId() {
+ return clientId;
+ }
+
+ /** Returns the Kafka API key. */
+ public ApiKeys apiKey() {
+ return apiKey;
+ }
+
+ /** Returns the Kafka request version. */
+ public short apiVersion() {
+ return apiVersion;
+ }
+
+ /** Returns the listener that accepted the request. */
+ public String listenerName() {
+ return listenerName;
+ }
+
+ /** Returns the local socket address. */
+ public SocketAddress localAddress() {
+ return localAddress;
+ }
+
+ /** Returns the remote socket address. */
+ public SocketAddress remoteAddress() {
+ return remoteAddress;
+ }
+
+ /** Returns the wall-clock time at which the request was received. */
+ public long receivedTimeMs() {
+ return receivedTimeMs;
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java
index 73555093ff0..de561509b66 100644
--- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java
@@ -17,27 +17,37 @@
package org.apache.fluss.kafka;
+import org.apache.fluss.kafka.api.metadata.MetadataHandler;
+import org.apache.fluss.kafka.api.versions.ApiVersionsHandler;
+import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend;
+import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry;
+import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher;
+import org.apache.fluss.kafka.error.KafkaErrorMapper;
+import org.apache.fluss.rpc.RpcGatewayService;
import org.apache.fluss.rpc.gateway.TabletServerGateway;
import org.apache.fluss.rpc.netty.server.RequestHandler;
import org.apache.fluss.rpc.protocol.RequestType;
-import org.apache.kafka.common.message.ApiVersionsResponseData;
-import org.apache.kafka.common.protocol.ApiKeys;
-import org.apache.kafka.common.protocol.Errors;
-import org.apache.kafka.common.record.RecordBatch;
-import org.apache.kafka.common.requests.AbstractRequest;
-import org.apache.kafka.common.requests.AbstractResponse;
-import org.apache.kafka.common.requests.ApiVersionsResponse;
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
-/** Kafka protocol implementation for request handler. */
+/** Entry point that dispatches Kafka protocol requests to registered API handlers. */
public class KafkaRequestHandler implements RequestHandler {
- // TODO: we may need a new abstraction between TabletService and ReplicaManager to avoid
- // affecting Fluss protocol when supporting compatibility with Kafka.
- private final TabletServerGateway gateway;
-
- public KafkaRequestHandler(TabletServerGateway gateway) {
- this.gateway = gateway;
+ private final KafkaRequestDispatcher dispatcher;
+
+ /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */
+ public KafkaRequestHandler(
+ RpcGatewayService service, TabletServerGateway gateway, String kafkaDatabase) {
+ checkNotNull(service);
+ checkNotNull(gateway);
+ checkNotNull(kafkaDatabase);
+ KafkaApiRegistry registry = new KafkaApiRegistry();
+ registry.register(new ApiVersionsHandler(registry));
+ registry.register(
+ new MetadataHandler(
+ new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase)));
+ registry.freeze();
+ this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper());
}
@Override
@@ -47,200 +57,15 @@ public RequestType requestType() {
@Override
public void processRequest(KafkaRequest request) {
- // See kafka.server.KafkaApis#handle
- switch (request.apiKey()) {
- case API_VERSIONS:
- handleApiVersionsRequest(request);
- break;
- case METADATA:
- handleMetadataRequest(request);
- break;
- case PRODUCE:
- handleProducerRequest(request);
- break;
- case FIND_COORDINATOR:
- handleFindCoordinatorRequest(request);
- break;
- case LIST_OFFSETS:
- handleListOffsetRequest(request);
- break;
- case OFFSET_FETCH:
- handleOffsetFetchRequest(request);
- break;
- case OFFSET_COMMIT:
- handleOffsetCommitRequest(request);
- break;
- case FETCH:
- handleFetchRequest(request);
- break;
- case JOIN_GROUP:
- handleJoinGroupRequest(request);
- break;
- case SYNC_GROUP:
- handleSyncGroupRequest(request);
- break;
- case HEARTBEAT:
- handleHeartbeatRequest(request);
- break;
- case LEAVE_GROUP:
- handleLeaveGroupRequest(request);
- break;
- case DESCRIBE_GROUPS:
- handleDescribeGroupsRequest(request);
- break;
- case LIST_GROUPS:
- handleListGroupsRequest(request);
- break;
- case DELETE_GROUPS:
- handleDeleteGroupsRequest(request);
- break;
- case SASL_HANDSHAKE:
- handleSaslHandshakeRequest(request);
- break;
- case SASL_AUTHENTICATE:
- handleSaslAuthenticateRequest(request);
- break;
- case CREATE_TOPICS:
- handleCreateTopicsRequest(request);
- break;
- case INIT_PRODUCER_ID:
- handleInitProducerIdRequest(request);
- break;
- case ADD_PARTITIONS_TO_TXN:
- handleAddPartitionsToTxnRequest(request);
- break;
- case ADD_OFFSETS_TO_TXN:
- handleAddOffsetsToTxnRequest(request);
- break;
- case TXN_OFFSET_COMMIT:
- handleTxnOffsetCommitRequest(request);
- break;
- case END_TXN:
- handleEndTxnRequest(request);
- break;
- case WRITE_TXN_MARKERS:
- handleWriteTxnMarkersRequest(request);
- break;
- case DESCRIBE_CONFIGS:
- handleDescribeConfigsRequest(request);
- break;
- case ALTER_CONFIGS:
- handleAlterConfigsRequest(request);
- break;
- case DELETE_TOPICS:
- handleDeleteTopicsRequest(request);
- break;
- case DELETE_RECORDS:
- handleDeleteRecordsRequest(request);
- break;
- case OFFSET_DELETE:
- handleOffsetDeleteRequest(request);
- break;
- case CREATE_PARTITIONS:
- handleCreatePartitionsRequest(request);
- break;
- case DESCRIBE_CLUSTER:
- handleDescribeClusterRequest(request);
- break;
- default:
- handleUnsupportedRequest(request);
- }
- }
-
- private void handleUnsupportedRequest(KafkaRequest request) {
- String message = String.format("Unsupported request with api key %s", request.apiKey());
- AbstractRequest abstractRequest = request.request();
- AbstractResponse response =
- abstractRequest.getErrorResponse(new UnsupportedOperationException(message));
- request.complete(response);
- }
-
- void handleApiVersionsRequest(KafkaRequest request) {
- short apiVersion = request.apiVersion();
- if (!ApiKeys.API_VERSIONS.isVersionSupported(apiVersion)) {
- request.fail(Errors.UNSUPPORTED_VERSION.exception());
- return;
- }
- ApiVersionsResponseData data = new ApiVersionsResponseData();
- for (ApiKeys apiKey : ApiKeys.values()) {
- if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) {
- ApiVersionsResponseData.ApiVersion apiVersionData =
- new ApiVersionsResponseData.ApiVersion()
- .setApiKey(apiKey.id)
- .setMinVersion(apiKey.oldestVersion())
- .setMaxVersion(apiKey.latestVersion());
- if (apiKey.equals(ApiKeys.METADATA)) {
- // Not support TopicId
- short v = apiKey.latestVersion() > 11 ? 11 : apiKey.latestVersion();
- apiVersionData.setMaxVersion(v);
- } else if (apiKey.equals(ApiKeys.FETCH)) {
- // Not support TopicId
- short v = apiKey.latestVersion() > 12 ? 12 : apiKey.latestVersion();
- apiVersionData.setMaxVersion(v);
- }
- data.apiKeys().add(apiVersionData);
- }
- }
- request.complete(new ApiVersionsResponse(data));
+ dispatcher
+ .dispatch(request)
+ .whenComplete(
+ (response, failure) -> {
+ if (failure == null) {
+ request.complete(response);
+ } else {
+ request.fail(failure);
+ }
+ });
}
-
- void handleProducerRequest(KafkaRequest request) {}
-
- void handleMetadataRequest(KafkaRequest request) {}
-
- void handleFindCoordinatorRequest(KafkaRequest request) {}
-
- void handleListOffsetRequest(KafkaRequest request) {}
-
- void handleOffsetFetchRequest(KafkaRequest request) {}
-
- void handleOffsetCommitRequest(KafkaRequest request) {}
-
- void handleFetchRequest(KafkaRequest request) {}
-
- void handleJoinGroupRequest(KafkaRequest request) {}
-
- void handleSyncGroupRequest(KafkaRequest request) {}
-
- void handleHeartbeatRequest(KafkaRequest request) {}
-
- void handleLeaveGroupRequest(KafkaRequest request) {}
-
- void handleDescribeGroupsRequest(KafkaRequest request) {}
-
- void handleListGroupsRequest(KafkaRequest request) {}
-
- void handleDeleteGroupsRequest(KafkaRequest request) {}
-
- void handleSaslHandshakeRequest(KafkaRequest request) {}
-
- void handleSaslAuthenticateRequest(KafkaRequest request) {}
-
- void handleCreateTopicsRequest(KafkaRequest request) {}
-
- void handleInitProducerIdRequest(KafkaRequest request) {}
-
- void handleAddPartitionsToTxnRequest(KafkaRequest request) {}
-
- void handleAddOffsetsToTxnRequest(KafkaRequest request) {}
-
- void handleTxnOffsetCommitRequest(KafkaRequest request) {}
-
- void handleEndTxnRequest(KafkaRequest request) {}
-
- void handleWriteTxnMarkersRequest(KafkaRequest request) {}
-
- void handleDescribeConfigsRequest(KafkaRequest request) {}
-
- void handleAlterConfigsRequest(KafkaRequest request) {}
-
- void handleDeleteTopicsRequest(KafkaRequest request) {}
-
- void handleDeleteRecordsRequest(KafkaRequest request) {}
-
- void handleOffsetDeleteRequest(KafkaRequest request) {}
-
- void handleCreatePartitionsRequest(KafkaRequest request) {}
-
- void handleDescribeClusterRequest(KafkaRequest request) {}
}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java
new file mode 100644
index 00000000000..ee8bfe7a5a8
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.api.metadata;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.KafkaRequestContext;
+import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata;
+import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker;
+import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition;
+import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError;
+import org.apache.fluss.kafka.backend.metadata.KafkaMetadataBackend;
+import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery;
+import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference;
+import org.apache.fluss.kafka.dispatcher.KafkaApiHandler;
+import org.apache.fluss.kafka.dispatcher.KafkaApiSpec;
+
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.InvalidRequestException;
+import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic;
+import org.apache.kafka.common.message.MetadataResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.MetadataRequest;
+import org.apache.kafka.common.requests.MetadataResponse;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Implements Kafka Metadata versions 0 through 11 using a narrow Fluss metadata backend. */
+@Internal
+public final class MetadataHandler implements KafkaApiHandler {
+
+ private static final short MAX_SUPPORTED_VERSION = 11;
+ private static final KafkaApiSpec API_SPEC =
+ new KafkaApiSpec(
+ ApiKeys.METADATA,
+ ApiKeys.METADATA.oldestVersion(),
+ (short) Math.min(ApiKeys.METADATA.latestVersion(), MAX_SUPPORTED_VERSION),
+ true);
+
+ private final KafkaMetadataBackend backend;
+
+ /** Creates a Metadata handler. */
+ public MetadataHandler(KafkaMetadataBackend backend) {
+ this.backend = checkNotNull(backend);
+ }
+
+ @Override
+ public KafkaApiSpec apiSpec() {
+ return API_SPEC;
+ }
+
+ @Override
+ public CompletableFuture handle(
+ KafkaRequestContext context, MetadataRequest request) {
+ List validTopics = new ArrayList<>();
+ List invalidTopics = new ArrayList<>();
+ if (!request.isAllTopics()) {
+ for (MetadataRequestTopic topic : request.data().topics()) {
+ if (topic.name() == null) {
+ throw new InvalidRequestException(
+ "Topic name must be set because topic ID lookup is not supported by "
+ + "Metadata versions 10 and 11.");
+ } else if (!Topic.isValid(topic.name())) {
+ invalidTopics.add(
+ new KafkaClusterMetadata.Topic(
+ topic.name(),
+ topic.topicId(),
+ TopicError.INVALID_TOPIC,
+ Collections.emptyList()));
+ } else {
+ // Kafka added the topic ID fields in v10, but ID-based Metadata lookup was not
+ // implemented until v12. This handler intentionally stops at v11.
+ validTopics.add(new TopicReference(topic.name(), Uuid.ZERO_UUID));
+ }
+ }
+ }
+
+ KafkaMetadataQuery query =
+ new KafkaMetadataQuery(
+ request.isAllTopics(),
+ validTopics,
+ context.listenerName(),
+ clientAddress(context.remoteAddress()));
+ return backend.getMetadata(query)
+ .thenApply(
+ metadata -> {
+ List topics =
+ new ArrayList<>(metadata.topics());
+ topics.addAll(invalidTopics);
+ return toResponse(
+ request.version(),
+ new KafkaClusterMetadata(metadata.brokers(), topics));
+ });
+ }
+
+ private static MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) {
+ MetadataResponseData data =
+ new MetadataResponseData()
+ .setThrottleTimeMs(0)
+ .setControllerId(MetadataResponse.NO_CONTROLLER_ID)
+ .setClusterAuthorizedOperations(
+ MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED);
+ for (Broker broker : metadata.brokers()) {
+ MetadataResponseData.MetadataResponseBroker responseBroker =
+ new MetadataResponseData.MetadataResponseBroker()
+ .setNodeId(broker.id())
+ .setHost(broker.host())
+ .setPort(broker.port());
+ if (broker.rack() != null) {
+ responseBroker.setRack(broker.rack());
+ }
+ data.brokers().add(responseBroker);
+ }
+ for (KafkaClusterMetadata.Topic topic : metadata.topics()) {
+ MetadataResponseData.MetadataResponseTopic responseTopic =
+ new MetadataResponseData.MetadataResponseTopic()
+ .setName(topic.name())
+ .setTopicId(topic.topicId())
+ .setErrorCode(toKafkaError(topic.error()).code())
+ .setIsInternal(topic.name() != null && Topic.isInternal(topic.name()))
+ .setTopicAuthorizedOperations(
+ MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED);
+ for (Partition partition : topic.partitions()) {
+ responseTopic
+ .partitions()
+ .add(
+ new MetadataResponseData.MetadataResponsePartition()
+ .setErrorCode(
+ partition.leaderAvailable()
+ ? Errors.NONE.code()
+ : Errors.LEADER_NOT_AVAILABLE.code())
+ .setPartitionIndex(partition.partitionId())
+ .setLeaderId(partition.leaderId())
+ .setLeaderEpoch(partition.leaderEpoch())
+ .setReplicaNodes(partition.replicas())
+ .setIsrNodes(partition.isr())
+ .setOfflineReplicas(partition.offlineReplicas()));
+ }
+ data.topics().add(responseTopic);
+ }
+ return new MetadataResponse(data, version);
+ }
+
+ private static Errors toKafkaError(TopicError error) {
+ switch (error) {
+ case NONE:
+ return Errors.NONE;
+ case UNKNOWN_TOPIC_OR_PARTITION:
+ return Errors.UNKNOWN_TOPIC_OR_PARTITION;
+ case UNKNOWN_TOPIC_ID:
+ return Errors.UNKNOWN_TOPIC_ID;
+ case INVALID_TOPIC:
+ return Errors.INVALID_TOPIC_EXCEPTION;
+ default:
+ throw new IllegalArgumentException("Unsupported metadata error " + error);
+ }
+ }
+
+ private static InetAddress clientAddress(SocketAddress remoteAddress) {
+ if (remoteAddress instanceof InetSocketAddress) {
+ return ((InetSocketAddress) remoteAddress).getAddress();
+ }
+ return null;
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java
new file mode 100644
index 00000000000..0eb14b26ff2
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.api.produce;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.KafkaRequestContext;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceResult;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult;
+import org.apache.fluss.kafka.dispatcher.KafkaApiHandler;
+import org.apache.fluss.kafka.dispatcher.KafkaApiSpec;
+
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.InvalidRequestException;
+import org.apache.kafka.common.errors.InvalidRequiredAcksException;
+import org.apache.kafka.common.errors.InvalidTopicException;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.internals.Topic;
+import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData;
+import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData;
+import org.apache.kafka.common.message.ProduceResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.record.BaseRecords;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.common.record.Records;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.ProduceRequest;
+import org.apache.kafka.common.requests.ProduceResponse;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Implements non-idempotent Kafka Produce versions 3 through 11. */
+@Internal
+public final class ProduceHandler implements KafkaApiHandler {
+
+ private static final short MIN_SUPPORTED_VERSION = 3;
+ private static final KafkaApiSpec API_SPEC =
+ new KafkaApiSpec(ApiKeys.PRODUCE, MIN_SUPPORTED_VERSION, (short) 11, true);
+
+ private final KafkaProduceBackend backend;
+
+ /** Creates a non-idempotent Produce handler. */
+ public ProduceHandler(KafkaProduceBackend backend) {
+ this.backend = checkNotNull(backend);
+ }
+
+ @Override
+ public KafkaApiSpec apiSpec() {
+ return API_SPEC;
+ }
+
+ @Override
+ public CompletableFuture handle(
+ KafkaRequestContext context, ProduceRequest request) {
+ validateRequest(request);
+ List topics = new ArrayList<>();
+ Map failures = new HashMap<>();
+ for (TopicProduceData topic : request.data().topicData()) {
+ List partitions = new ArrayList<>();
+ for (PartitionProduceData partition : topic.partitionData()) {
+ try {
+ if (!Topic.isValid(topic.name())) {
+ throw new InvalidTopicException("Invalid Kafka topic name " + topic.name());
+ }
+ if (partition.index() < 0) {
+ throw new InvalidRequestException("Negative Kafka partition ID.");
+ }
+ partitions.add(
+ new PartitionWrite(
+ partition.index(),
+ copyRecords(request.version(), partition.records())));
+ } catch (RuntimeException failure) {
+ failures.put(
+ new TopicPartition(topic.name(), partition.index()),
+ failedPartition(partition.index(), failure));
+ }
+ }
+ if (!partitions.isEmpty()) {
+ topics.add(new TopicWrite(topic.name(), partitions));
+ }
+ }
+ KafkaProduceCommand command =
+ new KafkaProduceCommand(
+ request.acks(),
+ request.timeout(),
+ topics,
+ context.listenerName(),
+ clientAddress(context.remoteAddress()));
+ CompletableFuture result;
+ try {
+ result =
+ topics.isEmpty()
+ ? CompletableFuture.completedFuture(
+ new KafkaProduceResult(Collections.emptyList()))
+ : checkNotNull(backend.write(command));
+ } catch (RuntimeException failure) {
+ result = new CompletableFuture<>();
+ result.completeExceptionally(failure);
+ }
+ return result.handle(
+ (written, failure) -> {
+ Map results = new HashMap<>(failures);
+ if (failure == null && written != null) {
+ for (TopicResult topic : written.topics()) {
+ for (PartitionResult partition : topic.partitions()) {
+ results.putIfAbsent(
+ new TopicPartition(
+ topic.topicName(), partition.partitionId()),
+ partition);
+ }
+ }
+ }
+ List ordered = new ArrayList<>();
+ for (TopicProduceData topic : request.data().topicData()) {
+ List partitions = new ArrayList<>();
+ for (PartitionProduceData partition : topic.partitionData()) {
+ TopicPartition key =
+ new TopicPartition(topic.name(), partition.index());
+ PartitionResult value = results.get(key);
+ if (value == null) {
+ value =
+ failedPartition(
+ partition.index(),
+ failure == null
+ ? new IllegalStateException(
+ "Produce backend omitted this partition.")
+ : failure);
+ }
+ partitions.add(value);
+ }
+ ordered.add(new TopicResult(topic.name(), partitions));
+ }
+ return toResponse(new KafkaProduceResult(ordered));
+ });
+ }
+
+ private static PartitionResult failedPartition(int partitionId, Throwable failure) {
+ while (failure instanceof CompletionException && failure.getCause() != null) {
+ failure = failure.getCause();
+ }
+ return new PartitionResult(
+ partitionId, Errors.forException(failure), -1L, failure.getMessage());
+ }
+
+ private static void validateRequest(ProduceRequest request) {
+ Set names = new HashSet<>();
+ for (TopicProduceData topic : request.data().topicData()) {
+ if (!names.add(topic.name())) {
+ throw new InvalidRequestException("Duplicate Kafka topic in Produce request.");
+ }
+ Set partitions = new HashSet<>();
+ for (PartitionProduceData partition : topic.partitionData()) {
+ if (!partitions.add(partition.index())) {
+ throw new InvalidRequestException(
+ "Duplicate Kafka partition in Produce request.");
+ }
+ }
+ }
+ if (request.transactionalId() != null) {
+ throw new InvalidRequestException(
+ "Transactional Produce is not supported by the Fluss Kafka compatibility layer.");
+ }
+ if (request.acks() != -1 && request.acks() != 0 && request.acks() != 1) {
+ throw new InvalidRequiredAcksException("Invalid required acks " + request.acks());
+ }
+ }
+
+ private static List copyRecords(
+ short version, BaseRecords baseRecords) {
+ if (!(baseRecords instanceof Records)) {
+ throw new InvalidRequestException("Unsupported Kafka records representation.");
+ }
+ ProduceRequest.validateRecords(version, baseRecords);
+ Records records = (Records) baseRecords;
+ if (records.sizeInBytes() == 0) {
+ throw new InvalidRequestException("Empty or truncated Kafka record batch.");
+ }
+ List copied = new ArrayList<>();
+ int validBytes = 0;
+ for (RecordBatch batch : records.batches()) {
+ validBytes += batch.sizeInBytes();
+ batch.ensureValid();
+ if (batch.hasProducerId() || batch.isTransactional() || batch.isControlBatch()) {
+ throw new InvalidRequestException(
+ "Idempotent, transactional, and control record batches are not supported.");
+ }
+ for (org.apache.kafka.common.record.Record record : batch) {
+ record.ensureValid();
+ copied.add(
+ new KafkaProduceCommand.Record(
+ record.timestamp(),
+ copyBuffer(record.hasKey() ? record.key() : null),
+ copyBuffer(record.hasValue() ? record.value() : null),
+ copyHeaders(record.headers())));
+ }
+ }
+ if (copied.isEmpty() || validBytes != records.sizeInBytes()) {
+ throw new InvalidRequestException("Empty or truncated Kafka record batch.");
+ }
+ return copied;
+ }
+
+ private static ProduceResponse toResponse(KafkaProduceResult result) {
+ ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0);
+ for (TopicResult topic : result.topics()) {
+ ProduceResponseData.TopicProduceResponse topicResponse =
+ new ProduceResponseData.TopicProduceResponse().setName(topic.topicName());
+ for (PartitionResult partition : topic.partitions()) {
+ topicResponse
+ .partitionResponses()
+ .add(
+ new ProduceResponseData.PartitionProduceResponse()
+ .setIndex(partition.partitionId())
+ .setErrorCode(partition.error().code())
+ .setBaseOffset(partition.baseOffset())
+ .setLogAppendTimeMs(-1L)
+ .setLogStartOffset(-1L)
+ .setErrorMessage(partition.errorMessage()));
+ }
+ data.responses().add(topicResponse);
+ }
+ return new ProduceResponse(data);
+ }
+
+ private static List copyHeaders(Header[] headers) {
+ List copied = new ArrayList<>(headers.length);
+ for (Header header : headers) {
+ copied.add(new RecordHeader(header.key(), header.value()));
+ }
+ return copied;
+ }
+
+ private static byte[] copyBuffer(ByteBuffer buffer) {
+ if (buffer == null) {
+ return null;
+ }
+ ByteBuffer duplicate = buffer.duplicate();
+ byte[] bytes = new byte[duplicate.remaining()];
+ duplicate.get(bytes);
+ return bytes;
+ }
+
+ private static InetAddress clientAddress(SocketAddress remoteAddress) {
+ if (remoteAddress instanceof InetSocketAddress) {
+ return ((InetSocketAddress) remoteAddress).getAddress();
+ }
+ return null;
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java
new file mode 100644
index 00000000000..c38d7bc6cb2
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.api.versions;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.KafkaRequestContext;
+import org.apache.fluss.kafka.dispatcher.KafkaApiHandler;
+import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry;
+import org.apache.fluss.kafka.dispatcher.KafkaApiSpec;
+
+import org.apache.kafka.common.message.ApiVersionsResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.ApiVersionsRequest;
+import org.apache.kafka.common.requests.ApiVersionsResponse;
+
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Implements ApiVersions from the capabilities actually registered on this server. */
+@Internal
+public final class ApiVersionsHandler implements KafkaApiHandler {
+
+ private static final KafkaApiSpec API_SPEC =
+ new KafkaApiSpec(
+ ApiKeys.API_VERSIONS,
+ ApiKeys.API_VERSIONS.oldestVersion(),
+ ApiKeys.API_VERSIONS.latestVersion(),
+ true);
+
+ private final KafkaApiRegistry registry;
+
+ /** Creates an ApiVersions handler backed by the server capability registry. */
+ public ApiVersionsHandler(KafkaApiRegistry registry) {
+ this.registry = checkNotNull(registry);
+ }
+
+ @Override
+ public KafkaApiSpec apiSpec() {
+ return API_SPEC;
+ }
+
+ @Override
+ public CompletableFuture handle(
+ KafkaRequestContext context, ApiVersionsRequest request) {
+ if (!request.isValid()) {
+ return CompletableFuture.completedFuture(
+ request.getErrorResponse(Errors.INVALID_REQUEST.exception()));
+ }
+ ApiVersionsResponseData data = new ApiVersionsResponseData();
+ for (KafkaApiSpec spec : registry.advertisedApiSpecs()) {
+ data.apiKeys()
+ .add(
+ new ApiVersionsResponseData.ApiVersion()
+ .setApiKey(spec.apiKey().id)
+ .setMinVersion(spec.minVersion())
+ .setMaxVersion(spec.maxVersion()));
+ }
+ return CompletableFuture.completedFuture(new ApiVersionsResponse(data));
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java
new file mode 100644
index 00000000000..5905624654f
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.backend.metadata;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker;
+import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition;
+import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic;
+import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError;
+import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference;
+import org.apache.fluss.kafka.mapping.KafkaTopicMapper;
+import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver;
+import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.rpc.RpcGatewayService;
+import org.apache.fluss.rpc.gateway.TabletServerGateway;
+import org.apache.fluss.rpc.messages.ListTablesRequest;
+import org.apache.fluss.rpc.messages.MetadataRequest;
+import org.apache.fluss.rpc.messages.MetadataResponse;
+import org.apache.fluss.rpc.messages.PbBucketMetadata;
+import org.apache.fluss.rpc.messages.PbServerNode;
+import org.apache.fluss.rpc.messages.PbTableMetadata;
+import org.apache.fluss.rpc.messages.PbTablePath;
+import org.apache.fluss.rpc.netty.server.Session;
+import org.apache.fluss.security.acl.FlussPrincipal;
+
+import org.apache.kafka.common.Uuid;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */
+@Internal
+public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GatewayKafkaMetadataBackend.class);
+
+ private final RpcGatewayService service;
+ private final TabletServerGateway gateway;
+ private final String databaseName;
+ private final KafkaTopicMapper topicMapper;
+ private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver();
+
+ /** Creates a metadata backend backed by the local TabletServer gateway. */
+ public GatewayKafkaMetadataBackend(
+ RpcGatewayService service, TabletServerGateway gateway, String databaseName) {
+ this.service = checkNotNull(service);
+ this.gateway = checkNotNull(gateway);
+ this.databaseName = checkNotNull(databaseName);
+ this.topicMapper = new KafkaTopicMapper(databaseName);
+ }
+
+ @Override
+ public CompletableFuture getMetadata(KafkaMetadataQuery query) {
+ if (query.allTopics() || containsTopicId(query.topics())) {
+ setCurrentSession(query);
+ return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName))
+ .thenCompose(
+ response ->
+ requestFlussMetadata(
+ query,
+ new LinkedHashSet<>(response.getTableNamesList())));
+ }
+
+ Set topicNames = new LinkedHashSet<>();
+ for (TopicReference topic : query.topics()) {
+ if (topic.topicName() != null) {
+ topicNames.add(topic.topicName());
+ }
+ }
+ return requestFlussMetadata(query, topicNames);
+ }
+
+ private CompletableFuture requestFlussMetadata(
+ KafkaMetadataQuery query, Set topicNames) {
+ return requestFlussMetadata(query, topicNames, true);
+ }
+
+ private CompletableFuture requestFlussMetadata(
+ KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) {
+ MetadataRequest request = new MetadataRequest();
+ for (String topicName : topicNames) {
+ TablePath tablePath = TablePath.of(databaseName, topicName);
+ if (!topicMapper.isMappedTable(tablePath)) {
+ continue;
+ }
+ tablePath = topicMapper.toTablePath(topicName);
+ request.addAllTablePaths(
+ Collections.singletonList(
+ new PbTablePath()
+ .setDatabaseName(tablePath.getDatabaseName())
+ .setTableName(tablePath.getTableName())));
+ }
+ setCurrentSession(query);
+ try {
+ return gateway.metadata(request)
+ .handle(
+ (response, failure) ->
+ failure == null
+ ? CompletableFuture.completedFuture(
+ toKafkaMetadata(query, response))
+ : recoverMetadataFailure(
+ query, failure, refreshAndRetry))
+ .thenCompose(future -> future);
+ } catch (Throwable failure) {
+ return recoverMetadataFailure(query, failure, refreshAndRetry);
+ }
+ }
+
+ private CompletableFuture recoverMetadataFailure(
+ KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) {
+ if (refreshAndRetry) {
+ return currentTopicNames(query)
+ .thenCompose(currentNames -> requestFlussMetadata(query, currentNames, false));
+ }
+ LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure));
+ CompletableFuture failed = new CompletableFuture<>();
+ failed.completeExceptionally(unwrap(failure));
+ return failed;
+ }
+
+ private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) {
+ setCurrentSession(query);
+ return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName))
+ .thenApply(
+ response -> {
+ Set currentNames =
+ new LinkedHashSet<>(response.getTableNamesList());
+ if (!query.allTopics() && !containsTopicId(query.topics())) {
+ Set requestedNames = new HashSet<>();
+ for (TopicReference topic : query.topics()) {
+ if (topic.topicName() != null) {
+ requestedNames.add(topic.topicName());
+ }
+ }
+ currentNames.retainAll(requestedNames);
+ }
+ return currentNames;
+ });
+ }
+
+ private KafkaClusterMetadata toKafkaMetadata(
+ KafkaMetadataQuery query, MetadataResponse response) {
+ List brokers = new ArrayList<>();
+ Set aliveBrokerIds = new HashSet<>();
+ for (PbServerNode server : response.getTabletServersList()) {
+ brokers.add(
+ new Broker(
+ server.getNodeId(),
+ server.getHost(),
+ server.getPort(),
+ server.hasRack() ? server.getRack() : null));
+ aliveBrokerIds.add(server.getNodeId());
+ }
+ Collections.sort(brokers, Comparator.comparingInt(Broker::id));
+
+ Map topicsByName = new HashMap<>();
+ Map topicsById = new HashMap<>();
+ for (PbTableMetadata table : response.getTableMetadatasList()) {
+ TablePath tablePath =
+ TablePath.of(
+ table.getTablePath().getDatabaseName(),
+ table.getTablePath().getTableName());
+ if (!topicMapper.isMappedTable(tablePath)) {
+ continue;
+ }
+ Topic topic = toKafkaTopic(table, aliveBrokerIds);
+ topicsByName.put(topic.name(), topic);
+ topicsById.put(topic.topicId(), topic);
+ }
+
+ List topics = new ArrayList<>();
+ if (query.allTopics()) {
+ for (Topic topic : topicsByName.values()) {
+ if (topic.error() == TopicError.NONE) {
+ topics.add(topic);
+ }
+ }
+ Collections.sort(topics, Comparator.comparing(Topic::name));
+ } else {
+ for (TopicReference reference : query.topics()) {
+ Topic topic =
+ reference.hasTopicId()
+ ? topicsById.get(reference.topicId())
+ : topicsByName.get(reference.topicName());
+ if (topic != null && matches(reference, topic)) {
+ topics.add(topic);
+ } else {
+ topics.add(missingTopic(reference));
+ }
+ }
+ }
+ return new KafkaClusterMetadata(brokers, topics);
+ }
+
+ private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) {
+ try {
+ schemaResolver.resolve(TableDescriptor.fromJsonBytes(table.getTableJson()));
+ } catch (IllegalArgumentException e) {
+ LOG.debug(
+ "Table {} does not define a supported Kafka mapping: {}",
+ table.getTablePath().getTableName(),
+ e.getMessage());
+ return new Topic(
+ table.getTablePath().getTableName(),
+ topicMapper.toTopicId(table.getTableId()),
+ TopicError.INVALID_TOPIC,
+ Collections.emptyList());
+ }
+ List partitions = new ArrayList<>();
+ for (PbBucketMetadata bucket : table.getBucketMetadatasList()) {
+ boolean leaderAvailable =
+ bucket.hasLeaderId() && aliveBrokerIds.contains(bucket.getLeaderId());
+ List replicas = new ArrayList<>();
+ List isr = new ArrayList<>();
+ List offlineReplicas = new ArrayList<>();
+ Set isrIds = new HashSet<>();
+ if (bucket.hasBucketEpoch()) {
+ for (int isrId : bucket.getIsrs()) {
+ isrIds.add(isrId);
+ }
+ } else if (leaderAvailable) {
+ // Legacy metadata has no authoritative ISR. Report only the available leader
+ // as a conservative fallback; live followers may still be out of sync.
+ isrIds.add(bucket.getLeaderId());
+ }
+ for (int replicaId : bucket.getReplicaIds()) {
+ replicas.add(replicaId);
+ if (isrIds.contains(replicaId)) {
+ isr.add(replicaId);
+ }
+ if (!aliveBrokerIds.contains(replicaId)) {
+ offlineReplicas.add(replicaId);
+ }
+ }
+ partitions.add(
+ new Partition(
+ bucket.getBucketId(),
+ leaderAvailable ? bucket.getLeaderId() : -1,
+ bucket.hasLeaderEpoch() ? bucket.getLeaderEpoch() : -1,
+ replicas,
+ isr,
+ offlineReplicas,
+ leaderAvailable));
+ }
+ Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId));
+ return new Topic(
+ table.getTablePath().getTableName(),
+ topicMapper.toTopicId(table.getTableId()),
+ TopicError.NONE,
+ partitions);
+ }
+
+ private static Topic missingTopic(TopicReference reference) {
+ TopicError error =
+ reference.hasTopicId()
+ ? TopicError.UNKNOWN_TOPIC_ID
+ : TopicError.UNKNOWN_TOPIC_OR_PARTITION;
+ return new Topic(
+ reference.topicName(), reference.topicId(), error, Collections.emptyList());
+ }
+
+ private static boolean matches(TopicReference reference, Topic topic) {
+ return (reference.topicName() == null || reference.topicName().equals(topic.name()))
+ && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId()));
+ }
+
+ private void setCurrentSession(KafkaMetadataQuery query) {
+ service.setCurrentSession(
+ new Session(
+ (short) 0,
+ query.listenerName(),
+ false,
+ query.clientAddress(),
+ FlussPrincipal.ANONYMOUS));
+ }
+
+ private static boolean containsTopicId(List topics) {
+ for (TopicReference topic : topics) {
+ if (topic.hasTopicId()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static Throwable unwrap(Throwable failure) {
+ Throwable current = failure;
+ while (current instanceof CompletionException && current.getCause() != null) {
+ current = current.getCause();
+ }
+ return current;
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java
new file mode 100644
index 00000000000..bbc634af6f0
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.backend.metadata;
+
+import org.apache.fluss.annotation.Internal;
+
+import org.apache.kafka.common.Uuid;
+
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Kafka-domain cluster metadata returned by a Fluss metadata backend. */
+@Internal
+public final class KafkaClusterMetadata {
+
+ private final List brokers;
+ private final List topics;
+
+ /** Creates cluster metadata. */
+ public KafkaClusterMetadata(List brokers, List topics) {
+ this.brokers = immutableCopy(brokers);
+ this.topics = immutableCopy(topics);
+ }
+
+ /** Returns Kafka-reachable brokers. */
+ public List brokers() {
+ return brokers;
+ }
+
+ /** Returns topic metadata and topic-level errors. */
+ public List topics() {
+ return topics;
+ }
+
+ private static List immutableCopy(List values) {
+ return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values)));
+ }
+
+ /** Kafka-reachable broker information. */
+ @Internal
+ public static final class Broker {
+
+ private final int id;
+ private final String host;
+ private final int port;
+ private final @Nullable String rack;
+
+ /** Creates broker information. */
+ public Broker(int id, String host, int port, @Nullable String rack) {
+ this.id = id;
+ this.host = checkNotNull(host);
+ this.port = port;
+ this.rack = rack;
+ }
+
+ /** Returns the Kafka broker ID. */
+ public int id() {
+ return id;
+ }
+
+ /** Returns the Kafka listener host. */
+ public String host() {
+ return host;
+ }
+
+ /** Returns the Kafka listener port. */
+ public int port() {
+ return port;
+ }
+
+ /** Returns the broker rack, if configured. */
+ public @Nullable String rack() {
+ return rack;
+ }
+ }
+
+ /** Topic-level error independent of a Kafka response schema version. */
+ @Internal
+ public enum TopicError {
+ NONE,
+ UNKNOWN_TOPIC_OR_PARTITION,
+ UNKNOWN_TOPIC_ID,
+ INVALID_TOPIC
+ }
+
+ /** Metadata for one Kafka topic. */
+ @Internal
+ public static final class Topic {
+
+ private final @Nullable String name;
+ private final Uuid topicId;
+ private final TopicError error;
+ private final List partitions;
+
+ /** Creates topic metadata. */
+ public Topic(
+ @Nullable String name, Uuid topicId, TopicError error, List partitions) {
+ this.name = name;
+ this.topicId = checkNotNull(topicId);
+ this.error = checkNotNull(error);
+ this.partitions = immutableCopy(partitions);
+ }
+
+ /** Returns the Kafka topic name, if known. */
+ public @Nullable String name() {
+ return name;
+ }
+
+ /** Returns the stable Kafka topic ID. */
+ public Uuid topicId() {
+ return topicId;
+ }
+
+ /** Returns the topic-level domain error. */
+ public TopicError error() {
+ return error;
+ }
+
+ /** Returns the topic partitions. */
+ public List partitions() {
+ return partitions;
+ }
+ }
+
+ /** Metadata for one Kafka partition backed by a Fluss bucket. */
+ @Internal
+ public static final class Partition {
+
+ private final int partitionId;
+ private final int leaderId;
+ private final int leaderEpoch;
+ private final List replicas;
+ private final List isr;
+ private final List offlineReplicas;
+ private final boolean leaderAvailable;
+
+ /** Creates partition metadata. */
+ public Partition(
+ int partitionId,
+ int leaderId,
+ int leaderEpoch,
+ List replicas,
+ List isr,
+ List offlineReplicas,
+ boolean leaderAvailable) {
+ this.partitionId = partitionId;
+ this.leaderId = leaderId;
+ this.leaderEpoch = leaderEpoch;
+ this.replicas = immutableCopy(replicas);
+ this.isr = immutableCopy(isr);
+ this.offlineReplicas = immutableCopy(offlineReplicas);
+ this.leaderAvailable = leaderAvailable;
+ }
+
+ /** Returns the Kafka partition ID. */
+ public int partitionId() {
+ return partitionId;
+ }
+
+ /** Returns the current leader ID, or {@code -1} when unavailable. */
+ public int leaderId() {
+ return leaderId;
+ }
+
+ /** Returns the leader epoch, or {@code -1} when unavailable. */
+ public int leaderEpoch() {
+ return leaderEpoch;
+ }
+
+ /** Returns assigned replica IDs. */
+ public List replicas() {
+ return replicas;
+ }
+
+ /** Returns replica IDs currently visible as in-sync. */
+ public List isr() {
+ return isr;
+ }
+
+ /** Returns assigned replicas whose TabletServers are unavailable. */
+ public List offlineReplicas() {
+ return offlineReplicas;
+ }
+
+ /** Returns whether the partition has a reachable leader. */
+ public boolean leaderAvailable() {
+ return leaderAvailable;
+ }
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java
new file mode 100644
index 00000000000..abb02a920ed
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.backend.metadata;
+
+import org.apache.fluss.annotation.Internal;
+
+import java.util.concurrent.CompletableFuture;
+
+/** Narrow backend used by the Kafka Metadata API. */
+@Internal
+public interface KafkaMetadataBackend {
+
+ /** Resolves Kafka-domain metadata asynchronously. */
+ CompletableFuture getMetadata(KafkaMetadataQuery query);
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java
new file mode 100644
index 00000000000..01e21fa4440
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.backend.metadata;
+
+import org.apache.fluss.annotation.Internal;
+
+import org.apache.kafka.common.Uuid;
+
+import javax.annotation.Nullable;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Domain query used by the Metadata API to access the Fluss adapter layer. */
+@Internal
+public final class KafkaMetadataQuery {
+
+ private final boolean allTopics;
+ private final List topics;
+ private final String listenerName;
+ private final @Nullable InetAddress clientAddress;
+
+ /** Creates a metadata query. */
+ public KafkaMetadataQuery(
+ boolean allTopics,
+ List topics,
+ String listenerName,
+ @Nullable InetAddress clientAddress) {
+ this.allTopics = allTopics;
+ this.topics = Collections.unmodifiableList(new ArrayList<>(checkNotNull(topics)));
+ this.listenerName = checkNotNull(listenerName);
+ this.clientAddress = clientAddress;
+ }
+
+ /** Returns whether all Kafka topics should be returned. */
+ public boolean allTopics() {
+ return allTopics;
+ }
+
+ /** Returns the explicitly requested topic identities. */
+ public List topics() {
+ return topics;
+ }
+
+ /** Returns the Kafka listener used by the client connection. */
+ public String listenerName() {
+ return listenerName;
+ }
+
+ /** Returns the client address when it is available. */
+ public @Nullable InetAddress clientAddress() {
+ return clientAddress;
+ }
+
+ /** Kafka topic name and ID supplied by a Metadata request. */
+ @Internal
+ public static final class TopicReference {
+
+ private final @Nullable String topicName;
+ private final Uuid topicId;
+
+ /** Creates a topic reference. */
+ public TopicReference(@Nullable String topicName, Uuid topicId) {
+ this.topicName = topicName;
+ this.topicId = checkNotNull(topicId);
+ }
+
+ /** Returns the requested topic name, if present. */
+ public @Nullable String topicName() {
+ return topicName;
+ }
+
+ /** Returns the requested topic ID, or {@link Uuid#ZERO_UUID} when absent. */
+ public Uuid topicId() {
+ return topicId;
+ }
+
+ /** Returns whether this reference identifies a topic by ID. */
+ public boolean hasTopicId() {
+ return !Uuid.ZERO_UUID.equals(topicId);
+ }
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java
new file mode 100644
index 00000000000..3b8428c5670
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.backend.produce;
+
+import org.apache.fluss.annotation.Internal;
+
+import java.util.concurrent.CompletableFuture;
+
+/** Narrow backend used by the Kafka Produce API. */
+@Internal
+public interface KafkaProduceBackend {
+ /** Writes copied Kafka records through the native Fluss write path. */
+ CompletableFuture write(KafkaProduceCommand command);
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java
new file mode 100644
index 00000000000..722f3d3d245
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.backend.produce;
+
+import org.apache.fluss.annotation.Internal;
+
+import javax.annotation.Nullable;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Protocol-independent write command used by the Kafka Produce backend. */
+@Internal
+public final class KafkaProduceCommand {
+
+ private final short acks;
+ private final int timeoutMs;
+ private final List topics;
+ private final String listenerName;
+ private final @Nullable InetAddress clientAddress;
+
+ /** Creates a Kafka write command. */
+ public KafkaProduceCommand(
+ short acks,
+ int timeoutMs,
+ List topics,
+ String listenerName,
+ @Nullable InetAddress clientAddress) {
+ this.acks = acks;
+ this.timeoutMs = timeoutMs;
+ this.topics = immutableCopy(topics);
+ this.listenerName = checkNotNull(listenerName);
+ this.clientAddress = clientAddress;
+ }
+
+ /** Returns Kafka required acknowledgements. */
+ public short acks() {
+ return acks;
+ }
+
+ /** Returns the Produce timeout in milliseconds. */
+ public int timeoutMs() {
+ return timeoutMs;
+ }
+
+ /** Returns the topic writes in request order. */
+ public List topics() {
+ return topics;
+ }
+
+ /** Returns the listener that received the request. */
+ public String listenerName() {
+ return listenerName;
+ }
+
+ /** Returns the client network address when available. */
+ public @Nullable InetAddress clientAddress() {
+ return clientAddress;
+ }
+
+ private static List immutableCopy(List values) {
+ return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values)));
+ }
+
+ /** Records addressed to one Kafka topic. */
+ @Internal
+ public static final class TopicWrite {
+ private final String topicName;
+ private final List partitions;
+
+ /** Creates the writes for one topic. */
+ public TopicWrite(String topicName, List partitions) {
+ this.topicName = checkNotNull(topicName);
+ this.partitions = immutableCopy(partitions);
+ }
+
+ /** Returns the Kafka topic name. */
+ public String topicName() {
+ return topicName;
+ }
+
+ /** Returns partition writes in request order. */
+ public List partitions() {
+ return partitions;
+ }
+ }
+
+ /** Records addressed to one Kafka partition. */
+ @Internal
+ public static final class PartitionWrite {
+ private final int partitionId;
+ private final List records;
+
+ /** Creates the writes for one partition. */
+ public PartitionWrite(int partitionId, List records) {
+ this.partitionId = partitionId;
+ this.records = immutableCopy(records);
+ }
+
+ /** Returns the Kafka partition ID. */
+ public int partitionId() {
+ return partitionId;
+ }
+
+ /** Returns copied records in append order. */
+ public List records() {
+ return records;
+ }
+ }
+
+ /** A copied Kafka record whose lifetime is independent of the network request buffer. */
+ @Internal
+ public static final class Record {
+ private final long timestamp;
+ private final @Nullable byte[] key;
+ private final @Nullable byte[] value;
+ private final List headers;
+
+ /** Creates a copied Kafka record. */
+ public Record(
+ long timestamp,
+ @Nullable byte[] key,
+ @Nullable byte[] value,
+ List headers) {
+ this.timestamp = timestamp;
+ this.key = copyNullable(key);
+ this.value = copyNullable(value);
+ this.headers = immutableCopy(headers);
+ }
+
+ /** Returns the Kafka record timestamp. */
+ public long timestamp() {
+ return timestamp;
+ }
+
+ /** Returns a copy of the nullable Kafka record key. */
+ public @Nullable byte[] key() {
+ return copyNullable(key);
+ }
+
+ /** Returns a copy of the nullable Kafka record value. */
+ public @Nullable byte[] value() {
+ return copyNullable(value);
+ }
+
+ /** Returns the copied Kafka headers in record order. */
+ public List headers() {
+ return headers;
+ }
+
+ private static @Nullable byte[] copyNullable(@Nullable byte[] value) {
+ return value == null ? null : value.clone();
+ }
+ }
+
+ /** A copied Kafka record header. */
+ @Internal
+ public static final class RecordHeader {
+ private final String name;
+ private final @Nullable byte[] value;
+
+ /** Creates a copied Kafka record header. */
+ public RecordHeader(String name, @Nullable byte[] value) {
+ this.name = checkNotNull(name);
+ this.value = value == null ? null : value.clone();
+ }
+
+ /** Returns the header name. */
+ public String name() {
+ return name;
+ }
+
+ /** Returns a copy of the nullable header value. */
+ public @Nullable byte[] value() {
+ return value == null ? null : value.clone();
+ }
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java
new file mode 100644
index 00000000000..218c566bde4
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.backend.produce;
+
+import org.apache.fluss.annotation.Internal;
+
+import org.apache.kafka.common.protocol.Errors;
+
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Result of a Kafka Produce backend invocation. */
+@Internal
+public final class KafkaProduceResult {
+ private final List topics;
+
+ /** Creates a Produce result. */
+ public KafkaProduceResult(List topics) {
+ this.topics = immutableCopy(topics);
+ }
+
+ /** Returns topic results in request order. */
+ public List topics() {
+ return topics;
+ }
+
+ private static List immutableCopy(List values) {
+ return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values)));
+ }
+
+ /** Results for one topic. */
+ @Internal
+ public static final class TopicResult {
+ private final String topicName;
+ private final List partitions;
+
+ /** Creates the result for one topic. */
+ public TopicResult(String topicName, List partitions) {
+ this.topicName = checkNotNull(topicName);
+ this.partitions = immutableCopy(partitions);
+ }
+
+ /** Returns the Kafka topic name. */
+ public String topicName() {
+ return topicName;
+ }
+
+ /** Returns the partition results in request order. */
+ public List partitions() {
+ return partitions;
+ }
+ }
+
+ /** Result for one partition. */
+ @Internal
+ public static final class PartitionResult {
+ private final int partitionId;
+ private final Errors error;
+ private final long baseOffset;
+ private final @Nullable String errorMessage;
+
+ /** Creates the result for one partition. */
+ public PartitionResult(
+ int partitionId, Errors error, long baseOffset, @Nullable String errorMessage) {
+ this.partitionId = partitionId;
+ this.error = checkNotNull(error);
+ this.baseOffset = baseOffset;
+ this.errorMessage = errorMessage;
+ }
+
+ /** Returns the Kafka partition ID. */
+ public int partitionId() {
+ return partitionId;
+ }
+
+ /** Returns the Kafka protocol error. */
+ public Errors error() {
+ return error;
+ }
+
+ /** Returns the first appended offset, or {@code -1} on failure. */
+ public long baseOffset() {
+ return baseOffset;
+ }
+
+ /** Returns an optional diagnostic error message. */
+ public @Nullable String errorMessage() {
+ return errorMessage;
+ }
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java
new file mode 100644
index 00000000000..36995b8e27f
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.dispatcher;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.KafkaRequestContext;
+
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.AbstractResponse;
+
+import java.util.concurrent.CompletableFuture;
+
+/** Handles one Kafka API without blocking the request processor thread. */
+@Internal
+public interface KafkaApiHandler {
+
+ /** Returns the capability implemented by this handler. */
+ KafkaApiSpec apiSpec();
+
+ /** Handles a parsed request asynchronously. */
+ CompletableFuture handle(KafkaRequestContext context, R request);
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java
new file mode 100644
index 00000000000..b4a1dfd8ea3
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.dispatcher;
+
+import org.apache.fluss.annotation.Internal;
+
+import org.apache.kafka.common.protocol.ApiKeys;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.fluss.utils.Preconditions.checkArgument;
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+import static org.apache.fluss.utils.Preconditions.checkState;
+
+/** Registry and single source of truth for Kafka APIs exposed by one server. */
+@Internal
+public final class KafkaApiRegistry {
+
+ private final Map> handlers = new HashMap<>();
+ private boolean frozen;
+
+ /** Creates an empty API registry. */
+ public KafkaApiRegistry() {}
+
+ /** Registers a handler. Registrations are rejected after {@link #freeze()} is called. */
+ public void register(KafkaApiHandler handler) {
+ checkNotNull(handler);
+ checkState(!frozen, "Kafka API registry is already frozen.");
+ ApiKeys apiKey = handler.apiSpec().apiKey();
+ checkArgument(!handlers.containsKey(apiKey), "Kafka API %s is already registered.", apiKey);
+ handlers.put(apiKey, handler);
+ }
+
+ /** Prevents further registrations. */
+ public void freeze() {
+ frozen = true;
+ }
+
+ /** Returns a routable handler, or {@code null} when the API is not exposed by this server. */
+ public KafkaApiHandler lookup(ApiKeys apiKey) {
+ KafkaApiHandler handler = handlers.get(apiKey);
+ if (handler == null || !handler.apiSpec().advertised()) {
+ return null;
+ }
+ return handler;
+ }
+
+ /** Returns the sorted API specifications advertised by this server. */
+ public List advertisedApiSpecs() {
+ List specs = new ArrayList<>();
+ for (KafkaApiHandler handler : handlers.values()) {
+ KafkaApiSpec spec = handler.apiSpec();
+ if (spec.advertised()) {
+ specs.add(spec);
+ }
+ }
+ Collections.sort(specs, Comparator.comparingInt(spec -> spec.apiKey().id));
+ return Collections.unmodifiableList(specs);
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java
new file mode 100644
index 00000000000..50d6a7ebab2
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.dispatcher;
+
+import org.apache.fluss.annotation.Internal;
+
+import org.apache.kafka.common.protocol.ApiKeys;
+
+import static org.apache.fluss.utils.Preconditions.checkArgument;
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Describes the versions actually supported by a Kafka API handler. */
+@Internal
+public final class KafkaApiSpec {
+
+ private final ApiKeys apiKey;
+ private final short minVersion;
+ private final short maxVersion;
+ private final boolean advertised;
+
+ /** Creates an API specification. */
+ public KafkaApiSpec(ApiKeys apiKey, short minVersion, short maxVersion, boolean advertised) {
+ this.apiKey = checkNotNull(apiKey);
+ checkArgument(minVersion >= 0, "Minimum version must not be negative.");
+ checkArgument(
+ minVersion <= maxVersion,
+ "Minimum version %s must not exceed maximum version %s.",
+ minVersion,
+ maxVersion);
+ checkArgument(
+ minVersion >= apiKey.oldestVersion() && maxVersion <= apiKey.latestVersion(),
+ "Version range [%s, %s] is outside the Kafka library range [%s, %s] for %s.",
+ minVersion,
+ maxVersion,
+ apiKey.oldestVersion(),
+ apiKey.latestVersion(),
+ apiKey);
+ this.minVersion = minVersion;
+ this.maxVersion = maxVersion;
+ this.advertised = advertised;
+ }
+
+ /** Returns the Kafka API key. */
+ public ApiKeys apiKey() {
+ return apiKey;
+ }
+
+ /** Returns the oldest supported request version. */
+ public short minVersion() {
+ return minVersion;
+ }
+
+ /** Returns the newest supported request version. */
+ public short maxVersion() {
+ return maxVersion;
+ }
+
+ /** Returns whether this API is allowed to be routed and advertised. */
+ public boolean advertised() {
+ return advertised;
+ }
+
+ /** Returns whether the supplied request version is supported. */
+ public boolean supportsVersion(short version) {
+ return version >= minVersion && version <= maxVersion;
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java
new file mode 100644
index 00000000000..efdfe33dd66
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.dispatcher;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.KafkaRequest;
+import org.apache.fluss.kafka.KafkaRequestContext;
+import org.apache.fluss.kafka.error.KafkaErrorMapper;
+
+import org.apache.kafka.common.errors.UnsupportedVersionException;
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.AbstractResponse;
+
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Validates and dispatches parsed Kafka requests to independently registered API handlers. */
+@Internal
+public final class KafkaRequestDispatcher {
+
+ private final KafkaApiRegistry registry;
+ private final KafkaErrorMapper errorMapper;
+
+ /** Creates a dispatcher backed by the supplied registry and error mapper. */
+ public KafkaRequestDispatcher(KafkaApiRegistry registry, KafkaErrorMapper errorMapper) {
+ this.registry = checkNotNull(registry);
+ this.errorMapper = checkNotNull(errorMapper);
+ }
+
+ /** Dispatches a request and always completes with a Kafka protocol response. */
+ public CompletableFuture dispatch(KafkaRequest request) {
+ AbstractRequest abstractRequest = request.request();
+ KafkaApiHandler handler = registry.lookup(request.apiKey());
+ if (handler == null) {
+ return completedErrorResponse(
+ abstractRequest,
+ new UnsupportedVersionException(
+ "Kafka API " + request.apiKey() + " is not supported by this server."));
+ }
+
+ KafkaApiSpec spec = handler.apiSpec();
+ if (!spec.supportsVersion(request.apiVersion())) {
+ return completedErrorResponse(
+ abstractRequest,
+ new UnsupportedVersionException(
+ String.format(
+ "Version %s is not supported for %s. Supported versions are [%s, %s].",
+ request.apiVersion(),
+ request.apiKey(),
+ spec.minVersion(),
+ spec.maxVersion())));
+ }
+
+ CompletableFuture responseFuture;
+ try {
+ responseFuture =
+ invoke(handler, KafkaRequestContext.fromRequest(request), abstractRequest);
+ if (responseFuture == null) {
+ throw new NullPointerException("Kafka API handler returned a null future.");
+ }
+ } catch (Throwable t) {
+ return completedErrorResponse(abstractRequest, t);
+ }
+
+ CompletableFuture result = new CompletableFuture<>();
+ responseFuture.whenComplete(
+ (response, failure) -> {
+ if (failure == null && response != null) {
+ result.complete(response);
+ } else {
+ Throwable responseFailure =
+ failure == null
+ ? new NullPointerException(
+ "Kafka API handler returned a null response.")
+ : failure;
+ result.complete(errorMapper.toResponse(abstractRequest, responseFailure));
+ }
+ });
+ return result;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static CompletableFuture invoke(
+ KafkaApiHandler handler, KafkaRequestContext context, AbstractRequest request) {
+ return ((KafkaApiHandler) handler).handle(context, request);
+ }
+
+ private CompletableFuture completedErrorResponse(
+ AbstractRequest request, Throwable failure) {
+ return CompletableFuture.completedFuture(errorMapper.toResponse(request, failure));
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java
new file mode 100644
index 00000000000..4396566396d
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.error;
+
+import org.apache.fluss.annotation.Internal;
+
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.AbstractResponse;
+
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+
+/** Maps failures from the compatibility layer to version-aware Kafka responses. */
+@Internal
+public final class KafkaErrorMapper {
+
+ /** Converts a failure to the error response defined by the parsed Kafka request. */
+ public AbstractResponse toResponse(AbstractRequest request, Throwable failure) {
+ return request.getErrorResponse(unwrap(failure));
+ }
+
+ private static Throwable unwrap(Throwable failure) {
+ Throwable current = failure;
+ while ((current instanceof CompletionException || current instanceof ExecutionException)
+ && current.getCause() != null) {
+ current = current.getCause();
+ }
+ return current;
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java
new file mode 100644
index 00000000000..ad0039b7e78
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.format;
+
+import org.apache.fluss.annotation.Internal;
+
+import java.util.Locale;
+
+/** Supported interpretations of Kafka record key and value bytes. */
+@Internal
+public enum KafkaDataFormat {
+ RAW("raw"),
+ STRING("string");
+
+ /** Fluss table custom property controlling the record key format. */
+ public static final String KEY_FORMAT_CONFIG = "kafka.key.format";
+
+ /** Fluss table custom property controlling the record value format. */
+ public static final String VALUE_FORMAT_CONFIG = "kafka.value.format";
+
+ /** Fluss fields populated from the Kafka record key. */
+ public static final String KEY_FIELDS_CONFIG = "kafka.key.fields";
+
+ /** Strategy for deriving fields populated from the Kafka record value. */
+ public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include";
+
+ /** Fluss column populated from the Kafka record timestamp. */
+ public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column";
+
+ /** Fluss column populated from the Kafka record headers. */
+ public static final String HEADERS_COLUMN_CONFIG = "kafka.metadata.headers.column";
+
+ private final String value;
+
+ KafkaDataFormat(String value) {
+ this.value = value;
+ }
+
+ /** Parses a table custom property value. */
+ public static KafkaDataFormat parse(String value) {
+ if (value == null) {
+ throw new IllegalArgumentException("Kafka data format must not be null.");
+ }
+ String normalized = value.trim().toLowerCase(Locale.ROOT);
+ for (KafkaDataFormat format : values()) {
+ if (format.value.equals(normalized)) {
+ return format;
+ }
+ }
+ throw new IllegalArgumentException(
+ "Unsupported Kafka data format '" + value + "'. Expected raw or string.");
+ }
+
+ /** Returns the persisted table property value. */
+ public String value() {
+ return value;
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java
new file mode 100644
index 00000000000..9f8ed63f543
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.mapping;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.metadata.TablePath;
+
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.internals.Topic;
+
+import static org.apache.fluss.utils.Preconditions.checkArgument;
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */
+@Internal
+public final class KafkaTopicMapper {
+
+ // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs.
+ private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L;
+
+ private final String databaseName;
+
+ /** Creates a topic mapper for one Fluss database. */
+ public KafkaTopicMapper(String databaseName) {
+ this.databaseName = checkNotNull(databaseName);
+ }
+
+ /** Maps a Kafka topic name to its Fluss table path. */
+ public TablePath toTablePath(String topicName) {
+ Topic.validate(topicName);
+ return TablePath.of(databaseName, topicName);
+ }
+
+ /** Returns whether a table belongs to this database and has a valid Kafka topic name. */
+ public boolean isMappedTable(TablePath tablePath) {
+ return databaseName.equals(tablePath.getDatabaseName())
+ && Topic.isValid(tablePath.getTableName());
+ }
+
+ /** Maps a Fluss table ID to a stable Kafka topic ID. */
+ public Uuid toTopicId(long tableId) {
+ checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId);
+ return new Uuid(TOPIC_ID_NAMESPACE, tableId);
+ }
+
+ /** Returns whether a Kafka topic ID can represent a Fluss table ID. */
+ public boolean isMappedTopicId(Uuid topicId) {
+ return topicId != null
+ && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE
+ && topicId.getLeastSignificantBits() >= 0L;
+ }
+
+ /** Extracts the Fluss table ID encoded in a Kafka topic ID. */
+ public long toTableId(Uuid topicId) {
+ checkArgument(isMappedTopicId(topicId), "Topic ID %s is not a Fluss topic ID.", topicId);
+ return topicId.getLeastSignificantBits();
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java
new file mode 100644
index 00000000000..b1715758a9d
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.schema;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.types.DataType;
+import org.apache.fluss.types.RowType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import static org.apache.fluss.utils.Preconditions.checkArgument;
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Ordered physical Fluss fields populated by one Kafka record component. */
+@Internal
+public final class KafkaFieldProjection {
+
+ private final List positions;
+ private final List names;
+ private final List dataTypes;
+
+ /** Creates a projection from physical row positions. */
+ public KafkaFieldProjection(RowType rowType, List positions) {
+ checkNotNull(rowType);
+ checkNotNull(positions);
+ List positionCopy = new ArrayList<>(positions.size());
+ List projectedNames = new ArrayList<>(positions.size());
+ List projectedTypes = new ArrayList<>(positions.size());
+ for (Integer position : positions) {
+ checkArgument(
+ position != null && position >= 0 && position < rowType.getFieldCount(),
+ "Invalid Kafka field projection position %s.",
+ position);
+ positionCopy.add(position);
+ projectedNames.add(rowType.getFieldNames().get(position));
+ projectedTypes.add(rowType.getTypeAt(position));
+ }
+ this.positions = Collections.unmodifiableList(positionCopy);
+ this.names = Collections.unmodifiableList(projectedNames);
+ this.dataTypes = Collections.unmodifiableList(projectedTypes);
+ }
+
+ /** Returns the number of projected fields. */
+ public int size() {
+ return positions.size();
+ }
+
+ /** Returns whether this projection owns no fields. */
+ public boolean isEmpty() {
+ return positions.isEmpty();
+ }
+
+ /** Returns the physical row position at the projection position. */
+ public int positionAt(int projectionPosition) {
+ return positions.get(projectionPosition);
+ }
+
+ /** Returns the physical field name at the projection position. */
+ public String nameAt(int projectionPosition) {
+ return names.get(projectionPosition);
+ }
+
+ /** Returns the physical data type at the projection position. */
+ public DataType dataTypeAt(int projectionPosition) {
+ return dataTypes.get(projectionPosition);
+ }
+
+ /** Returns the projected physical positions. */
+ public List positions() {
+ return positions;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof KafkaFieldProjection)) {
+ return false;
+ }
+ KafkaFieldProjection that = (KafkaFieldProjection) obj;
+ return Objects.equals(positions, that.positions)
+ && Objects.equals(names, that.names)
+ && Objects.equals(dataTypes, that.dataTypes);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(positions, names, dataTypes);
+ }
+
+ @Override
+ public String toString() {
+ return "KafkaFieldProjection{"
+ + "positions="
+ + positions
+ + ", "
+ + "names="
+ + names
+ + ", "
+ + "dataTypes="
+ + dataTypes
+ + "}";
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java
new file mode 100644
index 00000000000..64263e69286
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.schema;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.format.KafkaDataFormat;
+import org.apache.fluss.types.RowType;
+
+import javax.annotation.Nullable;
+
+import java.util.Objects;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Resolved Kafka key, value, and metadata mapping for one Fluss table schema. */
+@Internal
+public final class KafkaTopicSchema {
+
+ private final RowType rowType;
+ private final @Nullable KafkaDataFormat keyFormat;
+ private final KafkaFieldProjection keyProjection;
+ private final KafkaDataFormat valueFormat;
+ private final KafkaFieldProjection valueProjection;
+ private final int timestampPosition;
+ private final int headersPosition;
+
+ /** Creates a resolved Kafka topic schema. */
+ public KafkaTopicSchema(
+ RowType rowType,
+ @Nullable KafkaDataFormat keyFormat,
+ KafkaFieldProjection keyProjection,
+ KafkaDataFormat valueFormat,
+ KafkaFieldProjection valueProjection,
+ int timestampPosition,
+ int headersPosition) {
+ this.rowType = checkNotNull(rowType);
+ this.keyFormat = keyFormat;
+ this.keyProjection = checkNotNull(keyProjection);
+ this.valueFormat = checkNotNull(valueFormat);
+ this.valueProjection = checkNotNull(valueProjection);
+ this.timestampPosition = timestampPosition;
+ this.headersPosition = headersPosition;
+ }
+
+ /** Returns the physical Fluss row type. */
+ public RowType rowType() {
+ return rowType;
+ }
+
+ /** Returns the key format, or null when the Kafka key is not mapped. */
+ public @Nullable KafkaDataFormat keyFormat() {
+ return keyFormat;
+ }
+
+ /** Returns the key field projection. */
+ public KafkaFieldProjection keyProjection() {
+ return keyProjection;
+ }
+
+ /** Returns the value format. */
+ public KafkaDataFormat valueFormat() {
+ return valueFormat;
+ }
+
+ /** Returns the value field projection. */
+ public KafkaFieldProjection valueProjection() {
+ return valueProjection;
+ }
+
+ /** Returns the timestamp physical position, or -1 when it is not mapped. */
+ public int timestampPosition() {
+ return timestampPosition;
+ }
+
+ /** Returns the headers physical position, or -1 when they are not mapped. */
+ public int headersPosition() {
+ return headersPosition;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof KafkaTopicSchema)) {
+ return false;
+ }
+ KafkaTopicSchema that = (KafkaTopicSchema) obj;
+ return Objects.equals(rowType, that.rowType)
+ && Objects.equals(keyFormat, that.keyFormat)
+ && Objects.equals(keyProjection, that.keyProjection)
+ && Objects.equals(valueFormat, that.valueFormat)
+ && Objects.equals(valueProjection, that.valueProjection)
+ && Objects.equals(timestampPosition, that.timestampPosition)
+ && Objects.equals(headersPosition, that.headersPosition);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ rowType,
+ keyFormat,
+ keyProjection,
+ valueFormat,
+ valueProjection,
+ timestampPosition,
+ headersPosition);
+ }
+
+ @Override
+ public String toString() {
+ return "KafkaTopicSchema{"
+ + "rowType="
+ + rowType
+ + ", "
+ + "keyFormat="
+ + keyFormat
+ + ", "
+ + "keyProjection="
+ + keyProjection
+ + ", "
+ + "valueFormat="
+ + valueFormat
+ + ", "
+ + "valueProjection="
+ + valueProjection
+ + ", "
+ + "timestampPosition="
+ + timestampPosition
+ + ", "
+ + "headersPosition="
+ + headersPosition
+ + "}";
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java
new file mode 100644
index 00000000000..0e6a294c4d5
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.schema;
+
+import org.apache.fluss.annotation.Internal;
+
+/** Indicates that a Fluss table does not define a valid Kafka record mapping contract. */
+@Internal
+public final class KafkaTopicSchemaException extends IllegalArgumentException {
+
+ /** Creates a Kafka topic schema exception. */
+ public KafkaTopicSchemaException(String message) {
+ super(message);
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java
new file mode 100644
index 00000000000..840ce8a4f58
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.schema;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.config.Configuration;
+import org.apache.fluss.config.TableConfig;
+import org.apache.fluss.kafka.format.KafkaDataFormat;
+import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.types.ArrayType;
+import org.apache.fluss.types.BytesType;
+import org.apache.fluss.types.LocalZonedTimestampType;
+import org.apache.fluss.types.RowType;
+import org.apache.fluss.types.StringType;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+/** Resolves and validates the Kafka record mapping stored in Fluss table custom properties. */
+@Internal
+public final class KafkaTopicSchemaResolver {
+
+ private static final String INCLUDE_ALL = "ALL";
+ private static final String INCLUDE_EXCEPT_KEY = "EXCEPT_KEY";
+
+ private static final Set SUPPORTED_PROPERTIES =
+ new HashSet<>(
+ Arrays.asList(
+ KafkaDataFormat.KEY_FORMAT_CONFIG,
+ KafkaDataFormat.KEY_FIELDS_CONFIG,
+ KafkaDataFormat.VALUE_FORMAT_CONFIG,
+ KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG,
+ KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG,
+ KafkaDataFormat.HEADERS_COLUMN_CONFIG));
+
+ /** Resolves one table's Kafka record mapping contract. */
+ public KafkaTopicSchema resolve(TableDescriptor table) {
+ validateTableKind(table);
+ RowType rowType = table.getSchema().getRowType();
+ Map properties = table.getCustomProperties();
+
+ for (String property : properties.keySet()) {
+ if (property.startsWith("kafka.") && !SUPPORTED_PROPERTIES.contains(property)) {
+ throw invalid("Unsupported Kafka table property '" + property + "'.");
+ }
+ }
+
+ int timestampPosition =
+ resolveOptionalPosition(
+ rowType, properties.get(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG));
+ int headersPosition =
+ resolveOptionalPosition(
+ rowType, properties.get(KafkaDataFormat.HEADERS_COLUMN_CONFIG));
+ if (timestampPosition >= 0 && timestampPosition == headersPosition) {
+ throw invalid("Kafka timestamp and headers cannot map to the same Fluss column.");
+ }
+ validateMetadataColumns(rowType, timestampPosition, headersPosition);
+
+ String keyFormatValue = properties.get(KafkaDataFormat.KEY_FORMAT_CONFIG);
+ KafkaDataFormat keyFormat = keyFormatValue == null ? null : parseFormat(keyFormatValue);
+ List keyPositions =
+ resolveKeyPositions(
+ rowType, keyFormat, properties.get(KafkaDataFormat.KEY_FIELDS_CONFIG));
+ for (Integer keyPosition : keyPositions) {
+ if (keyPosition == timestampPosition || keyPosition == headersPosition) {
+ throw invalid(
+ "Kafka key field '"
+ + rowType.getFieldNames().get(keyPosition)
+ + "' cannot be a Kafka metadata column.");
+ }
+ }
+ KafkaFieldProjection keyProjection = new KafkaFieldProjection(rowType, keyPositions);
+
+ String valueFormatValue = properties.get(KafkaDataFormat.VALUE_FORMAT_CONFIG);
+ if (valueFormatValue == null) {
+ throw invalid(
+ "Missing required table property '"
+ + KafkaDataFormat.VALUE_FORMAT_CONFIG
+ + "'.");
+ }
+ KafkaDataFormat valueFormat = parseFormat(valueFormatValue);
+ String fieldsInclude =
+ normalizeFieldsInclude(properties.get(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG));
+ if (!keyPositions.isEmpty() && INCLUDE_ALL.equals(fieldsInclude)) {
+ throw invalid(
+ "Mapping Kafka key fields requires "
+ + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG
+ + "=EXCEPT_KEY.");
+ }
+
+ Set metadataPositions = new HashSet<>();
+ if (timestampPosition >= 0) {
+ metadataPositions.add(timestampPosition);
+ }
+ if (headersPosition >= 0) {
+ metadataPositions.add(headersPosition);
+ }
+ List valuePositions =
+ resolveValuePositions(rowType, fieldsInclude, keyPositions, metadataPositions);
+ KafkaFieldProjection valueProjection = new KafkaFieldProjection(rowType, valuePositions);
+ if (valueProjection.isEmpty()) {
+ throw invalid("Kafka value projection must contain at least one Fluss column.");
+ }
+
+ validateSingleFieldFormat(keyFormat, keyProjection, "key");
+ validateSingleFieldFormat(valueFormat, valueProjection, "value");
+ return new KafkaTopicSchema(
+ rowType,
+ keyFormat,
+ keyProjection,
+ valueFormat,
+ valueProjection,
+ timestampPosition,
+ headersPosition);
+ }
+
+ private static void validateTableKind(TableDescriptor table) {
+ if (table.hasPrimaryKey()) {
+ throw invalid("Kafka topic table must be a log table.");
+ }
+ if (table.isPartitioned()) {
+ throw invalid("Partitioned Fluss tables are not supported.");
+ }
+ if (new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat()
+ != LogFormat.ARROW) {
+ throw invalid("Kafka topic table must use the Arrow log format.");
+ }
+ }
+
+ private static List resolveKeyPositions(
+ RowType rowType, KafkaDataFormat keyFormat, String keyFieldsValue) {
+ if (keyFormat == null) {
+ if (keyFieldsValue != null) {
+ throw invalid(
+ KafkaDataFormat.KEY_FIELDS_CONFIG
+ + " requires "
+ + KafkaDataFormat.KEY_FORMAT_CONFIG
+ + ".");
+ }
+ return Collections.emptyList();
+ }
+ if (keyFieldsValue == null || keyFieldsValue.trim().isEmpty()) {
+ throw invalid(
+ "Missing required table property '" + KafkaDataFormat.KEY_FIELDS_CONFIG + "'.");
+ }
+ String[] fieldNames = keyFieldsValue.split(",", -1);
+ List positions = new ArrayList<>(fieldNames.length);
+ Set uniquePositions = new HashSet<>();
+ for (String fieldNameValue : fieldNames) {
+ String fieldName = fieldNameValue.trim();
+ if (fieldName.isEmpty()) {
+ throw invalid("Kafka key field names must not be empty.");
+ }
+ int position = rowType.getFieldIndex(fieldName);
+ if (position < 0) {
+ throw invalid("Kafka key field '" + fieldName + "' does not exist.");
+ }
+ if (!uniquePositions.add(position)) {
+ throw invalid("Duplicate Kafka key field '" + fieldName + "'.");
+ }
+ positions.add(position);
+ }
+ return positions;
+ }
+
+ private static String normalizeFieldsInclude(String value) {
+ String normalized = value == null ? INCLUDE_ALL : value.trim().toUpperCase(Locale.ROOT);
+ if (!INCLUDE_ALL.equals(normalized) && !INCLUDE_EXCEPT_KEY.equals(normalized)) {
+ throw invalid(
+ "Invalid "
+ + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG
+ + " '"
+ + value
+ + "'. Expected ALL or EXCEPT_KEY.");
+ }
+ return normalized;
+ }
+
+ private static List resolveValuePositions(
+ RowType rowType,
+ String fieldsInclude,
+ List keyPositions,
+ Set metadataPositions) {
+ Set excludedKeyPositions =
+ INCLUDE_EXCEPT_KEY.equals(fieldsInclude)
+ ? new HashSet<>(keyPositions)
+ : Collections.emptySet();
+ List positions = new ArrayList<>();
+ for (int position = 0; position < rowType.getFieldCount(); position++) {
+ if (!metadataPositions.contains(position) && !excludedKeyPositions.contains(position)) {
+ positions.add(position);
+ }
+ }
+ return positions;
+ }
+
+ private static int resolveOptionalPosition(RowType rowType, String fieldNameValue) {
+ if (fieldNameValue == null) {
+ return -1;
+ }
+ if (fieldNameValue.trim().isEmpty()) {
+ throw invalid("Kafka metadata column name must not be empty.");
+ }
+ String fieldName = fieldNameValue.trim();
+ int position = rowType.getFieldIndex(fieldName);
+ if (position < 0) {
+ throw invalid("Kafka metadata column '" + fieldName + "' does not exist.");
+ }
+ return position;
+ }
+
+ private static void validateMetadataColumns(
+ RowType rowType, int timestampPosition, int headersPosition) {
+ if (timestampPosition >= 0) {
+ if (!(rowType.getTypeAt(timestampPosition) instanceof LocalZonedTimestampType)
+ || rowType.getTypeAt(timestampPosition).isNullable()
+ || ((LocalZonedTimestampType) rowType.getTypeAt(timestampPosition))
+ .getPrecision()
+ != 3) {
+ throw invalid("Kafka timestamp column must be TIMESTAMP_LTZ(3) NOT NULL.");
+ }
+ }
+ if (headersPosition >= 0) {
+ validateHeadersType(rowType.getTypeAt(headersPosition));
+ }
+ }
+
+ private static void validateHeadersType(org.apache.fluss.types.DataType dataType) {
+ if (!(dataType instanceof ArrayType) || !dataType.isNullable()) {
+ throw invalid(
+ "Kafka headers column must be nullable "
+ + "ARRAY>.");
+ }
+ ArrayType arrayType = (ArrayType) dataType;
+ if (!(arrayType.getElementType() instanceof RowType)) {
+ throw invalid("Kafka headers elements must be rows.");
+ }
+ RowType headerType = (RowType) arrayType.getElementType();
+ if (!headerType.getFieldNames().equals(Arrays.asList("name", "value"))
+ || !(headerType.getTypeAt(0) instanceof StringType)
+ || !(headerType.getTypeAt(1) instanceof BytesType)
+ || !headerType.getTypeAt(1).isNullable()) {
+ throw invalid("Kafka headers elements must be ROW.");
+ }
+ }
+
+ private static void validateSingleFieldFormat(
+ KafkaDataFormat format, KafkaFieldProjection projection, String component) {
+ if (format == null) {
+ return;
+ }
+ if (format == KafkaDataFormat.RAW || format == KafkaDataFormat.STRING) {
+ if (projection.size() != 1) {
+ throw invalid(
+ "Kafka "
+ + component
+ + " format "
+ + format.value()
+ + " requires exactly one Fluss field.");
+ }
+ boolean validType =
+ format == KafkaDataFormat.RAW
+ ? projection.dataTypeAt(0) instanceof BytesType
+ : projection.dataTypeAt(0) instanceof StringType;
+ if (!validType) {
+ throw invalid(
+ "Kafka "
+ + component
+ + " field '"
+ + projection.nameAt(0)
+ + "' must be "
+ + (format == KafkaDataFormat.RAW ? "BYTES" : "STRING")
+ + " for format "
+ + format.value()
+ + ".");
+ }
+ }
+ }
+
+ private static KafkaDataFormat parseFormat(String value) {
+ try {
+ return KafkaDataFormat.parse(value);
+ } catch (IllegalArgumentException e) {
+ throw invalid(e.getMessage());
+ }
+ }
+
+ private static KafkaTopicSchemaException invalid(String message) {
+ return new KafkaTopicSchemaException(message);
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java
new file mode 100644
index 00000000000..717f5cd33ff
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.transcode;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record;
+import org.apache.fluss.kafka.format.KafkaDataFormat;
+import org.apache.fluss.kafka.schema.KafkaTopicSchema;
+import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.record.bytesview.BytesView;
+import org.apache.fluss.row.BinaryString;
+import org.apache.fluss.row.GenericRow;
+
+import javax.annotation.Nullable;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.fluss.utils.Preconditions.checkArgument;
+
+/** Converts raw/string Kafka records using the DDL mapping into owned Fluss Arrow log bytes. */
+@Internal
+public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder {
+ private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver();
+ private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder();
+
+ @Override
+ public BytesView transcode(List records, TableInfo tableInfo) throws Exception {
+ checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition.");
+ KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor());
+ KafkaRowAssembler assembler = new KafkaRowAssembler(schema);
+ List rows = new ArrayList<>(records.size());
+ for (Record record : records) {
+ Object[] key =
+ schema.keyFormat() == null
+ ? new Object[0]
+ : new Object[] {decode(schema.keyFormat(), record.key())};
+ rows.add(
+ assembler.assemble(
+ key,
+ new Object[] {decode(schema.valueFormat(), record.value())},
+ record.timestamp(),
+ record.headers()));
+ }
+ return arrowRecordEncoder.encode(rows, tableInfo);
+ }
+
+ private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) {
+ if (bytes == null || format == KafkaDataFormat.RAW) {
+ return bytes;
+ }
+ if (format != KafkaDataFormat.STRING) {
+ throw new KafkaRecordEncodingException("Unsupported Kafka data format: " + format);
+ }
+ try {
+ return BinaryString.fromString(
+ StandardCharsets.UTF_8
+ .newDecoder()
+ .onMalformedInput(CodingErrorAction.REPORT)
+ .onUnmappableCharacter(CodingErrorAction.REPORT)
+ .decode(ByteBuffer.wrap(bytes))
+ .toString());
+ } catch (CharacterCodingException e) {
+ throw new KafkaRecordEncodingException("Kafka string field is not valid UTF-8.", e);
+ }
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java
new file mode 100644
index 00000000000..f74680ccaa2
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.transcode;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.memory.UnmanagedPagedOutputView;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.record.ChangeType;
+import org.apache.fluss.record.MemoryLogRecordsArrowBuilder;
+import org.apache.fluss.record.bytesview.BytesView;
+import org.apache.fluss.row.GenericRow;
+import org.apache.fluss.row.arrow.ArrowWriter;
+import org.apache.fluss.row.arrow.ArrowWriterPool;
+import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
+import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator;
+
+import java.util.List;
+
+/** Encodes assembled physical Fluss rows into one native Arrow log batch. */
+@Internal
+public final class FlussArrowRecordEncoder {
+
+ private static final int INITIAL_PAGE_SIZE = 4096;
+
+ /** Encodes all rows using the table's current schema ID and Arrow compression settings. */
+ public BytesView encode(List rows, TableInfo tableInfo) throws Exception {
+ try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
+ ArrowWriterPool provider = new ArrowWriterPool(allocator)) {
+ ArrowWriter writer =
+ provider.getOrCreateWriter(
+ tableInfo.getTableId(),
+ tableInfo.getSchemaId(),
+ Integer.MAX_VALUE,
+ tableInfo.getRowType(),
+ tableInfo.getTableConfig().getArrowCompressionInfo());
+ long epoch = writer.getEpoch();
+ try {
+ MemoryLogRecordsArrowBuilder builder =
+ MemoryLogRecordsArrowBuilder.builder(
+ tableInfo.getSchemaId(),
+ writer,
+ new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE),
+ true,
+ null);
+ for (GenericRow row : rows) {
+ builder.append(ChangeType.APPEND_ONLY, row);
+ }
+ // The output view owns heap pages. The result remains valid after the Arrow
+ // writer and allocator close, including across asynchronous append completion.
+ return builder.build();
+ } finally {
+ writer.recycle(epoch);
+ }
+ }
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java
new file mode 100644
index 00000000000..b48dd9d894f
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.transcode;
+
+import org.apache.fluss.annotation.Internal;
+
+/** Indicates that Kafka record bytes cannot be decoded using the configured data format. */
+@Internal
+public final class KafkaRecordEncodingException extends IllegalArgumentException {
+
+ /** Creates a record encoding exception. */
+ public KafkaRecordEncodingException(String message) {
+ super(message);
+ }
+
+ /** Creates a record encoding exception. */
+ public KafkaRecordEncodingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java
new file mode 100644
index 00000000000..eeff2f3d14f
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.transcode;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.record.bytesview.BytesView;
+
+import java.util.List;
+
+/** Converts copied Kafka records into the native Fluss log representation. */
+@Internal
+public interface KafkaRecordTranscoder {
+ /** Transcodes records according to the target Fluss table schema and log format. */
+ BytesView transcode(List records, TableInfo tableInfo) throws Exception;
+}
diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java
new file mode 100644
index 00000000000..4ebd506dc3c
--- /dev/null
+++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.transcode;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader;
+import org.apache.fluss.kafka.schema.KafkaFieldProjection;
+import org.apache.fluss.kafka.schema.KafkaTopicSchema;
+import org.apache.fluss.row.BinaryString;
+import org.apache.fluss.row.GenericArray;
+import org.apache.fluss.row.GenericRow;
+import org.apache.fluss.row.TimestampLtz;
+
+import java.util.List;
+
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
+
+/** Assembles decoded Kafka key, value, and metadata into a physical Fluss row. */
+@Internal
+public final class KafkaRowAssembler {
+
+ private final KafkaTopicSchema topicSchema;
+
+ /** Creates an assembler for the resolved topic schema. */
+ public KafkaRowAssembler(KafkaTopicSchema topicSchema) {
+ this.topicSchema = checkNotNull(topicSchema);
+ }
+
+ /** Assembles one Fluss row. */
+ public GenericRow assemble(
+ Object[] keyValues, Object[] valueValues, long timestamp, List headers) {
+ GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount());
+ setProjectedFields(row, topicSchema.keyProjection(), keyValues);
+ setProjectedFields(row, topicSchema.valueProjection(), valueValues);
+ if (topicSchema.timestampPosition() >= 0) {
+ row.setField(topicSchema.timestampPosition(), TimestampLtz.fromEpochMillis(timestamp));
+ }
+ if (topicSchema.headersPosition() >= 0) {
+ row.setField(topicSchema.headersPosition(), toHeaders(headers));
+ }
+ return row;
+ }
+
+ private static void setProjectedFields(
+ GenericRow row, KafkaFieldProjection projection, Object[] values) {
+ if (values.length != projection.size()) {
+ throw new IllegalArgumentException(
+ "Kafka decoder returned "
+ + values.length
+ + " fields for a projection of "
+ + projection.size()
+ + ".");
+ }
+ for (int i = 0; i < values.length; i++) {
+ Object value = values[i];
+ if (value == null && !projection.dataTypeAt(i).isNullable()) {
+ throw new KafkaRecordEncodingException(
+ "Kafka record cannot populate NOT NULL field '"
+ + projection.nameAt(i)
+ + "' with null.");
+ }
+ row.setField(projection.positionAt(i), value);
+ }
+ }
+
+ private static GenericArray toHeaders(List headers) {
+ Object[] rows = new Object[headers.size()];
+ for (int i = 0; i < headers.size(); i++) {
+ RecordHeader header = headers.get(i);
+ rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.value());
+ }
+ return new GenericArray(rows);
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java
new file mode 100644
index 00000000000..a2b8a0dc60b
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka;
+
+import org.apache.fluss.rpc.netty.server.RequestChannel;
+import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf;
+import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled;
+import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel;
+
+import org.apache.kafka.common.message.ApiVersionsRequestData;
+import org.apache.kafka.common.message.ApiVersionsResponseData;
+import org.apache.kafka.common.message.ProduceRequestData;
+import org.apache.kafka.common.message.ProduceResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.ApiVersionsRequest;
+import org.apache.kafka.common.requests.ApiVersionsResponse;
+import org.apache.kafka.common.requests.ProduceRequest;
+import org.apache.kafka.common.requests.ProduceResponse;
+import org.apache.kafka.common.requests.RequestHeader;
+import org.apache.kafka.common.requests.RequestUtils;
+import org.apache.kafka.common.requests.ResponseHeader;
+import org.junit.jupiter.api.Test;
+
+import java.nio.ByteBuffer;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests response ordering and ownership in {@link KafkaCommandDecoder}. */
+public class KafkaCommandDecoderTest {
+
+ @Test
+ public void testAcksZeroSuppressesResponseAndUnblocksFollowingResponse() {
+ RequestChannel requestChannel = new RequestChannel(100);
+ EmbeddedChannel channel =
+ new EmbeddedChannel(
+ new KafkaCommandDecoder(new RequestChannel[] {requestChannel}, "KAFKA"));
+ short produceVersion = ApiKeys.PRODUCE.latestVersion();
+ ProduceRequest produceRequest =
+ new ProduceRequest(
+ new ProduceRequestData().setAcks((short) 0).setTimeoutMs(1000),
+ produceVersion);
+ RequestHeader produceHeader =
+ new RequestHeader(ApiKeys.PRODUCE, produceVersion, "client", 1);
+ ByteBuf produceBuffer = serialize(produceHeader, produceRequest);
+
+ short apiVersionsVersion = ApiKeys.API_VERSIONS.latestVersion();
+ ApiVersionsRequest apiVersionsRequest =
+ new ApiVersionsRequest.Builder(
+ new ApiVersionsRequestData(),
+ apiVersionsVersion,
+ apiVersionsVersion)
+ .build();
+ RequestHeader apiVersionsHeader =
+ new RequestHeader(ApiKeys.API_VERSIONS, apiVersionsVersion, "client", 2);
+ ByteBuf apiVersionsBuffer = serialize(apiVersionsHeader, apiVersionsRequest);
+
+ try {
+ channel.writeInbound(produceBuffer);
+ channel.writeInbound(apiVersionsBuffer);
+ KafkaRequest first = (KafkaRequest) requestChannel.pollRequest(1000);
+ KafkaRequest second = (KafkaRequest) requestChannel.pollRequest(1000);
+ assertThat(first).isNotNull();
+ assertThat(second).isNotNull();
+
+ second.complete(new ApiVersionsResponse(new ApiVersionsResponseData()));
+ channel.runPendingTasks();
+ Object blockedResponse = channel.readOutbound();
+ assertThat(blockedResponse).isNull();
+
+ first.complete(new ProduceResponse(new ProduceResponseData()));
+ channel.runPendingTasks();
+
+ ByteBuf response = channel.readOutbound();
+ try {
+ assertThat(response).isNotNull();
+ ResponseHeader responseHeader =
+ ResponseHeader.parse(
+ response.nioBuffer(),
+ apiVersionsHeader.toResponseHeader().headerVersion());
+ assertThat(responseHeader.correlationId()).isEqualTo(2);
+ Object additionalResponse = channel.readOutbound();
+ assertThat(additionalResponse).isNull();
+ } finally {
+ if (response != null) {
+ response.release();
+ }
+ }
+
+ assertThat(produceBuffer.refCnt()).isZero();
+ assertThat(apiVersionsBuffer.refCnt()).isZero();
+ } finally {
+ channel.finishAndReleaseAll();
+ }
+ }
+
+ private static ByteBuf serialize(RequestHeader header, AbstractRequest request) {
+ ByteBuffer serialized =
+ RequestUtils.serialize(
+ header.data(), header.headerVersion(), request.data(), request.version());
+ return Unpooled.wrappedBuffer(serialized);
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java
new file mode 100644
index 00000000000..2614b2df3fd
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java
@@ -0,0 +1,602 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka;
+
+import org.apache.fluss.exception.TableNotExistException;
+import org.apache.fluss.kafka.format.KafkaDataFormat;
+import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.rpc.TestingTabletGatewayService;
+import org.apache.fluss.rpc.messages.ListTablesRequest;
+import org.apache.fluss.rpc.messages.ListTablesResponse;
+import org.apache.fluss.rpc.messages.PbBucketMetadata;
+import org.apache.fluss.rpc.messages.PbServerNode;
+import org.apache.fluss.rpc.messages.PbTableMetadata;
+import org.apache.fluss.rpc.messages.PbTablePath;
+import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf;
+import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator;
+import org.apache.fluss.types.DataTypes;
+
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.message.MetadataRequestData;
+import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic;
+import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition;
+import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.protocol.types.RawTaggedField;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.MetadataRequest;
+import org.apache.kafka.common.requests.MetadataResponse;
+import org.apache.kafka.common.requests.RequestHeader;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Protocol compatibility tests for the Kafka Metadata API. */
+public class KafkaMetadataHandlerTest {
+
+ private static final Uuid TOPIC_ID = new Uuid(0x466c757373000000L, 123L);
+
+ @Test
+ public void testNamedTopicForEverySupportedVersion() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) {
+ MetadataRequest request =
+ new MetadataRequest(
+ new MetadataRequestData()
+ .setTopics(
+ MetadataRequest.convertToMetadataRequestTopic(
+ Collections.singletonList("topic"))),
+ version);
+ if (version >= 9) {
+ request.data()
+ .unknownTaggedFields()
+ .add(new RawTaggedField(100, new byte[] {1, 2, 3}));
+ request.data()
+ .topics()
+ .get(0)
+ .unknownTaggedFields()
+ .add(new RawTaggedField(101, new byte[] {4, 5, 6}));
+ }
+ MetadataResponse response = handle(service, request, version);
+
+ assertThat(response.brokers()).hasSize(2);
+ assertThat(response.controller()).isNull();
+ Node broker = response.brokers().iterator().next();
+ assertThat(broker.host()).isEqualTo("broker-1");
+ assertThat(broker.port()).isEqualTo(9092);
+ assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null);
+ MetadataResponseTopic topic = response.data().topics().find("topic");
+ assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code());
+ assertThat(topic.partitions()).hasSize(2);
+ assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID);
+ MetadataResponsePartition partition = topic.partitions().get(0);
+ assertThat(partition.partitionIndex()).isZero();
+ assertThat(partition.leaderId()).isEqualTo(1);
+ assertThat(partition.replicaNodes()).containsExactly(1, 2);
+ assertThat(partition.isrNodes()).containsExactly(1, 2);
+ assertThat(partition.offlineReplicas()).isEmpty();
+ }
+ assertThat(service.lastListenerName).isEqualTo("KAFKA");
+ }
+
+ @Test
+ public void testAllTopicsForEverySupportedVersion() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) {
+ MetadataRequest request = allTopicsRequest(version);
+ if (version >= 9) {
+ request.data()
+ .unknownTaggedFields()
+ .add(new RawTaggedField(102, new byte[] {7, 8, 9}));
+ }
+
+ MetadataResponse response = handle(service, request, version);
+
+ assertThat(response.data().topics())
+ .extracting(MetadataResponseTopic::name)
+ .containsExactly("other", "topic");
+ }
+ }
+
+ @Test
+ public void testAllTopicsAndMissingAndInvalidTopic() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+
+ MetadataResponse allTopics =
+ handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9);
+ assertThat(allTopics.data().topics())
+ .extracting(MetadataResponseTopic::name)
+ .containsExactlyInAnyOrder("other", "topic");
+
+ MetadataRequest requestedTopics =
+ new MetadataRequest.Builder(Arrays.asList("missing", "invalid topic"), false)
+ .build((short) 9);
+ MetadataResponse errors = handle(service, requestedTopics, (short) 9);
+ assertThat(errors.errors())
+ .containsEntry("missing", Errors.UNKNOWN_TOPIC_OR_PARTITION)
+ .containsEntry("invalid topic", Errors.INVALID_TOPIC_EXCEPTION);
+ }
+
+ @Test
+ public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ for (short version = 10; version <= 11; version++) {
+ MetadataRequest request =
+ new MetadataRequest(
+ new MetadataRequestData()
+ .setTopics(
+ Collections.singletonList(
+ new MetadataRequestTopic()
+ .setName("topic")
+ .setTopicId(
+ new Uuid(
+ 0x466c757373000000L,
+ 999L)))),
+ version);
+
+ MetadataResponse response = handle(service, request, version);
+
+ assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE);
+ assertThat(response.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID);
+ }
+ }
+
+ @Test
+ public void testV10AndV11RejectTopicIdOnlyLookup() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ Uuid requestedTopicId = new Uuid(0x466c757373000000L, 999L);
+ for (short version = 10; version <= 11; version++) {
+ MetadataRequest request =
+ new MetadataRequest(
+ new MetadataRequestData()
+ .setTopics(
+ Collections.singletonList(
+ new MetadataRequestTopic()
+ .setName(null)
+ .setTopicId(requestedTopicId))),
+ version);
+
+ MetadataResponse response = handle(service, request, version);
+
+ assertThat(response.data().topics()).hasSize(1);
+ MetadataResponseTopic responseTopic = response.data().topics().iterator().next();
+ assertThat(responseTopic.name()).isEmpty();
+ assertThat(responseTopic.topicId()).isEqualTo(requestedTopicId);
+ assertThat(responseTopic.errorCode()).isEqualTo(Errors.INVALID_REQUEST.code());
+ }
+ }
+
+ @Test
+ public void testTopicIdentityAcrossDeleteAndRecreate() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+
+ MetadataResponse initial = handle(service, namedTopicRequest("topic"), (short) 11);
+ assertThat(initial.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID);
+
+ service.removeTable("topic");
+ MetadataResponse deleted = handle(service, namedTopicRequest("topic"), (short) 11);
+ assertThat(deleted.errorCounts())
+ .containsExactlyEntriesOf(
+ Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1));
+
+ service.putTable("topic", 223L);
+ Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L);
+ MetadataResponse recreatedByName =
+ handle(
+ service,
+ new MetadataRequest.Builder(Collections.singletonList("topic"), false)
+ .build((short) 11),
+ (short) 11);
+ assertThat(recreatedByName.data().topics().find("topic").topicId())
+ .isEqualTo(recreatedTopicId)
+ .isNotEqualTo(TOPIC_ID);
+ }
+
+ @Test
+ public void testDeleteRaceBecomesUnknownTopicResult() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ service.removeTable("topic");
+ service.failNextMetadataAsMissing = true;
+
+ MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11);
+
+ assertThat(response.errorCounts())
+ .containsExactlyEntriesOf(
+ Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1));
+ }
+
+ @Test
+ public void testUnavailableLeaderUsesPartitionError() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ service.topicLeaderAvailable = false;
+ MetadataRequest request =
+ new MetadataRequest.Builder(Collections.singletonList("topic"), false)
+ .build((short) 11);
+
+ MetadataResponse response = handle(service, request, (short) 11);
+
+ MetadataResponseTopic topic = response.data().topics().find("topic");
+ assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code());
+ MetadataResponsePartition partition = topic.partitions().get(0);
+ assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code());
+ assertThat(partition.leaderId()).isEqualTo(-1);
+ assertThat(partition.replicaNodes()).containsExactly(1, 2, 3);
+ assertThat(partition.isrNodes()).containsExactly(1, 2);
+ assertThat(partition.offlineReplicas()).containsExactly(3);
+ }
+
+ @Test
+ public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ service.topicIsr = new int[] {1};
+ MetadataRequest request =
+ new MetadataRequest.Builder(Collections.singletonList("topic"), false)
+ .build((short) 11);
+
+ MetadataResponse response = handle(service, request, (short) 11);
+
+ MetadataResponsePartition partition =
+ response.data().topics().find("topic").partitions().get(0);
+ assertThat(partition.replicaNodes()).containsExactly(1, 2);
+ assertThat(partition.isrNodes()).containsExactly(1);
+ assertThat(partition.offlineReplicas()).isEmpty();
+ }
+
+ @Test
+ public void testLegacyMetadataUsesLeaderOnlyIsr() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ service.topicBucketEpoch = null;
+
+ MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11);
+
+ MetadataResponsePartition partition =
+ response.data().topics().find("topic").partitions().get(0);
+ assertThat(partition.leaderId()).isEqualTo(1);
+ assertThat(partition.replicaNodes()).containsExactly(1, 2);
+ assertThat(partition.isrNodes()).containsExactly(1);
+ assertThat(partition.offlineReplicas()).isEmpty();
+ }
+
+ @Test
+ public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ service.topicBucketEpoch = null;
+ service.topicLeaderAvailable = false;
+
+ MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11);
+
+ MetadataResponsePartition partition =
+ response.data().topics().find("topic").partitions().get(0);
+ assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code());
+ assertThat(partition.isrNodes()).isEmpty();
+ }
+
+ @Test
+ public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() {
+ for (int bucketEpoch : new int[] {7, -1}) {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ service.topicBucketEpoch = bucketEpoch;
+ service.topicIsr = new int[0];
+
+ MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11);
+
+ MetadataResponsePartition partition =
+ response.data().topics().find("topic").partitions().get(0);
+ assertThat(partition.isrNodes()).isEmpty();
+ }
+ }
+
+ @Test
+ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ service.failMetadata = true;
+ MetadataRequest request =
+ new MetadataRequest.Builder(Collections.singletonList("topic"), false)
+ .build((short) 11);
+
+ MetadataResponse response = handle(service, request, (short) 11);
+
+ assertThat(response.errorCounts())
+ .containsExactlyEntriesOf(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 1));
+ assertThat(response.brokers()).isEmpty();
+ }
+
+ @Test
+ public void testMetadataUsesDdlContractForEveryVersion() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ service.putTable(
+ "no_mapping",
+ 125L,
+ TableDescriptor.builder()
+ .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build())
+ .distributedBy(2)
+ .build());
+ service.putTable(
+ "bad_format",
+ 126L,
+ TableDescriptor.builder(defaultDescriptor())
+ .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json")
+ .build());
+ service.putTable(
+ "primary_key",
+ 127L,
+ TableDescriptor.builder(defaultDescriptor())
+ .schema(
+ Schema.newBuilder()
+ .column("body", DataTypes.BYTES().copy(false))
+ .primaryKey("body")
+ .build())
+ .build());
+ service.putTable(
+ "partitioned",
+ 128L,
+ TableDescriptor.builder(defaultDescriptor()).partitionedBy("body").build());
+ service.putTable(
+ "indexed",
+ 129L,
+ TableDescriptor.builder(defaultDescriptor()).logFormat(LogFormat.INDEXED).build());
+ service.putTable("invalid topic", 130L);
+ List invalidMappings =
+ Arrays.asList("no_mapping", "bad_format", "primary_key", "partitioned", "indexed");
+ for (short version = 0; version <= 11; version++) {
+ MetadataResponse all = handle(service, allTopicsRequest(version), version);
+ assertThat(all.data().topics())
+ .extracting(MetadataResponseTopic::name)
+ .containsExactly("other", "topic");
+ List requested = new ArrayList<>(invalidMappings);
+ requested.add("topic");
+ requested.add("missing");
+ MetadataResponse named =
+ handle(
+ service,
+ new MetadataRequest(
+ new MetadataRequestData()
+ .setTopics(
+ MetadataRequest.convertToMetadataRequestTopic(
+ requested)),
+ version),
+ version);
+ for (String name : invalidMappings) {
+ MetadataResponseTopic topic = named.data().topics().find(name);
+ assertThat(topic.errorCode()).isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code());
+ assertThat(topic.partitions()).isEmpty();
+ }
+ assertThat(named.data().topics().find("topic").errorCode())
+ .isEqualTo(Errors.NONE.code());
+ assertThat(named.data().topics().find("missing").errorCode())
+ .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code());
+ }
+ }
+
+ @Test
+ public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() {
+ TestingMetadataGatewayService service = new TestingMetadataGatewayService();
+ service.putTable(
+ "topic",
+ 123L,
+ TableDescriptor.builder(defaultDescriptor())
+ .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string")
+ .build());
+ MetadataResponse invalid = handle(service, namedTopicRequest("topic"), (short) 11);
+ assertThat(invalid.data().topics().find("topic").errorCode())
+ .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code());
+ service.putTable("topic", 123L);
+ MetadataResponse valid = handle(service, namedTopicRequest("topic"), (short) 11);
+ assertThat(valid.data().topics().find("topic").errorCode()).isEqualTo(Errors.NONE.code());
+ assertThat(valid.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID);
+ }
+
+ private static TableDescriptor defaultDescriptor() {
+ return TableDescriptor.builder()
+ .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build())
+ .distributedBy(2)
+ .logFormat(LogFormat.ARROW)
+ .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw")
+ .build();
+ }
+
+ private static MetadataRequest namedTopicRequest(String topicName) {
+ return new MetadataRequest(
+ new MetadataRequestData()
+ .setTopics(
+ Collections.singletonList(
+ new MetadataRequestTopic()
+ .setName(topicName)
+ .setTopicId(Uuid.ZERO_UUID))),
+ (short) 11);
+ }
+
+ private static MetadataRequest allTopicsRequest(short version) {
+ MetadataRequestData data = new MetadataRequestData();
+ data.setTopics(version == 0 ? Collections.emptyList() : null);
+ return new MetadataRequest(data, version);
+ }
+
+ private static MetadataResponse handle(
+ TestingMetadataGatewayService service, MetadataRequest requestBody, short version) {
+ KafkaRequestHandler handler = new KafkaRequestHandler(service, service, "kafka");
+ ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer();
+ KafkaRequest request;
+ try {
+ request =
+ new KafkaRequest(
+ ApiKeys.METADATA,
+ version,
+ new RequestHeader(ApiKeys.METADATA, version, "client-id", 1),
+ requestBody,
+ "KAFKA",
+ requestBuffer,
+ new TestingChannelHandlerContext(),
+ new CompletableFuture<>());
+ } finally {
+ // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest.
+ requestBuffer.release();
+ }
+ handler.processRequest(request);
+ ByteBuf responseBuffer = request.responseBuffer();
+ try {
+ return (MetadataResponse)
+ AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header());
+ } finally {
+ responseBuffer.release();
+ assertThat(requestBuffer.refCnt()).isZero();
+ }
+ }
+
+ private static final class TestingMetadataGatewayService extends TestingTabletGatewayService {
+
+ private final Map tables = new LinkedHashMap<>();
+ private final Map descriptors = new LinkedHashMap<>();
+ private String lastListenerName;
+ private boolean topicLeaderAvailable = true;
+ private int[] topicIsr = new int[] {1, 2};
+ private Integer topicBucketEpoch = 7;
+ private boolean failMetadata;
+ private boolean failNextMetadataAsMissing;
+
+ private TestingMetadataGatewayService() {
+ putTable("topic", 123L);
+ putTable("other", 124L);
+ }
+
+ @Override
+ public CompletableFuture listTables(ListTablesRequest request) {
+ assertThat(request.getDatabaseName()).isEqualTo("kafka");
+ return CompletableFuture.completedFuture(
+ new ListTablesResponse().addAllTableNames(new ArrayList<>(tables.keySet())));
+ }
+
+ @Override
+ public CompletableFuture metadata(
+ org.apache.fluss.rpc.messages.MetadataRequest request) {
+ lastListenerName = currentListenerName();
+ if (failMetadata) {
+ CompletableFuture failure =
+ new CompletableFuture<>();
+ failure.completeExceptionally(new IllegalStateException("metadata unavailable"));
+ return failure;
+ }
+ if (failNextMetadataAsMissing) {
+ failNextMetadataAsMissing = false;
+ throw new TableNotExistException("table was deleted");
+ }
+ List topics = new ArrayList<>();
+ for (PbTablePath tablePath : request.getTablePathsList()) {
+ Long tableId = tables.get(tablePath.getTableName());
+ if (tableId != null) {
+ topics.add(
+ tableMetadata(
+ tablePath.getTableName(),
+ tableId,
+ !"topic".equals(tablePath.getTableName())
+ || topicLeaderAvailable,
+ "topic".equals(tablePath.getTableName())
+ ? topicIsr
+ : new int[] {1, 2},
+ "topic".equals(tablePath.getTableName())
+ ? topicBucketEpoch
+ : Integer.valueOf(7))
+ .setTableJson(
+ descriptors
+ .get(tablePath.getTableName())
+ .toJsonBytes()));
+ }
+ }
+ return CompletableFuture.completedFuture(
+ new org.apache.fluss.rpc.messages.MetadataResponse()
+ .addAllTabletServers(
+ Arrays.asList(
+ new PbServerNode()
+ .setNodeId(1)
+ .setHost("broker-1")
+ .setPort(9092)
+ .setRack("rack-a"),
+ new PbServerNode()
+ .setNodeId(2)
+ .setHost("broker-2")
+ .setPort(9093)))
+ .addAllTableMetadatas(topics));
+ }
+
+ private void putTable(String topic, long tableId) {
+ putTable(topic, tableId, defaultDescriptor());
+ }
+
+ private void putTable(String topic, long tableId, TableDescriptor descriptor) {
+ tables.put(topic, tableId);
+ descriptors.put(topic, descriptor);
+ }
+
+ private void removeTable(String topic) {
+ tables.remove(topic);
+ descriptors.remove(topic);
+ }
+
+ private static PbTableMetadata tableMetadata(
+ String topic,
+ long tableId,
+ boolean leaderAvailable,
+ int[] isr,
+ Integer bucketEpoch) {
+ PbTableMetadata table =
+ new PbTableMetadata()
+ .setTablePath(
+ new PbTablePath().setDatabaseName("kafka").setTableName(topic))
+ .setTableId(tableId)
+ .addAllBucketMetadatas(
+ Arrays.asList(
+ new PbBucketMetadata()
+ .setBucketId(0)
+ .setLeaderId(leaderAvailable ? 1 : 3)
+ .setLeaderEpoch(5)
+ .setReplicaIds(
+ leaderAvailable
+ ? new int[] {1, 2}
+ : new int[] {1, 2, 3})
+ .setIsrs(
+ bucketEpoch == null ? new int[0] : isr),
+ new PbBucketMetadata()
+ .setBucketId(1)
+ .setLeaderId(2)
+ .setLeaderEpoch(6)
+ .setReplicaIds(new int[] {1, 2})
+ .setIsrs(
+ bucketEpoch == null
+ ? new int[0]
+ : new int[] {1, 2})));
+ if (bucketEpoch != null) {
+ for (PbBucketMetadata bucket : table.getBucketMetadatasList()) {
+ bucket.setBucketEpoch(bucketEpoch);
+ }
+ }
+ return table;
+ }
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java
new file mode 100644
index 00000000000..bf043823689
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka;
+
+import org.apache.fluss.kafka.api.produce.ProduceHandler;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceResult;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult;
+import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry;
+import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher;
+import org.apache.fluss.kafka.error.KafkaErrorMapper;
+import org.apache.fluss.rpc.netty.server.RequestChannel;
+import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf;
+import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled;
+import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel;
+
+import org.apache.kafka.common.compress.Compression;
+import org.apache.kafka.common.errors.TimeoutException;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+import org.apache.kafka.common.message.ProduceRequestData;
+import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData;
+import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData;
+import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.record.MemoryRecords;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.common.record.SimpleRecord;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.ProduceRequest;
+import org.apache.kafka.common.requests.ProduceResponse;
+import org.apache.kafka.common.requests.RequestHeader;
+import org.apache.kafka.common.requests.RequestUtils;
+import org.apache.kafka.common.requests.ResponseHeader;
+import org.junit.jupiter.api.Test;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Protocol tests independent of the Fluss append and record conversion backends. */
+class KafkaProduceProtocolTest {
+
+ @Test
+ void testSupportedVersionsCopyRecordsAndForwardContext() {
+ for (short version = 3; version <= 11; version++) {
+ AtomicReference copied = new AtomicReference<>();
+ ProduceResponse response =
+ dispatch(
+ request(version, (short) -1, partition(0, records())),
+ command -> {
+ copied.set(command);
+ return successful(command);
+ });
+ assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE);
+ assertThat(
+ response.data()
+ .responses()
+ .find("topic")
+ .partitionResponses()
+ .get(0)
+ .baseOffset())
+ .isEqualTo(42L);
+ KafkaProduceCommand command = copied.get();
+ assertThat(command.acks()).isEqualTo((short) -1);
+ assertThat(command.timeoutMs()).isEqualTo(4321);
+ assertThat(command.listenerName()).isEqualTo("KAFKA");
+ KafkaProduceCommand.Record record =
+ command.topics().get(0).partitions().get(0).records().get(0);
+ assertThat(record.timestamp()).isEqualTo(123L);
+ assertThat(record.key()).containsExactly((byte) 1);
+ assertThat(record.value()).containsExactly((byte) 2);
+ assertThat(record.headers()).hasSize(1);
+ assertThat(record.headers().get(0).value()).isNull();
+ byte[] key = record.key();
+ key[0] = 9;
+ assertThat(record.key()).containsExactly((byte) 1);
+ }
+ }
+
+ @Test
+ void testCorruptPartitionDoesNotSuppressValidPartition() {
+ MemoryRecords corrupt = records();
+ corrupt.buffer().put(corrupt.sizeInBytes() - 1, (byte) 99);
+ ProduceResponse response =
+ dispatch(
+ request(
+ (short) 11,
+ (short) 1,
+ partition(0, corrupt),
+ partition(1, records())),
+ command -> {
+ assertThat(command.topics().get(0).partitions()).hasSize(1);
+ assertThat(command.topics().get(0).partitions().get(0).partitionId())
+ .isEqualTo(1);
+ return successful(command);
+ });
+ assertErrors(response, Errors.CORRUPT_MESSAGE, Errors.NONE);
+ }
+
+ @Test
+ void testInvalidTopicIsIsolated() {
+ ProduceRequest request = request((short) 11, (short) 1, partition(0, records()));
+ request.data()
+ .topicData()
+ .add(
+ new TopicProduceData()
+ .setName("bad/name")
+ .setPartitionData(
+ Collections.singletonList(partition(0, records()))));
+ ProduceResponse response =
+ dispatch(
+ request,
+ command -> {
+ assertThat(command.topics()).hasSize(1);
+ return successful(command);
+ });
+ assertThat(response.errorCounts())
+ .containsEntry(Errors.NONE, 1)
+ .containsEntry(Errors.INVALID_TOPIC_EXCEPTION, 1);
+ }
+
+ @Test
+ void testEmptyNegativeAndIdempotentPartitionsAreRejectedLocally() {
+ MemoryRecords idempotent =
+ MemoryRecords.withIdempotentRecords(
+ Compression.NONE, 10L, (short) 0, 0, new SimpleRecord(new byte[] {1}));
+ ProduceResponse response =
+ dispatch(
+ request(
+ (short) 11,
+ (short) 1,
+ partition(0, MemoryRecords.EMPTY),
+ partition(-1, records()),
+ partition(2, idempotent),
+ partition(3, records())),
+ command -> {
+ assertThat(command.topics().get(0).partitions()).hasSize(1);
+ return successful(command);
+ });
+ assertErrors(
+ response,
+ Errors.INVALID_RECORD,
+ Errors.INVALID_REQUEST,
+ Errors.INVALID_REQUEST,
+ Errors.NONE);
+ }
+
+ @Test
+ void testRequestValidationNeverCallsBackend() {
+ KafkaProduceBackend unused =
+ command -> {
+ throw new AssertionError("Backend must not be called");
+ };
+ assertErrors(
+ dispatch(request((short) 11, (short) 2, partition(0, records())), unused),
+ Errors.INVALID_REQUIRED_ACKS);
+ ProduceRequest transactional = request((short) 11, (short) 1, partition(0, records()));
+ transactional.data().setTransactionalId("transaction");
+ assertErrors(
+ dispatch(new ProduceRequest(transactional.data(), transactional.version()), unused),
+ Errors.INVALID_REQUEST);
+ assertErrors(
+ dispatch(
+ request(
+ (short) 11,
+ (short) 1,
+ partition(0, records()),
+ partition(0, records())),
+ unused),
+ Errors.INVALID_REQUEST);
+ assertErrors(
+ dispatch(request((short) 2, (short) 1, partition(0, records())), unused),
+ Errors.UNSUPPORTED_VERSION);
+ }
+
+ @Test
+ void testBackendFailuresAndMissingResultsPreserveValidationErrors() {
+ for (boolean synchronous : Arrays.asList(true, false)) {
+ ProduceResponse response =
+ dispatch(
+ request(
+ (short) 11,
+ (short) 1,
+ partition(0, MemoryRecords.EMPTY),
+ partition(1, records())),
+ command -> {
+ if (synchronous) {
+ throw new TimeoutException("append timed out");
+ }
+ CompletableFuture failed =
+ new CompletableFuture<>();
+ failed.completeExceptionally(
+ new TimeoutException("append timed out"));
+ return failed;
+ });
+ assertErrors(response, Errors.INVALID_RECORD, Errors.REQUEST_TIMED_OUT);
+ }
+ assertErrors(
+ dispatch(
+ request((short) 11, (short) 1, partition(0, records())),
+ command ->
+ CompletableFuture.completedFuture(
+ new KafkaProduceResult(Collections.emptyList()))),
+ Errors.UNKNOWN_SERVER_ERROR);
+ }
+
+ @Test
+ void testAcksZeroSuccessAndFailureReleaseBufferWithoutSendingResponse() {
+ for (boolean failure : Arrays.asList(false, true)) {
+ RequestChannel requests = new RequestChannel(100);
+ EmbeddedChannel channel =
+ new EmbeddedChannel(
+ new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA"));
+ ProduceRequest body = request((short) 11, (short) 0, partition(0, records()));
+ RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, (short) 11, "producer", 1);
+ ByteBuf buffer =
+ Unpooled.wrappedBuffer(
+ RequestUtils.serialize(
+ header.data(),
+ header.headerVersion(),
+ body.data(),
+ body.version()));
+ CompletableFuture pending = new CompletableFuture<>();
+ try {
+ channel.writeInbound(buffer);
+ KafkaRequest parsed = (KafkaRequest) requests.pollRequest(1000);
+ dispatcher(command -> pending)
+ .dispatch(parsed)
+ .whenComplete(
+ (response, error) -> {
+ if (error == null) {
+ parsed.complete(response);
+ } else {
+ parsed.fail(error);
+ }
+ });
+ assertThat(parsed.future()).isNotDone();
+ if (failure) {
+ pending.completeExceptionally(new TimeoutException("failure"));
+ } else {
+ pending.complete(
+ new KafkaProduceResult(
+ Collections.singletonList(
+ new TopicResult(
+ "topic",
+ Collections.singletonList(
+ new PartitionResult(
+ 0, Errors.NONE, 42L, null))))));
+ }
+ channel.runPendingTasks();
+ assertThat(parsed.future()).isDone();
+ assertThat((Object) channel.readOutbound()).isNull();
+ assertThat(buffer.refCnt()).isZero();
+ } finally {
+ channel.finishAndReleaseAll();
+ }
+ }
+ }
+
+ private static ProduceResponse dispatch(ProduceRequest body, KafkaProduceBackend backend) {
+ ByteBuf buffer = Unpooled.buffer(1);
+ RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, body.version(), "producer", 1);
+ KafkaRequest request =
+ new KafkaRequest(
+ ApiKeys.PRODUCE,
+ body.version(),
+ header,
+ body,
+ "KAFKA",
+ buffer,
+ new TestingChannelHandlerContext(),
+ new CompletableFuture<>());
+ buffer.release();
+ request.complete(dispatcher(backend).dispatch(request).join());
+ ByteBuf response = request.responseBuffer();
+ try {
+ ByteBuffer bytes = response.nioBuffer();
+ ResponseHeader.parse(bytes, header.toResponseHeader().headerVersion());
+ return (ProduceResponse)
+ AbstractResponse.parseResponse(ApiKeys.PRODUCE, bytes, body.version());
+ } finally {
+ response.release();
+ }
+ }
+
+ private static KafkaRequestDispatcher dispatcher(KafkaProduceBackend backend) {
+ KafkaApiRegistry registry = new KafkaApiRegistry();
+ registry.register(new ProduceHandler(backend));
+ registry.freeze();
+ return new KafkaRequestDispatcher(registry, new KafkaErrorMapper());
+ }
+
+ private static CompletableFuture successful(KafkaProduceCommand command) {
+ List topics = new ArrayList<>();
+ for (KafkaProduceCommand.TopicWrite topic : command.topics()) {
+ List partitions = new ArrayList<>();
+ for (KafkaProduceCommand.PartitionWrite partition : topic.partitions()) {
+ partitions.add(
+ new PartitionResult(partition.partitionId(), Errors.NONE, 42L, null));
+ }
+ topics.add(new TopicResult(topic.topicName(), partitions));
+ }
+ return CompletableFuture.completedFuture(new KafkaProduceResult(topics));
+ }
+
+ private static ProduceRequest request(
+ short version, short acks, PartitionProduceData... partitions) {
+ TopicProduceData topic =
+ new TopicProduceData().setName("topic").setPartitionData(Arrays.asList(partitions));
+ return new ProduceRequest(
+ new ProduceRequestData()
+ .setAcks(acks)
+ .setTimeoutMs(4321)
+ .setTopicData(
+ new ProduceRequestData.TopicProduceDataCollection(
+ Collections.singletonList(topic).iterator())),
+ version);
+ }
+
+ private static PartitionProduceData partition(int index, MemoryRecords records) {
+ return new PartitionProduceData().setIndex(index).setRecords(records);
+ }
+
+ private static MemoryRecords records() {
+ return MemoryRecords.withRecords(
+ RecordBatch.MAGIC_VALUE_V2,
+ 0L,
+ Compression.NONE,
+ new SimpleRecord(
+ 123L,
+ new byte[] {1},
+ new byte[] {2},
+ new Header[] {new RecordHeader("header", null)}));
+ }
+
+ private static void assertErrors(ProduceResponse response, Errors... errors) {
+ assertThat(response.data().responses().find("topic").partitionResponses())
+ .extracting(PartitionProduceResponse::errorCode)
+ .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new));
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java
index 24e4ce8a6ce..c4476c69d59 100644
--- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java
@@ -22,18 +22,28 @@
import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator;
import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext;
+import org.apache.kafka.common.message.ApiVersionsRequestData;
+import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion;
+import org.apache.kafka.common.message.CreateTopicsRequestData;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.AbstractResponse;
import org.apache.kafka.common.requests.ApiVersionsRequest;
import org.apache.kafka.common.requests.ApiVersionsResponse;
+import org.apache.kafka.common.requests.CreateTopicsRequest;
+import org.apache.kafka.common.requests.CreateTopicsResponse;
import org.apache.kafka.common.requests.RequestHeader;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import java.util.Collections;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.tuple;
/** Tests for {@link KafkaRequestHandler}. */
public class KafkaRequestHandlerTest {
@@ -46,73 +56,161 @@ public void testKafkaApiVersionsNotSupported() {
new ApiVersionsRequest.Builder().build(latestVersion);
ChannelHandlerContext ctx = new TestingChannelHandlerContext();
KafkaRequest request =
- new KafkaRequest(
+ newRequest(
ApiKeys.API_VERSIONS,
(short) (latestVersion + 1), // unsupported version
new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0),
apiVersionsRequest,
- ByteBufAllocator.DEFAULT.buffer(),
- ctx,
- new CompletableFuture<>());
- handler.handleApiVersionsRequest(request);
+ ctx);
+ handler.processRequest(request);
- ByteBuf responseBuffer = request.responseBuffer();
- ApiVersionsResponse response =
- (ApiVersionsResponse)
- AbstractResponse.parseResponse(
- responseBuffer.nioBuffer(), request.header());
+ ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request);
Map errorCounts = response.errorCounts();
assertThat(1).isEqualTo(errorCounts.size());
assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION));
}
+ @ParameterizedTest
+ @ValueSource(shorts = {0, 1, 2, 3, 4})
+ public void testKafkaApiVersionsRequest(short version) {
+ KafkaRequestHandler handler = createKafkaRequestHandler();
+ ApiVersionsResponse response = requestApiVersions(handler, version);
+
+ assertSuccessfulResponseDefaults(response);
+ assertBrokerCapabilities(response);
+ }
+
+ private static ApiVersionsResponse requestApiVersions(
+ KafkaRequestHandler handler, short version) {
+ ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build(version);
+ ChannelHandlerContext ctx = new TestingChannelHandlerContext();
+ KafkaRequest request =
+ newRequest(
+ ApiKeys.API_VERSIONS,
+ version,
+ new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0),
+ apiVersionsRequest,
+ ctx);
+ handler.processRequest(request);
+
+ return parseApiVersionsResponse(request);
+ }
+
+ private static ApiVersionsResponse parseApiVersionsResponse(KafkaRequest request) {
+ return (ApiVersionsResponse) parseResponse(request);
+ }
+
+ private static void assertSuccessfulResponseDefaults(ApiVersionsResponse response) {
+ assertThat(response.errorCounts())
+ .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1));
+ assertThat(response.data().throttleTimeMs()).isZero();
+ assertThat(response.data().supportedFeatures()).isEmpty();
+ assertThat(response.data().finalizedFeaturesEpoch()).isEqualTo(-1L);
+ assertThat(response.data().finalizedFeatures()).isEmpty();
+ assertThat(response.data().zkMigrationReady()).isFalse();
+ }
+
+ private static void assertBrokerCapabilities(ApiVersionsResponse response) {
+ assertThat(response.data().apiKeys())
+ .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion)
+ .containsExactly(
+ tuple(ApiKeys.METADATA.id, ApiKeys.METADATA.oldestVersion(), (short) 11),
+ tuple(
+ ApiKeys.API_VERSIONS.id,
+ ApiKeys.API_VERSIONS.oldestVersion(),
+ ApiKeys.API_VERSIONS.latestVersion()));
+ }
+
@Test
- public void testKafkaApiVersionsRequest() {
+ public void testInvalidApiVersionsRequest() {
KafkaRequestHandler handler = createKafkaRequestHandler();
short latestVersion = ApiKeys.API_VERSIONS.latestVersion();
- ApiVersionsRequest apiVersionsRequest =
- new ApiVersionsRequest.Builder().build(latestVersion);
- ChannelHandlerContext ctx = new TestingChannelHandlerContext();
+ ApiVersionsRequest requestBody =
+ new ApiVersionsRequest.Builder(
+ new ApiVersionsRequestData()
+ .setClientSoftwareName("invalid client name")
+ .setClientSoftwareVersion("1.0"),
+ latestVersion,
+ latestVersion)
+ .build(latestVersion);
KafkaRequest request =
- new KafkaRequest(
+ newRequest(
ApiKeys.API_VERSIONS,
latestVersion,
new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0),
- apiVersionsRequest,
- ByteBufAllocator.DEFAULT.buffer(),
- ctx,
- new CompletableFuture<>());
- handler.handleApiVersionsRequest(request);
+ requestBody,
+ new TestingChannelHandlerContext());
+
+ handler.processRequest(request);
+
+ ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request);
+ assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1);
+ }
+
+ @Test
+ public void testUnregisteredApiIsNotRouted() {
+ KafkaRequestHandler handler = createKafkaRequestHandler();
+ short version = ApiKeys.CREATE_TOPICS.latestVersion();
+ CreateTopicsRequestData requestData =
+ new CreateTopicsRequestData()
+ .setTimeoutMs(1000)
+ .setTopics(
+ new CreateTopicsRequestData.CreatableTopicCollection(
+ Collections.singletonList(
+ new CreateTopicsRequestData.CreatableTopic()
+ .setName("topic")
+ .setNumPartitions(1)
+ .setReplicationFactor((short) 1))
+ .iterator()));
+ CreateTopicsRequest requestBody =
+ new CreateTopicsRequest.Builder(requestData).build(version);
+ KafkaRequest request =
+ newRequest(
+ ApiKeys.CREATE_TOPICS,
+ version,
+ new RequestHeader(ApiKeys.CREATE_TOPICS, version, "client-id", 0),
+ requestBody,
+ new TestingChannelHandlerContext());
+
+ handler.processRequest(request);
+
+ CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request);
+ assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1);
+ }
+ private static KafkaRequest newRequest(
+ ApiKeys apiKey,
+ short apiVersion,
+ RequestHeader header,
+ AbstractRequest requestBody,
+ ChannelHandlerContext context) {
+ ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer();
+ try {
+ return new KafkaRequest(
+ apiKey,
+ apiVersion,
+ header,
+ requestBody,
+ requestBuffer,
+ context,
+ new CompletableFuture<>());
+ } finally {
+ // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest.
+ requestBuffer.release();
+ }
+ }
+
+ private static AbstractResponse parseResponse(KafkaRequest request) {
ByteBuf responseBuffer = request.responseBuffer();
- ApiVersionsResponse response =
- (ApiVersionsResponse)
- AbstractResponse.parseResponse(
- responseBuffer.nioBuffer(), request.header());
- Map errorCounts = response.errorCounts();
- assertThat(1).isEqualTo(errorCounts.size());
- assertThat(1).isEqualTo(errorCounts.get(Errors.NONE));
- response.data()
- .apiKeys()
- .forEach(
- apiVersion -> {
- if (ApiKeys.METADATA.id == apiVersion.apiKey()) {
- assertThat((short) 11)
- .isGreaterThanOrEqualTo(apiVersion.maxVersion());
- } else if (ApiKeys.FETCH.id == apiVersion.apiKey()) {
- assertThat((short) 12)
- .isGreaterThanOrEqualTo(apiVersion.maxVersion());
- } else {
- ApiKeys apiKeys = ApiKeys.forId(apiVersion.apiKey());
- assertThat(apiVersion.minVersion())
- .isEqualTo(apiKeys.oldestVersion());
- assertThat(apiVersion.maxVersion())
- .isEqualTo(apiKeys.latestVersion());
- }
- });
+ try {
+ return AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header());
+ } finally {
+ responseBuffer.release();
+ }
}
private static KafkaRequestHandler createKafkaRequestHandler() {
- return new KafkaRequestHandler(new TestingTabletGatewayService());
+ TestingTabletGatewayService service = new TestingTabletGatewayService();
+ return new KafkaRequestHandler(service, service, "kafka");
}
}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java
new file mode 100644
index 00000000000..2aa9caf9ba9
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka;
+
+import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf;
+import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext;
+
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.requests.ApiVersionsRequest;
+import org.apache.kafka.common.requests.RequestHeader;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.CompletableFuture;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/** Tests for {@link KafkaRequest}. */
+public class KafkaRequestTest {
+
+ @Test
+ public void testReleaseBufferIsIdempotent() {
+ short version = ApiKeys.API_VERSIONS.oldestVersion();
+ ByteBuf buffer = mock(ByteBuf.class);
+ when(buffer.retain()).thenReturn(buffer);
+ KafkaRequest request =
+ new KafkaRequest(
+ ApiKeys.API_VERSIONS,
+ version,
+ new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 1),
+ new ApiVersionsRequest.Builder().build(version),
+ buffer,
+ mock(ChannelHandlerContext.class),
+ new CompletableFuture<>());
+
+ request.releaseBuffer();
+ request.releaseBuffer();
+
+ verify(buffer).retain();
+ verify(buffer).release();
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java
new file mode 100644
index 00000000000..84a889c16bd
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.dispatcher;
+
+import org.apache.fluss.kafka.KafkaRequestContext;
+
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.ApiVersionsRequest;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.CompletableFuture;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link KafkaApiRegistry}. */
+public class KafkaApiRegistryTest {
+
+ @Test
+ public void testRejectDuplicateRegistrationAndRegistrationAfterFreeze() {
+ KafkaApiRegistry registry = brokerRegistry();
+ TestingApiVersionsHandler handler = new TestingApiVersionsHandler(true);
+ registry.register(handler);
+
+ assertThatThrownBy(() -> registry.register(handler))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("already registered");
+
+ registry.freeze();
+ assertThatThrownBy(() -> registry.register(handler))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("already frozen");
+ }
+
+ @Test
+ public void testOnlyAdvertiseEnabledHandlers() {
+ KafkaApiRegistry registry = brokerRegistry();
+ registry.register(new TestingApiVersionsHandler(true));
+ assertThat(registry.advertisedApiSpecs()).hasSize(1);
+
+ KafkaApiRegistry hiddenRegistry = brokerRegistry();
+ hiddenRegistry.register(new TestingApiVersionsHandler(false));
+ assertThat(hiddenRegistry.advertisedApiSpecs()).isEmpty();
+ assertThat(hiddenRegistry.lookup(ApiKeys.API_VERSIONS)).isNull();
+ }
+
+ @Test
+ public void testAdvertisedSpecIsSameSpecUsedForRouting() {
+ KafkaApiRegistry registry = brokerRegistry();
+ TestingApiVersionsHandler handler = new TestingApiVersionsHandler(true);
+ registry.register(handler);
+ registry.freeze();
+
+ KafkaApiSpec advertisedSpec = registry.advertisedApiSpecs().get(0);
+ KafkaApiHandler routedHandler = registry.lookup(ApiKeys.API_VERSIONS);
+
+ assertThat(routedHandler).isSameAs(handler);
+ assertThat(routedHandler.apiSpec()).isSameAs(advertisedSpec);
+ for (short version : ApiKeys.API_VERSIONS.allVersions()) {
+ assertThat(advertisedSpec.supportsVersion(version)).isTrue();
+ }
+ assertThat(
+ advertisedSpec.supportsVersion(
+ (short) (ApiKeys.API_VERSIONS.latestVersion() + 1)))
+ .isFalse();
+ }
+
+ @Test
+ public void testRejectInvalidVersionRange() {
+ assertThatThrownBy(() -> new KafkaApiSpec(ApiKeys.API_VERSIONS, (short) 1, (short) 0, true))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(
+ () ->
+ new KafkaApiSpec(
+ ApiKeys.API_VERSIONS,
+ ApiKeys.API_VERSIONS.oldestVersion(),
+ (short) (ApiKeys.API_VERSIONS.latestVersion() + 1),
+ true))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ private static KafkaApiRegistry brokerRegistry() {
+ return new KafkaApiRegistry();
+ }
+
+ private static final class TestingApiVersionsHandler
+ implements KafkaApiHandler {
+
+ private final KafkaApiSpec spec;
+
+ private TestingApiVersionsHandler(boolean advertised) {
+ this.spec =
+ new KafkaApiSpec(
+ ApiKeys.API_VERSIONS,
+ ApiKeys.API_VERSIONS.oldestVersion(),
+ ApiKeys.API_VERSIONS.latestVersion(),
+ advertised);
+ }
+
+ @Override
+ public KafkaApiSpec apiSpec() {
+ return spec;
+ }
+
+ @Override
+ public CompletableFuture handle(
+ KafkaRequestContext context, ApiVersionsRequest request) {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java
new file mode 100644
index 00000000000..97f42350198
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.dispatcher;
+
+import org.apache.fluss.kafka.KafkaRequest;
+import org.apache.fluss.kafka.KafkaRequestContext;
+import org.apache.fluss.kafka.error.KafkaErrorMapper;
+import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext;
+
+import org.apache.kafka.common.errors.InvalidRequestException;
+import org.apache.kafka.common.message.ApiVersionsResponseData;
+import org.apache.kafka.common.protocol.ApiKeys;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.ApiVersionsRequest;
+import org.apache.kafka.common.requests.ApiVersionsResponse;
+import org.apache.kafka.common.requests.RequestHeader;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/** Tests request routing and failure handling without a concrete API implementation. */
+class KafkaRequestDispatcherTest {
+
+ @Test
+ void testUnregisteredApiReturnsUnsupportedVersion() {
+ KafkaApiRegistry registry = new KafkaApiRegistry();
+ registry.freeze();
+
+ AbstractResponse response =
+ new KafkaRequestDispatcher(registry, new KafkaErrorMapper())
+ .dispatch(request((short) 0))
+ .join();
+
+ assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1);
+ }
+
+ @Test
+ void testUnsupportedVersionDoesNotInvokeHandler() {
+ KafkaRequestDispatcher dispatcher =
+ dispatcher(
+ (context, request) -> {
+ throw new AssertionError(
+ "Unsupported versions must not be dispatched.");
+ });
+
+ AbstractResponse response = dispatcher.dispatch(request((short) 1)).join();
+
+ assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1);
+ }
+
+ @Test
+ void testDispatchWaitsForHandlerAndPreservesContext() {
+ CompletableFuture handlerResult = new CompletableFuture<>();
+ AtomicReference receivedContext = new AtomicReference<>();
+ KafkaRequestDispatcher dispatcher =
+ dispatcher(
+ (context, request) -> {
+ receivedContext.set(context);
+ return handlerResult;
+ });
+ KafkaRequest request = request((short) 0);
+
+ CompletableFuture result = dispatcher.dispatch(request);
+
+ assertThat(result).isNotDone();
+ assertThat(receivedContext.get().clientId()).isEqualTo("client");
+ assertThat(receivedContext.get().correlationId()).isEqualTo(42);
+ assertThat(receivedContext.get().listenerName()).isEqualTo("KAFKA");
+ assertThat(receivedContext.get().apiKey()).isEqualTo(ApiKeys.API_VERSIONS);
+ assertThat(receivedContext.get().apiVersion()).isZero();
+ AbstractResponse response = new ApiVersionsResponse(new ApiVersionsResponseData());
+ handlerResult.complete(response);
+ assertThat(result.join()).isSameAs(response);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testSynchronousAndAsynchronousFailuresBecomeErrorResponses(boolean synchronous) {
+ InvalidRequestException failure = new InvalidRequestException("invalid request");
+ KafkaRequestDispatcher dispatcher =
+ dispatcher(
+ (context, request) -> {
+ if (synchronous) {
+ throw failure;
+ }
+ CompletableFuture result = new CompletableFuture<>();
+ result.completeExceptionally(new CompletionException(failure));
+ return result;
+ });
+
+ AbstractResponse response = dispatcher.dispatch(request((short) 0)).join();
+
+ assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testNullFutureAndNullResponseBecomeErrorResponses(boolean nullFuture) {
+ KafkaRequestDispatcher dispatcher =
+ dispatcher(
+ (context, request) ->
+ nullFuture ? null : CompletableFuture.completedFuture(null));
+
+ AbstractResponse response = dispatcher.dispatch(request((short) 0)).join();
+
+ assertThat(response.errorCounts()).containsEntry(Errors.UNKNOWN_SERVER_ERROR, 1);
+ }
+
+ private static KafkaRequestDispatcher dispatcher(
+ BiFunction>
+ action) {
+ KafkaApiRegistry registry = new KafkaApiRegistry();
+ registry.register(
+ new KafkaApiHandler() {
+ @Override
+ public KafkaApiSpec apiSpec() {
+ return new KafkaApiSpec(ApiKeys.API_VERSIONS, (short) 0, (short) 0, true);
+ }
+
+ @Override
+ public CompletableFuture handle(
+ KafkaRequestContext context, ApiVersionsRequest request) {
+ return action.apply(context, request);
+ }
+ });
+ registry.freeze();
+ return new KafkaRequestDispatcher(registry, new KafkaErrorMapper());
+ }
+
+ private static KafkaRequest request(short version) {
+ KafkaRequest request = mock(KafkaRequest.class);
+ when(request.apiKey()).thenReturn(ApiKeys.API_VERSIONS);
+ when(request.apiVersion()).thenReturn(version);
+ when(request.request()).thenReturn(new ApiVersionsRequest.Builder().build(version));
+ when(request.header())
+ .thenReturn(new RequestHeader(ApiKeys.API_VERSIONS, version, "client", 42));
+ when(request.listenerName()).thenReturn("KAFKA");
+ when(request.ctx()).thenReturn(mock(ChannelHandlerContext.class));
+ return request;
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java
new file mode 100644
index 00000000000..20c32ded941
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.mapping;
+
+import org.apache.fluss.metadata.TablePath;
+
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.errors.InvalidTopicException;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link KafkaTopicMapper}. */
+public class KafkaTopicMapperTest {
+
+ @Test
+ public void testTopicNameAndIdMapping() {
+ KafkaTopicMapper mapper = new KafkaTopicMapper("kafka");
+
+ assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic");
+ Uuid topicId = mapper.toTopicId(123L);
+ assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID);
+ assertThat(mapper.isMappedTopicId(topicId)).isTrue();
+ assertThat(mapper.toTableId(topicId)).isEqualTo(123L);
+
+ Uuid firstTableTopicId = mapper.toTopicId(0L);
+ assertThat(firstTableTopicId).isNotEqualTo(Uuid.ZERO_UUID);
+ assertThat(mapper.isMappedTopicId(firstTableTopicId)).isTrue();
+ assertThat(mapper.toTableId(firstTableTopicId)).isZero();
+ }
+
+ @Test
+ public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() {
+ KafkaTopicMapper mapper = new KafkaTopicMapper("kafka");
+ assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue();
+ assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse();
+ assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse();
+ assertThatThrownBy(() -> mapper.toTablePath("invalid topic"))
+ .isInstanceOf(InvalidTopicException.class);
+ assertThatThrownBy(() -> mapper.toTopicId(-1L))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse();
+ assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java
new file mode 100644
index 00000000000..0c92b8a7300
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.schema;
+
+import org.apache.fluss.kafka.format.KafkaDataFormat;
+import org.apache.fluss.kafka.mapping.KafkaTopicMapper;
+import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.rpc.messages.MetadataRequest;
+import org.apache.fluss.rpc.messages.PbTableMetadata;
+import org.apache.fluss.server.testutils.FlussClusterExtension;
+import org.apache.fluss.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Verifies Kafka mapping properties survive the native Fluss table creation path. */
+public class KafkaTableMappingITCase {
+
+ @RegisterExtension
+ public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION =
+ FlussClusterExtension.builder().setNumOfTabletServers(1).build();
+
+ @Test
+ public void testResolveMappingFromCreatedTableMetadata() throws Exception {
+ TableDescriptor descriptor =
+ TableDescriptor.builder()
+ .schema(
+ Schema.newBuilder()
+ .column("event_key", DataTypes.STRING())
+ .column("event_body", DataTypes.BYTES())
+ .build())
+ .distributedBy(2)
+ .logFormat(LogFormat.ARROW)
+ .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key")
+ .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw")
+ .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY")
+ .build();
+ KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl");
+ TablePath tablePath = mapper.toTablePath("events");
+ long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor);
+ FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata();
+ MetadataRequest request = new MetadataRequest();
+ request.addTablePath()
+ .setDatabaseName(tablePath.getDatabaseName())
+ .setTableName(tablePath.getTableName());
+ PbTableMetadata metadata =
+ FLUSS_CLUSTER_EXTENSION
+ .newCoordinatorClient()
+ .metadata(request)
+ .get()
+ .getTableMetadatasList()
+ .get(0);
+ TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson());
+ KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted);
+
+ assertThat(metadata.getTableId()).isEqualTo(tableId);
+ assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId);
+ assertThat(metadata.getBucketMetadatasList()).hasSize(2);
+ assertThat(persisted.getCustomProperties())
+ .containsAllEntriesOf(descriptor.getCustomProperties());
+ assertThat(mapping.keyProjection().positions()).containsExactly(0);
+ assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING);
+ assertThat(mapping.valueProjection().positions()).containsExactly(1);
+ assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW);
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java
new file mode 100644
index 00000000000..4c698b5ad6b
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.schema;
+
+import org.apache.fluss.kafka.format.KafkaDataFormat;
+import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.types.DataType;
+import org.apache.fluss.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests the DDL contract independently of Kafka request handling and record decoding. */
+public class KafkaTopicSchemaResolverTest {
+
+ private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver();
+
+ @Test
+ public void testRawMappingSurvivesTableMetadataSerialization() {
+ Schema schema =
+ Schema.newBuilder()
+ .column("message", DataTypes.BYTES())
+ .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false))
+ .column("attributes", headersType())
+ .column("message_key", DataTypes.BYTES())
+ .build();
+ TableDescriptor table =
+ table(schema, "raw")
+ .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key")
+ .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY")
+ .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at")
+ .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes")
+ .build();
+ KafkaTopicSchema mapping =
+ resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()));
+
+ assertThat(mapping.rowType()).isEqualTo(schema.getRowType());
+ assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.RAW);
+ assertThat(mapping.keyProjection().positions()).containsExactly(3);
+ assertThat(mapping.keyProjection().nameAt(0)).isEqualTo("message_key");
+ assertThat(mapping.valueProjection().positions()).containsExactly(0);
+ assertThat(mapping.valueProjection().dataTypeAt(0)).isEqualTo(DataTypes.BYTES());
+ assertThat(mapping.timestampPosition()).isEqualTo(1);
+ assertThat(mapping.headersPosition()).isEqualTo(2);
+ assertThat(mapping).isEqualTo(resolver.resolve(table));
+ assertThatThrownBy(() -> mapping.keyProjection().positions().add(1))
+ .isInstanceOf(UnsupportedOperationException.class);
+ }
+
+ @Test
+ public void testValueOnlyStringAndNullableColumns() {
+ for (boolean nullable : new boolean[] {true, false}) {
+ KafkaTopicSchema mapping =
+ resolver.resolve(
+ table(
+ Schema.newBuilder()
+ .column(
+ "body",
+ DataTypes.STRING().copy(nullable))
+ .build(),
+ " STRING ")
+ .build());
+ assertThat(mapping.keyFormat()).isNull();
+ assertThat(mapping.keyProjection().isEmpty()).isTrue();
+ assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.STRING);
+ assertThat(mapping.valueProjection().positions()).containsExactly(0);
+ assertThat(mapping.valueProjection().dataTypeAt(0).isNullable()).isEqualTo(nullable);
+ assertThat(mapping.timestampPosition()).isEqualTo(-1);
+ assertThat(mapping.headersPosition()).isEqualTo(-1);
+ }
+ }
+
+ @Test
+ public void testMixedFormatsAndMetadataAreOptional() {
+ KafkaTopicSchema mapping =
+ resolver.resolve(
+ keyValueTable()
+ .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id")
+ .customProperty(
+ KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, " except_key ")
+ .build());
+ assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING);
+ assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW);
+ assertThat(mapping.keyProjection().positions()).containsExactly(0);
+ assertThat(mapping.valueProjection().positions()).containsExactly(1);
+ }
+
+ @Test
+ public void testRejectsUnsupportedTableKinds() {
+ Schema primaryKeySchema =
+ Schema.newBuilder()
+ .column("id", DataTypes.STRING().copy(false))
+ .primaryKey("id")
+ .build();
+ assertInvalid(table(primaryKeySchema, "string"), "must be a log table");
+ assertInvalid(keyValueTable().partitionedBy("id"), "Partitioned Fluss tables");
+ assertInvalid(valueTable().logFormat(LogFormat.INDEXED), "Arrow log format");
+ }
+
+ @Test
+ public void testRequiresExplicitValueFormat() {
+ assertInvalid(
+ TableDescriptor.builder()
+ .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build())
+ .distributedBy(1)
+ .customProperty("fluss.value.format", "raw"),
+ KafkaDataFormat.VALUE_FORMAT_CONFIG);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"json", "avro", ""})
+ public void testRejectsUnavailableFormats(String format) {
+ assertInvalid(
+ valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format),
+ "Unsupported Kafka data format");
+ assertInvalid(
+ keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format),
+ "Unsupported Kafka data format");
+ }
+
+ @Test
+ public void testRejectsUnsupportedKafkaOptions() {
+ assertInvalid(
+ valueTable().customProperty("kafka.value.rescue-column", "body"),
+ "Unsupported Kafka table property");
+ assertInvalid(
+ valueTable().customProperty("kafka.key.field", "body"),
+ "Unsupported Kafka table property");
+ assertThat(resolver.resolve(valueTable().customProperty("owner", "team").build()))
+ .isNotNull();
+ }
+
+ @Test
+ public void testKeyFormatAndFieldMustBeSpecifiedTogether() {
+ assertInvalid(
+ keyValueTable().customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"),
+ "requires kafka.key.format");
+ assertInvalid(
+ keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"),
+ "kafka.key.fields");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"", " ", "missing", "id,id", "id,", ",id", "id,,body"})
+ public void testRejectsInvalidKeyFields(String fields) {
+ assertThatThrownBy(
+ () ->
+ resolver.resolve(
+ keyValueTable()
+ .customProperty(
+ KafkaDataFormat.KEY_FORMAT_CONFIG, "string")
+ .customProperty(
+ KafkaDataFormat.KEY_FIELDS_CONFIG, fields)
+ .customProperty(
+ KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG,
+ "EXCEPT_KEY")
+ .build()))
+ .isInstanceOf(KafkaTopicSchemaException.class);
+ }
+
+ @Test
+ public void testRejectsAmbiguousAndEmptyValueProjections() {
+ assertInvalid(
+ keyValueTable()
+ .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"),
+ "requires");
+ assertInvalid(
+ valueTable().customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "bad"),
+ "Expected ALL or EXCEPT_KEY");
+ assertInvalid(
+ valueTable()
+ .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "body")
+ .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"),
+ "at least one Fluss column");
+ assertInvalid(keyValueTable(), "exactly one Fluss field");
+ assertInvalid(
+ keyValueTable()
+ .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id,body")
+ .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"),
+ "at least one Fluss column");
+ }
+
+ @Test
+ public void testRejectsWrongPhysicalTypes() {
+ assertInvalid(
+ valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"),
+ "must be STRING");
+ assertInvalid(
+ keyValueTable()
+ .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id")
+ .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"),
+ "must be BYTES");
+ }
+
+ @Test
+ public void testRejectsWrongTimestampTypes() {
+ for (DataType type :
+ new DataType[] {
+ DataTypes.STRING(),
+ DataTypes.TIMESTAMP_LTZ(3),
+ DataTypes.TIMESTAMP_LTZ(6).copy(false)
+ }) {
+ assertInvalid(
+ table(
+ Schema.newBuilder()
+ .column("body", DataTypes.BYTES())
+ .column("ts", type)
+ .build(),
+ "raw")
+ .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"),
+ "TIMESTAMP_LTZ(3) NOT NULL");
+ }
+ }
+
+ @Test
+ public void testRejectsWrongHeaderTypes() {
+ for (DataType type :
+ new DataType[] {
+ DataTypes.STRING(),
+ headersType().copy(false),
+ DataTypes.ARRAY(DataTypes.STRING()),
+ DataTypes.ARRAY(
+ DataTypes.ROW(
+ DataTypes.FIELD("key", DataTypes.STRING()),
+ DataTypes.FIELD("value", DataTypes.BYTES()))),
+ DataTypes.ARRAY(
+ DataTypes.ROW(
+ DataTypes.FIELD("name", DataTypes.STRING()),
+ DataTypes.FIELD("value", DataTypes.BYTES().copy(false))))
+ }) {
+ assertInvalid(
+ table(
+ Schema.newBuilder()
+ .column("body", DataTypes.BYTES())
+ .column("attributes", type)
+ .build(),
+ "raw")
+ .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes"),
+ "headers");
+ }
+ }
+
+ @Test
+ public void testRejectsConflictingOrMissingMetadataColumns() {
+ assertInvalid(
+ valueTable().customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "missing"),
+ "does not exist");
+ assertInvalid(
+ valueTable().customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, " "),
+ "must not be empty");
+ TableDescriptor.Builder table =
+ table(
+ Schema.newBuilder()
+ .column("body", DataTypes.BYTES())
+ .column("ts", DataTypes.TIMESTAMP_LTZ(3).copy(false))
+ .build(),
+ "raw")
+ .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts");
+ assertInvalid(
+ TableDescriptor.builder(table.build())
+ .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "ts"),
+ "same Fluss column");
+ assertInvalid(
+ TableDescriptor.builder(table.build())
+ .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "ts"),
+ "metadata column");
+ }
+
+ private void assertInvalid(TableDescriptor.Builder table, String message) {
+ assertThatThrownBy(() -> resolver.resolve(table.build()))
+ .isInstanceOf(KafkaTopicSchemaException.class)
+ .hasMessageContaining(message);
+ }
+
+ private static TableDescriptor.Builder keyValueTable() {
+ return table(
+ Schema.newBuilder()
+ .column("id", DataTypes.STRING())
+ .column("body", DataTypes.BYTES())
+ .build(),
+ "raw");
+ }
+
+ private static TableDescriptor.Builder valueTable() {
+ return table(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "raw");
+ }
+
+ private static TableDescriptor.Builder table(Schema schema, String valueFormat) {
+ return TableDescriptor.builder()
+ .schema(schema)
+ .distributedBy(2)
+ .logFormat(LogFormat.ARROW)
+ .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat);
+ }
+
+ private static DataType headersType() {
+ return DataTypes.ARRAY(
+ DataTypes.ROW(
+ DataTypes.FIELD("name", DataTypes.STRING()),
+ DataTypes.FIELD("value", DataTypes.BYTES())));
+ }
+}
diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java
new file mode 100644
index 00000000000..6936b5c8d61
--- /dev/null
+++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.kafka.transcode;
+
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record;
+import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader;
+import org.apache.fluss.kafka.format.KafkaDataFormat;
+import org.apache.fluss.kafka.schema.KafkaTopicSchemaException;
+import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.record.LogRecord;
+import org.apache.fluss.record.LogRecordBatch;
+import org.apache.fluss.record.LogRecordReadContext;
+import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.TestingSchemaGetter;
+import org.apache.fluss.record.bytesview.BytesView;
+import org.apache.fluss.row.GenericRow;
+import org.apache.fluss.row.InternalArray;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf;
+import org.apache.fluss.types.DataTypes;
+import org.apache.fluss.utils.CloseableIterator;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.function.Consumer;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.catchThrowable;
+
+/** Verifies mapped Arrow records after transcoding resources have closed. */
+class ArrowKafkaRecordTranscoderTest {
+ private final ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder();
+
+ @Test
+ void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception {
+ for (boolean string : new boolean[] {false, true}) {
+ TableInfo table = envelope(string);
+ Record record =
+ new Record(
+ 123L,
+ bytes("键"),
+ bytes("message"),
+ Arrays.asList(
+ new RecordHeader("duplicate", bytes("first")),
+ new RecordHeader("duplicate", null)));
+ BytesView encoded = transcoder.transcode(Arrays.asList(record, record), table);
+ read(
+ encoded,
+ table,
+ 2,
+ row -> {
+ if (string) {
+ assertThat(row.getString(0).toString()).isEqualTo("message");
+ assertThat(row.getString(3).toString()).isEqualTo("键");
+ } else {
+ assertThat(row.getBytes(0)).isEqualTo(bytes("message"));
+ assertThat(row.getBytes(3)).isEqualTo(bytes("键"));
+ }
+ assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L);
+ InternalArray headers = row.getArray(2);
+ assertThat(headers.size()).isEqualTo(2);
+ assertThat(headers.getRow(0, 2).getString(0).toString())
+ .isEqualTo("duplicate");
+ assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("first"));
+ assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue();
+ });
+ }
+ }
+
+ @Test
+ void testNullsAndEmptyBytesRemainDistinct() throws Exception {
+ TableInfo table = envelope(false);
+ read(
+ transcoder.transcode(
+ Collections.singletonList(
+ new Record(1L, null, null, Collections.emptyList())),
+ table),
+ table,
+ 1,
+ row -> {
+ assertThat(row.isNullAt(0)).isTrue();
+ assertThat(row.isNullAt(3)).isTrue();
+ assertThat(row.getArray(2).size()).isZero();
+ });
+ read(
+ transcoder.transcode(
+ Collections.singletonList(
+ new Record(1L, new byte[0], new byte[0], Collections.emptyList())),
+ table),
+ table,
+ 1,
+ row -> {
+ assertThat(row.getBytes(0)).isEmpty();
+ assertThat(row.getBytes(3)).isEmpty();
+ });
+ }
+
+ @Test
+ void testUnmappedKeyIsIgnoredAndMixedFormatsWork() throws Exception {
+ TableInfo valueOnly = valueTable(false);
+ read(
+ transcoder.transcode(
+ Collections.singletonList(
+ new Record(
+ 1L,
+ new byte[] {(byte) 0xff},
+ bytes("value"),
+ Collections.emptyList())),
+ valueOnly),
+ valueOnly,
+ 1,
+ row -> assertThat(row.getString(0).toString()).isEqualTo("value"));
+ Schema schema =
+ Schema.newBuilder()
+ .column("body", DataTypes.BYTES())
+ .column("id", DataTypes.STRING())
+ .build();
+ TableInfo mixed =
+ table(
+ TableDescriptor.builder()
+ .schema(schema)
+ .distributedBy(1)
+ .logFormat(LogFormat.ARROW)
+ .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw")
+ .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id")
+ .customProperty(
+ KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY")
+ .build());
+ read(
+ transcoder.transcode(
+ Collections.singletonList(
+ new Record(
+ 1L,
+ bytes("key"),
+ new byte[] {(byte) 0xff},
+ Collections.emptyList())),
+ mixed),
+ mixed,
+ 1,
+ row -> {
+ assertThat(row.getString(1).toString()).isEqualTo("key");
+ assertThat(row.getBytes(0)).containsExactly((byte) 0xff);
+ });
+ }
+
+ @Test
+ void testRejectsMalformedUtf8AndNotNullViolationsThenRemainsUsable() throws Exception {
+ TableInfo table = valueTable(false);
+ assertThatThrownBy(
+ () ->
+ transcoder.transcode(
+ Collections.singletonList(
+ new Record(
+ 1L,
+ null,
+ new byte[] {(byte) 0xc3, 0x28},
+ Collections.emptyList())),
+ table))
+ .isInstanceOf(KafkaRecordEncodingException.class)
+ .hasMessageContaining("UTF-8");
+ assertThatThrownBy(
+ () ->
+ transcoder.transcode(
+ Collections.singletonList(
+ new Record(
+ 1L, null, null, Collections.emptyList())),
+ valueTable(true)))
+ .isInstanceOf(KafkaRecordEncodingException.class)
+ .hasMessageContaining("NOT NULL");
+ read(
+ transcoder.transcode(
+ Collections.singletonList(
+ new Record(1L, null, bytes("ok"), Collections.emptyList())),
+ table),
+ table,
+ 1,
+ row -> assertThat(row.getString(0).toString()).isEqualTo("ok"));
+ }
+
+ @Test
+ void testRejectsInvalidMappingAndEmptyPartition() {
+ TableInfo invalid =
+ table(
+ TableDescriptor.builder()
+ .schema(
+ Schema.newBuilder()
+ .column("body", DataTypes.STRING())
+ .build())
+ .distributedBy(1)
+ .build());
+ assertThatThrownBy(
+ () ->
+ transcoder.transcode(
+ Collections.singletonList(
+ new Record(
+ 1L,
+ null,
+ bytes("value"),
+ Collections.emptyList())),
+ invalid))
+ .isInstanceOf(KafkaTopicSchemaException.class);
+ assertThatThrownBy(() -> transcoder.transcode(Collections.emptyList(), valueTable(false)))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("empty");
+ }
+
+ @Test
+ void testArrowWriterFailureClosesAllocatorWithoutLeaking() {
+ Throwable failure =
+ catchThrowable(
+ () ->
+ new FlussArrowRecordEncoder()
+ .encode(
+ Collections.singletonList(GenericRow.of(123)),
+ valueTable(false)));
+ assertThat(failure).isInstanceOf(ClassCastException.class);
+ assertThat(failure.getSuppressed()).isEmpty();
+ }
+
+ private static TableInfo valueTable(boolean notNull) {
+ return table(
+ TableDescriptor.builder()
+ .schema(
+ Schema.newBuilder()
+ .column("body", DataTypes.STRING().copy(!notNull))
+ .build())
+ .distributedBy(1)
+ .logFormat(LogFormat.ARROW)
+ .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string")
+ .build());
+ }
+
+ private static TableInfo envelope(boolean string) {
+ Schema schema =
+ Schema.newBuilder()
+ .column("body", string ? DataTypes.STRING() : DataTypes.BYTES())
+ .column("time", DataTypes.TIMESTAMP_LTZ(3).copy(false))
+ .column(
+ "attrs",
+ DataTypes.ARRAY(
+ DataTypes.ROW(
+ DataTypes.FIELD("name", DataTypes.STRING()),
+ DataTypes.FIELD("value", DataTypes.BYTES()))))
+ .column("id", string ? DataTypes.STRING() : DataTypes.BYTES())
+ .build();
+ return table(
+ TableDescriptor.builder()
+ .schema(schema)
+ .distributedBy(1)
+ .logFormat(LogFormat.ARROW)
+ .customProperty(
+ KafkaDataFormat.VALUE_FORMAT_CONFIG, string ? "string" : "raw")
+ .customProperty(
+ KafkaDataFormat.KEY_FORMAT_CONFIG, string ? "string" : "raw")
+ .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id")
+ .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY")
+ .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "time")
+ .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attrs")
+ .build());
+ }
+
+ private static TableInfo table(TableDescriptor descriptor) {
+ return TableInfo.of(TablePath.of("kafka", "topic"), 42L, 3, descriptor, null, 1L, 1L);
+ }
+
+ private static void read(
+ BytesView encoded, TableInfo table, int count, Consumer verify)
+ throws Exception {
+ ByteBuf buffer = encoded.getByteBuf();
+ try {
+ LogRecordBatch batch =
+ MemoryLogRecords.pointToByteBuffer(buffer.nioBuffer())
+ .batches()
+ .iterator()
+ .next();
+ batch.ensureValid();
+ assertThat(batch.schemaId()).isEqualTo((short) table.getSchemaId());
+ assertThat(batch.getRecordCount()).isEqualTo(count);
+ try (LogRecordReadContext context =
+ LogRecordReadContext.createArrowReadContext(
+ table.getRowType(),
+ table.getSchemaId(),
+ new TestingSchemaGetter(
+ new SchemaInfo(
+ table.getSchema(), table.getSchemaId())));
+ CloseableIterator records = batch.records(context)) {
+ for (int i = 0; i < count; i++) {
+ assertThat(records.hasNext()).isTrue();
+ verify.accept(records.next().getRow());
+ }
+ assertThat(records.hasNext()).isFalse();
+ }
+ } finally {
+ buffer.release();
+ }
+ }
+
+ private static byte[] bytes(String text) {
+ return text.getBytes(StandardCharsets.UTF_8);
+ }
+}
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java
index 03d798fb371..df2256985c2 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java
@@ -232,8 +232,17 @@ private static List loadProtocols(
NetworkProtocolPlugin kafkaPlugin =
loadProtocolPlugin(NetworkProtocolPlugin.KAFKA_PROTOCOL_NAME);
kafkaPlugin.setup(conf);
- listeners.removeAll(kafkaPlugin.listenerNames());
- protocolPlugins.add(kafkaPlugin);
+ List kafkaListenerNames = kafkaPlugin.listenerNames();
+ boolean hasKafkaEndpoint =
+ endpoints.stream()
+ .anyMatch(
+ endpoint ->
+ kafkaListenerNames.contains(
+ endpoint.getListenerName()));
+ if (hasKafkaEndpoint) {
+ listeners.removeAll(kafkaListenerNames);
+ protocolPlugins.add(kafkaPlugin);
+ }
}
// Add the Fluss protocol plugin in the end to allow other protocol