From 7f4261913af66d19aa4cecc5279307e1b04a5c38 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 8 Sep 2026 11:29:44 +0800 Subject: [PATCH 01/11] [kafka] Add request dispatch and transport framework Introduce API registration, request context, version validation, and asynchronous error mapping. Fix request buffer ownership and response serialization cleanup while preserving the existing ApiVersions entry point. Validated with mvn -o -pl fluss-rpc,fluss-kafka verify. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 555/555 AI-Contributed/UT: 525/525 --- .../fluss/kafka/KafkaChannelInitializer.java | 5 +- .../fluss/kafka/KafkaCommandDecoder.java | 45 +++-- .../fluss/kafka/KafkaProtocolPlugin.java | 1 + .../org/apache/fluss/kafka/KafkaRequest.java | 43 ++++- .../fluss/kafka/KafkaRequestContext.java | 96 ++++++++++ .../kafka/dispatcher/KafkaApiHandler.java | 37 ++++ .../kafka/dispatcher/KafkaApiRegistry.java | 80 +++++++++ .../fluss/kafka/dispatcher/KafkaApiSpec.java | 82 +++++++++ .../dispatcher/KafkaRequestDispatcher.java | 108 ++++++++++++ .../fluss/kafka/error/KafkaErrorMapper.java | 45 +++++ .../fluss/kafka/KafkaCommandDecoderTest.java | 118 +++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 56 ++++-- .../apache/fluss/kafka/KafkaRequestTest.java | 58 ++++++ .../dispatcher/KafkaApiRegistryTest.java | 127 ++++++++++++++ .../KafkaRequestDispatcherTest.java | 166 ++++++++++++++++++ .../fluss/rpc/netty/server/NettyServer.java | 13 +- 16 files changed, 1039 insertions(+), 41 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java index 5e7551a9af7..29bdc745ca9 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java @@ -31,17 +31,20 @@ public class KafkaChannelInitializer extends NettyChannelInitializer { private final RequestChannel[] requestChannels; + private final String listenerName; private final int maxRequestSize; private final LengthFieldPrepender prepender = new LengthFieldPrepender(4); private final boolean preferHeap; public KafkaChannelInitializer( RequestChannel[] requestChannels, + String listenerName, long maxIdleTimeSeconds, int maxRequestSize, boolean preferHeap) { super(maxIdleTimeSeconds); this.requestChannels = requestChannels; + this.listenerName = listenerName; this.maxRequestSize = maxRequestSize; this.preferHeap = preferHeap; } @@ -53,6 +56,6 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast(prepender); addFrameDecoder(ch, maxRequestSize, 4, preferHeap); ch.pipeline().addLast("flowController", new FlowControlHandler()); - ch.pipeline().addLast(new KafkaCommandDecoder(requestChannels)); + ch.pipeline().addLast(new KafkaCommandDecoder(requestChannels, listenerName)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java index 43a0533b2d3..637a1aaa1fe 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java @@ -27,6 +27,7 @@ import org.apache.fluss.utils.MathUtils; import org.apache.kafka.common.errors.LeaderNotAvailableException; +import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; @@ -55,6 +56,7 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler { private final RequestChannel[] requestChannels; private final int numChannels; + private final String listenerName; // Need to use a Queue to store the inflight responses, because Kafka clients require the // responses to be sent in order. @@ -65,18 +67,18 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler { protected volatile ChannelHandlerContext ctx; protected SocketAddress remoteAddress; - public KafkaCommandDecoder(RequestChannel[] requestChannels) { + public KafkaCommandDecoder(RequestChannel[] requestChannels, String listenerName) { super(false); this.requestChannels = requestChannels; this.numChannels = requestChannels.length; + this.listenerName = listenerName; } @Override public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Exception { CompletableFuture future = new CompletableFuture<>(); - boolean needRelease = false; try { - KafkaRequest request = parseRequest(ctx, future, buffer); + KafkaRequest request = parseRequest(ctx, future, buffer, listenerName); inflightResponses.addLast(request); future.whenCompleteAsync((r, t) -> sendResponse(ctx), ctx.executor()); int channelIndex = @@ -86,16 +88,15 @@ public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Excep if (!isActive.get()) { LOG.warn("Received a request on an inactive channel: {}", remoteAddress); request.fail(new LeaderNotAvailableException("Channel is inactive")); - needRelease = true; } } catch (Throwable t) { - needRelease = true; LOG.error("Error handling request", t); future.completeExceptionally(t); } finally { - if (needRelease) { - ReferenceCountUtil.release(buffer); - } + // KafkaRequest retains the buffer to transfer ownership to request processing. Release + // the decoder's ownership on every path. KafkaRequest.releaseBuffer() is idempotent + // because worker cleanup and response completion can both release that ownership. + ReferenceCountUtil.release(buffer); } } @@ -184,19 +185,39 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E } private static KafkaRequest parseRequest( - ChannelHandlerContext ctx, CompletableFuture future, ByteBuf buffer) { + ChannelHandlerContext ctx, + CompletableFuture future, + ByteBuf buffer, + String listenerName) { ByteBuffer nioBuffer = buffer.nioBuffer(); RequestHeader header = RequestHeader.parse(nioBuffer); if (isUnsupportedApiVersionRequest(header)) { ApiVersionsRequest request = - new ApiVersionsRequest.Builder(header.apiVersion()).build(); + new ApiVersionsRequest( + new ApiVersionsRequestData(), + API_VERSIONS.oldestVersion(), + header.apiVersion()); return new KafkaRequest( - API_VERSIONS, header.apiVersion(), header, request, buffer, ctx, future); + API_VERSIONS, + header.apiVersion(), + header, + request, + listenerName, + buffer, + ctx, + future); } RequestAndSize request = AbstractRequest.parseRequest(header.apiKey(), header.apiVersion(), nioBuffer); return new KafkaRequest( - header.apiKey(), header.apiVersion(), header, request.request, buffer, ctx, future); + header.apiKey(), + header.apiVersion(), + header, + request.request, + listenerName, + buffer, + ctx, + future); } private static boolean isUnsupportedApiVersionRequest(RequestHeader header) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index d92ba5e68fc..c966f745b8c 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -53,6 +53,7 @@ public ChannelHandler createChannelHandler( RequestChannel[] requestChannels, String listenerName) { return new KafkaChannelInitializer( requestChannels, + listenerName, conf.get(ConfigOptions.KAFKA_CONNECTION_MAX_IDLE_TIME).getSeconds(), (int) conf.get(ConfigOptions.NETTY_SERVER_MAX_REQUEST_SIZE).getBytes(), conf.getBoolean(ConfigOptions.NETTY_CLIENT_ALLOCATOR_HEAP_BUFFER_FIRST)); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java index 25e409a7455..0d2799a7a18 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java @@ -35,6 +35,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; /** Represents a request received from Kafka protocol channel. */ @@ -46,10 +47,12 @@ public class KafkaRequest implements RpcRequest { private final long requestId = ID_GENERATOR.getAndIncrement(); private final RequestHeader header; private final AbstractRequest request; + private final String listenerName; private final ByteBuf buffer; private final ChannelHandlerContext ctx; private final long startTimeMs; private final CompletableFuture future; + private final AtomicBoolean bufferReleased = new AtomicBoolean(); private volatile boolean cancelled = false; protected KafkaRequest( @@ -60,10 +63,23 @@ protected KafkaRequest( ByteBuf buffer, ChannelHandlerContext ctx, CompletableFuture future) { + this(apiKey, apiVersion, header, request, "UNKNOWN", buffer, ctx, future); + } + + protected KafkaRequest( + ApiKeys apiKey, + short apiVersion, + RequestHeader header, + AbstractRequest request, + String listenerName, + ByteBuf buffer, + ChannelHandlerContext ctx, + CompletableFuture future) { this.apiKey = apiKey; this.apiVersion = apiVersion; this.header = header; this.request = request; + this.listenerName = listenerName; this.buffer = buffer.retain(); this.ctx = ctx; this.startTimeMs = System.currentTimeMillis(); @@ -77,7 +93,9 @@ public RequestType getRequestType() { @Override public void releaseBuffer() { - ReferenceCountUtil.safeRelease(buffer); + if (bufferReleased.compareAndSet(false, true)) { + ReferenceCountUtil.safeRelease(buffer); + } } public ApiKeys apiKey() { @@ -100,6 +118,10 @@ public T request() { return (T) request; } + public String listenerName() { + return listenerName; + } + public ChannelHandlerContext ctx() { return ctx; } @@ -149,12 +171,17 @@ private ByteBuf serialize(AbstractResponse response) { int headerSize = headerData.size(cache, headerVersion); ApiMessage apiMessage = response.data(); int messageSize = apiMessage.size(cache, apiVersion); - final ByteBuf buffer = ctx.alloc().buffer(headerSize + messageSize); - buffer.writerIndex(headerSize + messageSize); - final ByteBuffer nioBuffer = buffer.nioBuffer(); - final ByteBufferAccessor writable = new ByteBufferAccessor(nioBuffer); - headerData.write(writable, cache, headerVersion); - apiMessage.write(writable, cache, apiVersion); - return buffer; + final ByteBuf responseBuffer = ctx.alloc().buffer(headerSize + messageSize); + try { + responseBuffer.writerIndex(headerSize + messageSize); + final ByteBuffer nioBuffer = responseBuffer.nioBuffer(); + final ByteBufferAccessor writable = new ByteBufferAccessor(nioBuffer); + headerData.write(writable, cache, headerVersion); + apiMessage.write(writable, cache, apiVersion); + return responseBuffer; + } catch (Throwable t) { + ReferenceCountUtil.safeRelease(responseBuffer); + throw t; + } } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java new file mode 100644 index 00000000000..e75a20babcc --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.shaded.netty4.io.netty.channel.Channel; + +import org.apache.kafka.common.protocol.ApiKeys; + +import java.net.SocketAddress; + +/** Immutable wire-level context made available to Kafka API handlers. */ +@Internal +public final class KafkaRequestContext { + + private final int correlationId; + private final String clientId; + private final ApiKeys apiKey; + private final short apiVersion; + private final String listenerName; + private final SocketAddress localAddress; + private final SocketAddress remoteAddress; + private final long receivedTimeMs; + + private KafkaRequestContext(KafkaRequest request) { + this.correlationId = request.header().correlationId(); + this.clientId = request.header().clientId(); + this.apiKey = request.apiKey(); + this.apiVersion = request.apiVersion(); + this.listenerName = request.listenerName(); + Channel channel = request.ctx().channel(); + this.localAddress = channel == null ? null : channel.localAddress(); + this.remoteAddress = channel == null ? null : channel.remoteAddress(); + this.receivedTimeMs = request.startTimeMs(); + } + + /** Creates a context from a network request. */ + public static KafkaRequestContext fromRequest(KafkaRequest request) { + return new KafkaRequestContext(request); + } + + /** Returns the request correlation ID. */ + public int correlationId() { + return correlationId; + } + + /** Returns the client ID, or {@code null} when the request did not provide one. */ + public String clientId() { + return clientId; + } + + /** Returns the Kafka API key. */ + public ApiKeys apiKey() { + return apiKey; + } + + /** Returns the Kafka request version. */ + public short apiVersion() { + return apiVersion; + } + + /** Returns the listener that accepted the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the local socket address. */ + public SocketAddress localAddress() { + return localAddress; + } + + /** Returns the remote socket address. */ + public SocketAddress remoteAddress() { + return remoteAddress; + } + + /** Returns the wall-clock time at which the request was received. */ + public long receivedTimeMs() { + return receivedTimeMs; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java new file mode 100644 index 00000000000..36995b8e27f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; + +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; + +import java.util.concurrent.CompletableFuture; + +/** Handles one Kafka API without blocking the request processor thread. */ +@Internal +public interface KafkaApiHandler { + + /** Returns the capability implemented by this handler. */ + KafkaApiSpec apiSpec(); + + /** Handles a parsed request asynchronously. */ + CompletableFuture handle(KafkaRequestContext context, R request); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java new file mode 100644 index 00000000000..b4a1dfd8ea3 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.ApiKeys; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; + +/** Registry and single source of truth for Kafka APIs exposed by one server. */ +@Internal +public final class KafkaApiRegistry { + + private final Map> handlers = new HashMap<>(); + private boolean frozen; + + /** Creates an empty API registry. */ + public KafkaApiRegistry() {} + + /** Registers a handler. Registrations are rejected after {@link #freeze()} is called. */ + public void register(KafkaApiHandler handler) { + checkNotNull(handler); + checkState(!frozen, "Kafka API registry is already frozen."); + ApiKeys apiKey = handler.apiSpec().apiKey(); + checkArgument(!handlers.containsKey(apiKey), "Kafka API %s is already registered.", apiKey); + handlers.put(apiKey, handler); + } + + /** Prevents further registrations. */ + public void freeze() { + frozen = true; + } + + /** Returns a routable handler, or {@code null} when the API is not exposed by this server. */ + public KafkaApiHandler lookup(ApiKeys apiKey) { + KafkaApiHandler handler = handlers.get(apiKey); + if (handler == null || !handler.apiSpec().advertised()) { + return null; + } + return handler; + } + + /** Returns the sorted API specifications advertised by this server. */ + public List advertisedApiSpecs() { + List specs = new ArrayList<>(); + for (KafkaApiHandler handler : handlers.values()) { + KafkaApiSpec spec = handler.apiSpec(); + if (spec.advertised()) { + specs.add(spec); + } + } + Collections.sort(specs, Comparator.comparingInt(spec -> spec.apiKey().id)); + return Collections.unmodifiableList(specs); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java new file mode 100644 index 00000000000..50d6a7ebab2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.ApiKeys; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Describes the versions actually supported by a Kafka API handler. */ +@Internal +public final class KafkaApiSpec { + + private final ApiKeys apiKey; + private final short minVersion; + private final short maxVersion; + private final boolean advertised; + + /** Creates an API specification. */ + public KafkaApiSpec(ApiKeys apiKey, short minVersion, short maxVersion, boolean advertised) { + this.apiKey = checkNotNull(apiKey); + checkArgument(minVersion >= 0, "Minimum version must not be negative."); + checkArgument( + minVersion <= maxVersion, + "Minimum version %s must not exceed maximum version %s.", + minVersion, + maxVersion); + checkArgument( + minVersion >= apiKey.oldestVersion() && maxVersion <= apiKey.latestVersion(), + "Version range [%s, %s] is outside the Kafka library range [%s, %s] for %s.", + minVersion, + maxVersion, + apiKey.oldestVersion(), + apiKey.latestVersion(), + apiKey); + this.minVersion = minVersion; + this.maxVersion = maxVersion; + this.advertised = advertised; + } + + /** Returns the Kafka API key. */ + public ApiKeys apiKey() { + return apiKey; + } + + /** Returns the oldest supported request version. */ + public short minVersion() { + return minVersion; + } + + /** Returns the newest supported request version. */ + public short maxVersion() { + return maxVersion; + } + + /** Returns whether this API is allowed to be routed and advertised. */ + public boolean advertised() { + return advertised; + } + + /** Returns whether the supplied request version is supported. */ + public boolean supportsVersion(short version) { + return version >= minVersion && version <= maxVersion; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java new file mode 100644 index 00000000000..efdfe33dd66 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequest; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.error.KafkaErrorMapper; + +import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Validates and dispatches parsed Kafka requests to independently registered API handlers. */ +@Internal +public final class KafkaRequestDispatcher { + + private final KafkaApiRegistry registry; + private final KafkaErrorMapper errorMapper; + + /** Creates a dispatcher backed by the supplied registry and error mapper. */ + public KafkaRequestDispatcher(KafkaApiRegistry registry, KafkaErrorMapper errorMapper) { + this.registry = checkNotNull(registry); + this.errorMapper = checkNotNull(errorMapper); + } + + /** Dispatches a request and always completes with a Kafka protocol response. */ + public CompletableFuture dispatch(KafkaRequest request) { + AbstractRequest abstractRequest = request.request(); + KafkaApiHandler handler = registry.lookup(request.apiKey()); + if (handler == null) { + return completedErrorResponse( + abstractRequest, + new UnsupportedVersionException( + "Kafka API " + request.apiKey() + " is not supported by this server.")); + } + + KafkaApiSpec spec = handler.apiSpec(); + if (!spec.supportsVersion(request.apiVersion())) { + return completedErrorResponse( + abstractRequest, + new UnsupportedVersionException( + String.format( + "Version %s is not supported for %s. Supported versions are [%s, %s].", + request.apiVersion(), + request.apiKey(), + spec.minVersion(), + spec.maxVersion()))); + } + + CompletableFuture responseFuture; + try { + responseFuture = + invoke(handler, KafkaRequestContext.fromRequest(request), abstractRequest); + if (responseFuture == null) { + throw new NullPointerException("Kafka API handler returned a null future."); + } + } catch (Throwable t) { + return completedErrorResponse(abstractRequest, t); + } + + CompletableFuture result = new CompletableFuture<>(); + responseFuture.whenComplete( + (response, failure) -> { + if (failure == null && response != null) { + result.complete(response); + } else { + Throwable responseFailure = + failure == null + ? new NullPointerException( + "Kafka API handler returned a null response.") + : failure; + result.complete(errorMapper.toResponse(abstractRequest, responseFailure)); + } + }); + return result; + } + + @SuppressWarnings("unchecked") + private static CompletableFuture invoke( + KafkaApiHandler handler, KafkaRequestContext context, AbstractRequest request) { + return ((KafkaApiHandler) handler).handle(context, request); + } + + private CompletableFuture completedErrorResponse( + AbstractRequest request, Throwable failure) { + return CompletableFuture.completedFuture(errorMapper.toResponse(request, failure)); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java new file mode 100644 index 00000000000..4396566396d --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.error; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; + +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; + +/** Maps failures from the compatibility layer to version-aware Kafka responses. */ +@Internal +public final class KafkaErrorMapper { + + /** Converts a failure to the error response defined by the parsed Kafka request. */ + public AbstractResponse toResponse(AbstractRequest request, Throwable failure) { + return request.getErrorResponse(unwrap(failure)); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while ((current instanceof CompletionException || current instanceof ExecutionException) + && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java new file mode 100644 index 00000000000..a2b8a0dc60b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; + +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests response ordering and ownership in {@link KafkaCommandDecoder}. */ +public class KafkaCommandDecoderTest { + + @Test + public void testAcksZeroSuppressesResponseAndUnblocksFollowingResponse() { + RequestChannel requestChannel = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requestChannel}, "KAFKA")); + short produceVersion = ApiKeys.PRODUCE.latestVersion(); + ProduceRequest produceRequest = + new ProduceRequest( + new ProduceRequestData().setAcks((short) 0).setTimeoutMs(1000), + produceVersion); + RequestHeader produceHeader = + new RequestHeader(ApiKeys.PRODUCE, produceVersion, "client", 1); + ByteBuf produceBuffer = serialize(produceHeader, produceRequest); + + short apiVersionsVersion = ApiKeys.API_VERSIONS.latestVersion(); + ApiVersionsRequest apiVersionsRequest = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData(), + apiVersionsVersion, + apiVersionsVersion) + .build(); + RequestHeader apiVersionsHeader = + new RequestHeader(ApiKeys.API_VERSIONS, apiVersionsVersion, "client", 2); + ByteBuf apiVersionsBuffer = serialize(apiVersionsHeader, apiVersionsRequest); + + try { + channel.writeInbound(produceBuffer); + channel.writeInbound(apiVersionsBuffer); + KafkaRequest first = (KafkaRequest) requestChannel.pollRequest(1000); + KafkaRequest second = (KafkaRequest) requestChannel.pollRequest(1000); + assertThat(first).isNotNull(); + assertThat(second).isNotNull(); + + second.complete(new ApiVersionsResponse(new ApiVersionsResponseData())); + channel.runPendingTasks(); + Object blockedResponse = channel.readOutbound(); + assertThat(blockedResponse).isNull(); + + first.complete(new ProduceResponse(new ProduceResponseData())); + channel.runPendingTasks(); + + ByteBuf response = channel.readOutbound(); + try { + assertThat(response).isNotNull(); + ResponseHeader responseHeader = + ResponseHeader.parse( + response.nioBuffer(), + apiVersionsHeader.toResponseHeader().headerVersion()); + assertThat(responseHeader.correlationId()).isEqualTo(2); + Object additionalResponse = channel.readOutbound(); + assertThat(additionalResponse).isNull(); + } finally { + if (response != null) { + response.release(); + } + } + + assertThat(produceBuffer.refCnt()).isZero(); + assertThat(apiVersionsBuffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + + private static ByteBuf serialize(RequestHeader header, AbstractRequest request) { + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + return Unpooled.wrappedBuffer(serialized); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 24e4ce8a6ce..8613d83df32 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; @@ -46,21 +47,15 @@ public void testKafkaApiVersionsNotSupported() { new ApiVersionsRequest.Builder().build(latestVersion); ChannelHandlerContext ctx = new TestingChannelHandlerContext(); KafkaRequest request = - new KafkaRequest( + newRequest( ApiKeys.API_VERSIONS, (short) (latestVersion + 1), // unsupported version new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), apiVersionsRequest, - ByteBufAllocator.DEFAULT.buffer(), - ctx, - new CompletableFuture<>()); + ctx); handler.handleApiVersionsRequest(request); - ByteBuf responseBuffer = request.responseBuffer(); - ApiVersionsResponse response = - (ApiVersionsResponse) - AbstractResponse.parseResponse( - responseBuffer.nioBuffer(), request.header()); + ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); Map errorCounts = response.errorCounts(); assertThat(1).isEqualTo(errorCounts.size()); assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); @@ -74,21 +69,15 @@ public void testKafkaApiVersionsRequest() { new ApiVersionsRequest.Builder().build(latestVersion); ChannelHandlerContext ctx = new TestingChannelHandlerContext(); KafkaRequest request = - new KafkaRequest( + newRequest( ApiKeys.API_VERSIONS, latestVersion, new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), apiVersionsRequest, - ByteBufAllocator.DEFAULT.buffer(), - ctx, - new CompletableFuture<>()); + ctx); handler.handleApiVersionsRequest(request); - ByteBuf responseBuffer = request.responseBuffer(); - ApiVersionsResponse response = - (ApiVersionsResponse) - AbstractResponse.parseResponse( - responseBuffer.nioBuffer(), request.header()); + ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); Map errorCounts = response.errorCounts(); assertThat(1).isEqualTo(errorCounts.size()); assertThat(1).isEqualTo(errorCounts.get(Errors.NONE)); @@ -112,6 +101,37 @@ public void testKafkaApiVersionsRequest() { }); } + private static KafkaRequest newRequest( + ApiKeys apiKey, + short apiVersion, + RequestHeader header, + AbstractRequest requestBody, + ChannelHandlerContext context) { + ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); + try { + return new KafkaRequest( + apiKey, + apiVersion, + header, + requestBody, + requestBuffer, + context, + new CompletableFuture<>()); + } finally { + // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest. + requestBuffer.release(); + } + } + + private static AbstractResponse parseResponse(KafkaRequest request) { + ByteBuf responseBuffer = request.responseBuffer(); + try { + return AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + } + } + private static KafkaRequestHandler createKafkaRequestHandler() { return new KafkaRequestHandler(new TestingTabletGatewayService()); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java new file mode 100644 index 00000000000..2aa9caf9ba9 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; + +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for {@link KafkaRequest}. */ +public class KafkaRequestTest { + + @Test + public void testReleaseBufferIsIdempotent() { + short version = ApiKeys.API_VERSIONS.oldestVersion(); + ByteBuf buffer = mock(ByteBuf.class); + when(buffer.retain()).thenReturn(buffer); + KafkaRequest request = + new KafkaRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 1), + new ApiVersionsRequest.Builder().build(version), + buffer, + mock(ChannelHandlerContext.class), + new CompletableFuture<>()); + + request.releaseBuffer(); + request.releaseBuffer(); + + verify(buffer).retain(); + verify(buffer).release(); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java new file mode 100644 index 00000000000..84a889c16bd --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.kafka.KafkaRequestContext; + +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link KafkaApiRegistry}. */ +public class KafkaApiRegistryTest { + + @Test + public void testRejectDuplicateRegistrationAndRegistrationAfterFreeze() { + KafkaApiRegistry registry = brokerRegistry(); + TestingApiVersionsHandler handler = new TestingApiVersionsHandler(true); + registry.register(handler); + + assertThatThrownBy(() -> registry.register(handler)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("already registered"); + + registry.freeze(); + assertThatThrownBy(() -> registry.register(handler)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already frozen"); + } + + @Test + public void testOnlyAdvertiseEnabledHandlers() { + KafkaApiRegistry registry = brokerRegistry(); + registry.register(new TestingApiVersionsHandler(true)); + assertThat(registry.advertisedApiSpecs()).hasSize(1); + + KafkaApiRegistry hiddenRegistry = brokerRegistry(); + hiddenRegistry.register(new TestingApiVersionsHandler(false)); + assertThat(hiddenRegistry.advertisedApiSpecs()).isEmpty(); + assertThat(hiddenRegistry.lookup(ApiKeys.API_VERSIONS)).isNull(); + } + + @Test + public void testAdvertisedSpecIsSameSpecUsedForRouting() { + KafkaApiRegistry registry = brokerRegistry(); + TestingApiVersionsHandler handler = new TestingApiVersionsHandler(true); + registry.register(handler); + registry.freeze(); + + KafkaApiSpec advertisedSpec = registry.advertisedApiSpecs().get(0); + KafkaApiHandler routedHandler = registry.lookup(ApiKeys.API_VERSIONS); + + assertThat(routedHandler).isSameAs(handler); + assertThat(routedHandler.apiSpec()).isSameAs(advertisedSpec); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + assertThat(advertisedSpec.supportsVersion(version)).isTrue(); + } + assertThat( + advertisedSpec.supportsVersion( + (short) (ApiKeys.API_VERSIONS.latestVersion() + 1))) + .isFalse(); + } + + @Test + public void testRejectInvalidVersionRange() { + assertThatThrownBy(() -> new KafkaApiSpec(ApiKeys.API_VERSIONS, (short) 1, (short) 0, true)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy( + () -> + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + (short) (ApiKeys.API_VERSIONS.latestVersion() + 1), + true)) + .isInstanceOf(IllegalArgumentException.class); + } + + private static KafkaApiRegistry brokerRegistry() { + return new KafkaApiRegistry(); + } + + private static final class TestingApiVersionsHandler + implements KafkaApiHandler { + + private final KafkaApiSpec spec; + + private TestingApiVersionsHandler(boolean advertised) { + this.spec = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + advertised); + } + + @Override + public KafkaApiSpec apiSpec() { + return spec; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java new file mode 100644 index 00000000000..97f42350198 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.kafka.KafkaRequest; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; + +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Tests request routing and failure handling without a concrete API implementation. */ +class KafkaRequestDispatcherTest { + + @Test + void testUnregisteredApiReturnsUnsupportedVersion() { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.freeze(); + + AbstractResponse response = + new KafkaRequestDispatcher(registry, new KafkaErrorMapper()) + .dispatch(request((short) 0)) + .join(); + + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); + } + + @Test + void testUnsupportedVersionDoesNotInvokeHandler() { + KafkaRequestDispatcher dispatcher = + dispatcher( + (context, request) -> { + throw new AssertionError( + "Unsupported versions must not be dispatched."); + }); + + AbstractResponse response = dispatcher.dispatch(request((short) 1)).join(); + + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); + } + + @Test + void testDispatchWaitsForHandlerAndPreservesContext() { + CompletableFuture handlerResult = new CompletableFuture<>(); + AtomicReference receivedContext = new AtomicReference<>(); + KafkaRequestDispatcher dispatcher = + dispatcher( + (context, request) -> { + receivedContext.set(context); + return handlerResult; + }); + KafkaRequest request = request((short) 0); + + CompletableFuture result = dispatcher.dispatch(request); + + assertThat(result).isNotDone(); + assertThat(receivedContext.get().clientId()).isEqualTo("client"); + assertThat(receivedContext.get().correlationId()).isEqualTo(42); + assertThat(receivedContext.get().listenerName()).isEqualTo("KAFKA"); + assertThat(receivedContext.get().apiKey()).isEqualTo(ApiKeys.API_VERSIONS); + assertThat(receivedContext.get().apiVersion()).isZero(); + AbstractResponse response = new ApiVersionsResponse(new ApiVersionsResponseData()); + handlerResult.complete(response); + assertThat(result.join()).isSameAs(response); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testSynchronousAndAsynchronousFailuresBecomeErrorResponses(boolean synchronous) { + InvalidRequestException failure = new InvalidRequestException("invalid request"); + KafkaRequestDispatcher dispatcher = + dispatcher( + (context, request) -> { + if (synchronous) { + throw failure; + } + CompletableFuture result = new CompletableFuture<>(); + result.completeExceptionally(new CompletionException(failure)); + return result; + }); + + AbstractResponse response = dispatcher.dispatch(request((short) 0)).join(); + + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testNullFutureAndNullResponseBecomeErrorResponses(boolean nullFuture) { + KafkaRequestDispatcher dispatcher = + dispatcher( + (context, request) -> + nullFuture ? null : CompletableFuture.completedFuture(null)); + + AbstractResponse response = dispatcher.dispatch(request((short) 0)).join(); + + assertThat(response.errorCounts()).containsEntry(Errors.UNKNOWN_SERVER_ERROR, 1); + } + + private static KafkaRequestDispatcher dispatcher( + BiFunction> + action) { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register( + new KafkaApiHandler() { + @Override + public KafkaApiSpec apiSpec() { + return new KafkaApiSpec(ApiKeys.API_VERSIONS, (short) 0, (short) 0, true); + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + return action.apply(context, request); + } + }); + registry.freeze(); + return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); + } + + private static KafkaRequest request(short version) { + KafkaRequest request = mock(KafkaRequest.class); + when(request.apiKey()).thenReturn(ApiKeys.API_VERSIONS); + when(request.apiVersion()).thenReturn(version); + when(request.request()).thenReturn(new ApiVersionsRequest.Builder().build(version)); + when(request.header()) + .thenReturn(new RequestHeader(ApiKeys.API_VERSIONS, version, "client", 42)); + when(request.listenerName()).thenReturn("KAFKA"); + when(request.ctx()).thenReturn(mock(ChannelHandlerContext.class)); + return request; + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java index 03d798fb371..df2256985c2 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java @@ -232,8 +232,17 @@ private static List loadProtocols( NetworkProtocolPlugin kafkaPlugin = loadProtocolPlugin(NetworkProtocolPlugin.KAFKA_PROTOCOL_NAME); kafkaPlugin.setup(conf); - listeners.removeAll(kafkaPlugin.listenerNames()); - protocolPlugins.add(kafkaPlugin); + List kafkaListenerNames = kafkaPlugin.listenerNames(); + boolean hasKafkaEndpoint = + endpoints.stream() + .anyMatch( + endpoint -> + kafkaListenerNames.contains( + endpoint.getListenerName())); + if (hasKafkaEndpoint) { + listeners.removeAll(kafkaListenerNames); + protocolPlugins.add(kafkaPlugin); + } } // Add the Fluss protocol plugin in the end to allow other protocol From c557efc22af0b8225f1ebbdd629423cdf409a193 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 16 Sep 2026 17:35:40 +0800 Subject: [PATCH 02/11] [kafka] Complete dispatcher futures when error mapping fails Catch failures while mapping asynchronous handler results so the dispatcher future completes exceptionally instead of remaining pending. Cover handler futures that fail before and after dispatch registers its callback. Validated with Java 11: Maven reactor build and 17 targeted Kafka tests, including Checkstyle, Spotless, and license checks. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 27/27 AI-Contributed/UT: 28/28 --- .../dispatcher/KafkaRequestDispatcher.java | 27 +++++++++++------- .../KafkaRequestDispatcherTest.java | 28 +++++++++++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java index efdfe33dd66..235303f5f0e 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java @@ -43,7 +43,7 @@ public KafkaRequestDispatcher(KafkaApiRegistry registry, KafkaErrorMapper errorM this.errorMapper = checkNotNull(errorMapper); } - /** Dispatches a request and always completes with a Kafka protocol response. */ + /** Dispatches a request and maps handler failures to Kafka protocol responses. */ public CompletableFuture dispatch(KafkaRequest request) { AbstractRequest abstractRequest = request.request(); KafkaApiHandler handler = registry.lookup(request.apiKey()); @@ -81,15 +81,22 @@ public CompletableFuture dispatch(KafkaRequest request) { CompletableFuture result = new CompletableFuture<>(); responseFuture.whenComplete( (response, failure) -> { - if (failure == null && response != null) { - result.complete(response); - } else { - Throwable responseFailure = - failure == null - ? new NullPointerException( - "Kafka API handler returned a null response.") - : failure; - result.complete(errorMapper.toResponse(abstractRequest, responseFailure)); + try { + if (failure == null && response != null) { + result.complete(response); + } else { + Throwable responseFailure = + failure == null + ? new NullPointerException( + "Kafka API handler returned a null response.") + : failure; + result.complete( + errorMapper.toResponse(abstractRequest, responseFailure)); + } + } catch (Throwable completionFailure) { + // Completion callbacks must never leave the ordered Kafka response queue + // waiting on a future that can no longer become terminal. + result.completeExceptionally(completionFailure); } }); return result; diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java index 97f42350198..a1e244bf310 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java @@ -40,6 +40,8 @@ import java.util.function.BiFunction; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -131,6 +133,32 @@ void testNullFutureAndNullResponseBecomeErrorResponses(boolean nullFuture) { assertThat(response.errorCounts()).containsEntry(Errors.UNKNOWN_SERVER_ERROR, 1); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testErrorMappingFailureCompletesDispatcherFutureExceptionally( + boolean handlerAlreadyCompleted) { + CompletableFuture handlerResult = new CompletableFuture<>(); + KafkaRequestDispatcher dispatcher = dispatcher((context, request) -> handlerResult); + KafkaRequest request = request((short) 0); + ApiVersionsRequest requestBody = mock(ApiVersionsRequest.class); + IllegalStateException mappingFailure = new IllegalStateException("error mapping failure"); + when(requestBody.getErrorResponse(any(Throwable.class))).thenThrow(mappingFailure); + when(request.request()).thenReturn(requestBody); + InvalidRequestException handlerFailure = new InvalidRequestException("invalid request"); + if (handlerAlreadyCompleted) { + handlerResult.completeExceptionally(handlerFailure); + } + + CompletableFuture result = dispatcher.dispatch(request); + if (!handlerAlreadyCompleted) { + assertThat(result).isNotDone(); + handlerResult.completeExceptionally(handlerFailure); + } + + assertThat(result).isCompletedExceptionally(); + assertThatThrownBy(result::join).hasCause(mappingFailure); + } + private static KafkaRequestDispatcher dispatcher( BiFunction> action) { From 5742b36627fe156a8cf499f0aa7b4b8c4163e9a7 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 8 Sep 2026 11:31:07 +0800 Subject: [PATCH 03/11] [kafka] Serve ApiVersions from registered capabilities Route requests through the dispatcher and advertise only implemented APIs. Return version-aware errors for unsupported APIs and invalid requests. Validated with mvn -o -pl fluss-kafka verify (23 unit tests and 1 IT). Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 313/313 AI-Contributed/UT: 137/137 --- .../fluss/kafka/KafkaProtocolPlugin.java | 3 +- .../fluss/kafka/KafkaRequestHandler.java | 232 ++---------------- .../api/versions/ApiVersionsHandler.java | 78 ++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 137 ++++++++--- 4 files changed, 207 insertions(+), 243 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index c966f745b8c..939c35a2d95 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -66,7 +66,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " + service.getClass().getSimpleName()); } - TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(gateway); + return new KafkaRequestHandler(); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 73555093ff0..2df16a8bd7f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,27 +17,24 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import org.apache.kafka.common.message.ApiVersionsResponseData; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.requests.AbstractRequest; -import org.apache.kafka.common.requests.AbstractResponse; -import org.apache.kafka.common.requests.ApiVersionsResponse; - -/** Kafka protocol implementation for request handler. */ +/** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { - // TODO: we may need a new abstraction between TabletService and ReplicaManager to avoid - // affecting Fluss protocol when supporting compatibility with Kafka. - private final TabletServerGateway gateway; + private final KafkaRequestDispatcher dispatcher; - public KafkaRequestHandler(TabletServerGateway gateway) { - this.gateway = gateway; + /** Creates a Kafka request handler with the implemented server capabilities. */ + public KafkaRequestHandler() { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ApiVersionsHandler(registry)); + registry.freeze(); + this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @Override @@ -47,200 +44,15 @@ public RequestType requestType() { @Override public void processRequest(KafkaRequest request) { - // See kafka.server.KafkaApis#handle - switch (request.apiKey()) { - case API_VERSIONS: - handleApiVersionsRequest(request); - break; - case METADATA: - handleMetadataRequest(request); - break; - case PRODUCE: - handleProducerRequest(request); - break; - case FIND_COORDINATOR: - handleFindCoordinatorRequest(request); - break; - case LIST_OFFSETS: - handleListOffsetRequest(request); - break; - case OFFSET_FETCH: - handleOffsetFetchRequest(request); - break; - case OFFSET_COMMIT: - handleOffsetCommitRequest(request); - break; - case FETCH: - handleFetchRequest(request); - break; - case JOIN_GROUP: - handleJoinGroupRequest(request); - break; - case SYNC_GROUP: - handleSyncGroupRequest(request); - break; - case HEARTBEAT: - handleHeartbeatRequest(request); - break; - case LEAVE_GROUP: - handleLeaveGroupRequest(request); - break; - case DESCRIBE_GROUPS: - handleDescribeGroupsRequest(request); - break; - case LIST_GROUPS: - handleListGroupsRequest(request); - break; - case DELETE_GROUPS: - handleDeleteGroupsRequest(request); - break; - case SASL_HANDSHAKE: - handleSaslHandshakeRequest(request); - break; - case SASL_AUTHENTICATE: - handleSaslAuthenticateRequest(request); - break; - case CREATE_TOPICS: - handleCreateTopicsRequest(request); - break; - case INIT_PRODUCER_ID: - handleInitProducerIdRequest(request); - break; - case ADD_PARTITIONS_TO_TXN: - handleAddPartitionsToTxnRequest(request); - break; - case ADD_OFFSETS_TO_TXN: - handleAddOffsetsToTxnRequest(request); - break; - case TXN_OFFSET_COMMIT: - handleTxnOffsetCommitRequest(request); - break; - case END_TXN: - handleEndTxnRequest(request); - break; - case WRITE_TXN_MARKERS: - handleWriteTxnMarkersRequest(request); - break; - case DESCRIBE_CONFIGS: - handleDescribeConfigsRequest(request); - break; - case ALTER_CONFIGS: - handleAlterConfigsRequest(request); - break; - case DELETE_TOPICS: - handleDeleteTopicsRequest(request); - break; - case DELETE_RECORDS: - handleDeleteRecordsRequest(request); - break; - case OFFSET_DELETE: - handleOffsetDeleteRequest(request); - break; - case CREATE_PARTITIONS: - handleCreatePartitionsRequest(request); - break; - case DESCRIBE_CLUSTER: - handleDescribeClusterRequest(request); - break; - default: - handleUnsupportedRequest(request); - } - } - - private void handleUnsupportedRequest(KafkaRequest request) { - String message = String.format("Unsupported request with api key %s", request.apiKey()); - AbstractRequest abstractRequest = request.request(); - AbstractResponse response = - abstractRequest.getErrorResponse(new UnsupportedOperationException(message)); - request.complete(response); - } - - void handleApiVersionsRequest(KafkaRequest request) { - short apiVersion = request.apiVersion(); - if (!ApiKeys.API_VERSIONS.isVersionSupported(apiVersion)) { - request.fail(Errors.UNSUPPORTED_VERSION.exception()); - return; - } - ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (ApiKeys apiKey : ApiKeys.values()) { - if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { - ApiVersionsResponseData.ApiVersion apiVersionData = - new ApiVersionsResponseData.ApiVersion() - .setApiKey(apiKey.id) - .setMinVersion(apiKey.oldestVersion()) - .setMaxVersion(apiKey.latestVersion()); - if (apiKey.equals(ApiKeys.METADATA)) { - // Not support TopicId - short v = apiKey.latestVersion() > 11 ? 11 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } else if (apiKey.equals(ApiKeys.FETCH)) { - // Not support TopicId - short v = apiKey.latestVersion() > 12 ? 12 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } - data.apiKeys().add(apiVersionData); - } - } - request.complete(new ApiVersionsResponse(data)); + dispatcher + .dispatch(request) + .whenComplete( + (response, failure) -> { + if (failure == null) { + request.complete(response); + } else { + request.fail(failure); + } + }); } - - void handleProducerRequest(KafkaRequest request) {} - - void handleMetadataRequest(KafkaRequest request) {} - - void handleFindCoordinatorRequest(KafkaRequest request) {} - - void handleListOffsetRequest(KafkaRequest request) {} - - void handleOffsetFetchRequest(KafkaRequest request) {} - - void handleOffsetCommitRequest(KafkaRequest request) {} - - void handleFetchRequest(KafkaRequest request) {} - - void handleJoinGroupRequest(KafkaRequest request) {} - - void handleSyncGroupRequest(KafkaRequest request) {} - - void handleHeartbeatRequest(KafkaRequest request) {} - - void handleLeaveGroupRequest(KafkaRequest request) {} - - void handleDescribeGroupsRequest(KafkaRequest request) {} - - void handleListGroupsRequest(KafkaRequest request) {} - - void handleDeleteGroupsRequest(KafkaRequest request) {} - - void handleSaslHandshakeRequest(KafkaRequest request) {} - - void handleSaslAuthenticateRequest(KafkaRequest request) {} - - void handleCreateTopicsRequest(KafkaRequest request) {} - - void handleInitProducerIdRequest(KafkaRequest request) {} - - void handleAddPartitionsToTxnRequest(KafkaRequest request) {} - - void handleAddOffsetsToTxnRequest(KafkaRequest request) {} - - void handleTxnOffsetCommitRequest(KafkaRequest request) {} - - void handleEndTxnRequest(KafkaRequest request) {} - - void handleWriteTxnMarkersRequest(KafkaRequest request) {} - - void handleDescribeConfigsRequest(KafkaRequest request) {} - - void handleAlterConfigsRequest(KafkaRequest request) {} - - void handleDeleteTopicsRequest(KafkaRequest request) {} - - void handleDeleteRecordsRequest(KafkaRequest request) {} - - void handleOffsetDeleteRequest(KafkaRequest request) {} - - void handleCreatePartitionsRequest(KafkaRequest request) {} - - void handleDescribeClusterRequest(KafkaRequest request) {} } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java new file mode 100644 index 00000000000..c38d7bc6cb2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements ApiVersions from the capabilities actually registered on this server. */ +@Internal +public final class ApiVersionsHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + true); + + private final KafkaApiRegistry registry; + + /** Creates an ApiVersions handler backed by the server capability registry. */ + public ApiVersionsHandler(KafkaApiRegistry registry) { + this.registry = checkNotNull(registry); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + if (!request.isValid()) { + return CompletableFuture.completedFuture( + request.getErrorResponse(Errors.INVALID_REQUEST.exception())); + } + ApiVersionsResponseData data = new ApiVersionsResponseData(); + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + data.apiKeys() + .add( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(spec.apiKey().id) + .setMinVersion(spec.minVersion()) + .setMaxVersion(spec.maxVersion())); + } + return CompletableFuture.completedFuture(new ApiVersionsResponse(data)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 8613d83df32..e6b8e961128 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,24 +17,32 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; /** Tests for {@link KafkaRequestHandler}. */ public class KafkaRequestHandlerTest { @@ -53,7 +61,7 @@ public void testKafkaApiVersionsNotSupported() { new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), apiVersionsRequest, ctx); - handler.handleApiVersionsRequest(request); + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); Map errorCounts = response.errorCounts(); @@ -61,44 +69,111 @@ public void testKafkaApiVersionsNotSupported() { assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); } + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + public void testKafkaApiVersionsRequest(short version) { + KafkaRequestHandler handler = createKafkaRequestHandler(); + ApiVersionsResponse response = requestApiVersions(handler, version); + + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + } + + private static ApiVersionsResponse requestApiVersions( + KafkaRequestHandler handler, short version) { + ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build(version); + ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + KafkaRequest request = + newRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), + apiVersionsRequest, + ctx); + handler.processRequest(request); + + return parseApiVersionsResponse(request); + } + + private static ApiVersionsResponse parseApiVersionsResponse(KafkaRequest request) { + return (ApiVersionsResponse) parseResponse(request); + } + + private static void assertSuccessfulResponseDefaults(ApiVersionsResponse response) { + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1)); + assertThat(response.data().throttleTimeMs()).isZero(); + assertThat(response.data().supportedFeatures()).isEmpty(); + assertThat(response.data().finalizedFeaturesEpoch()).isEqualTo(-1L); + assertThat(response.data().finalizedFeatures()).isEmpty(); + assertThat(response.data().zkMigrationReady()).isFalse(); + } + + private static void assertBrokerCapabilities(ApiVersionsResponse response) { + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion())); + } + @Test - public void testKafkaApiVersionsRequest() { + public void testInvalidApiVersionsRequest() { KafkaRequestHandler handler = createKafkaRequestHandler(); short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + ApiVersionsRequest requestBody = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData() + .setClientSoftwareName("invalid client name") + .setClientSoftwareVersion("1.0"), + latestVersion, + latestVersion) + .build(latestVersion); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, latestVersion, new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, - ctx); - handler.handleApiVersionsRequest(request); + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.NONE)); - response.data() - .apiKeys() - .forEach( - apiVersion -> { - if (ApiKeys.METADATA.id == apiVersion.apiKey()) { - assertThat((short) 11) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else if (ApiKeys.FETCH.id == apiVersion.apiKey()) { - assertThat((short) 12) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else { - ApiKeys apiKeys = ApiKeys.forId(apiVersion.apiKey()); - assertThat(apiVersion.minVersion()) - .isEqualTo(apiKeys.oldestVersion()); - assertThat(apiVersion.maxVersion()) - .isEqualTo(apiKeys.latestVersion()); - } - }); + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + } + + @Test + public void testUnregisteredApiIsNotRouted() { + KafkaRequestHandler handler = createKafkaRequestHandler(); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + CreateTopicsRequestData requestData = + new CreateTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + new CreateTopicsRequestData.CreatableTopicCollection( + Collections.singletonList( + new CreateTopicsRequestData.CreatableTopic() + .setName("topic") + .setNumPartitions(1) + .setReplicationFactor((short) 1)) + .iterator())); + CreateTopicsRequest requestBody = + new CreateTopicsRequest.Builder(requestData).build(version); + KafkaRequest request = + newRequest( + ApiKeys.CREATE_TOPICS, + version, + new RequestHeader(ApiKeys.CREATE_TOPICS, version, "client-id", 0), + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); + + CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request); + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); } private static KafkaRequest newRequest( @@ -133,6 +208,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(new TestingTabletGatewayService()); + return new KafkaRequestHandler(); } } From d72a3bcaf763fdbae24e44b86707246da62a03e3 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 15:28:16 +0800 Subject: [PATCH 04/11] [kafka] Define DDL table mapping for Kafka compatibility Extract topic identity and the raw/string table mapping contract before Metadata. Validate table kinds, field projections and metadata columns independently of request handling and record decoding. Validation: Java 11, mvn -o -pl fluss-kafka clean verify (47 unit tests and 2 integration tests); Checkstyle, Spotless and RAT passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 769/769 AI-Contributed/UT: 480/480 --- fluss-kafka/pom.xml | 7 + .../fluss/kafka/format/KafkaDataFormat.java | 73 ++++ .../fluss/kafka/mapping/KafkaTopicMapper.java | 73 ++++ .../kafka/schema/KafkaFieldProjection.java | 123 +++++++ .../fluss/kafka/schema/KafkaTopicSchema.java | 150 ++++++++ .../schema/KafkaTopicSchemaException.java | 30 ++ .../schema/KafkaTopicSchemaResolver.java | 313 +++++++++++++++++ .../kafka/mapping/KafkaTopicMapperTest.java | 62 ++++ .../kafka/schema/KafkaTableMappingITCase.java | 88 +++++ .../schema/KafkaTopicSchemaResolverTest.java | 330 ++++++++++++++++++ 10 files changed, 1249 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index f48a94ab9e9..b77695848ca 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,6 +64,13 @@ + + org.apache.curator + curator-test + ${curator.version} + test + + org.apache.fluss fluss-test-utils diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java new file mode 100644 index 00000000000..ad0039b7e78 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import java.util.Locale; + +/** Supported interpretations of Kafka record key and value bytes. */ +@Internal +public enum KafkaDataFormat { + RAW("raw"), + STRING("string"); + + /** Fluss table custom property controlling the record key format. */ + public static final String KEY_FORMAT_CONFIG = "kafka.key.format"; + + /** Fluss table custom property controlling the record value format. */ + public static final String VALUE_FORMAT_CONFIG = "kafka.value.format"; + + /** Fluss fields populated from the Kafka record key. */ + public static final String KEY_FIELDS_CONFIG = "kafka.key.fields"; + + /** Strategy for deriving fields populated from the Kafka record value. */ + public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; + + /** Fluss column populated from the Kafka record timestamp. */ + public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; + + /** Fluss column populated from the Kafka record headers. */ + public static final String HEADERS_COLUMN_CONFIG = "kafka.metadata.headers.column"; + + private final String value; + + KafkaDataFormat(String value) { + this.value = value; + } + + /** Parses a table custom property value. */ + public static KafkaDataFormat parse(String value) { + if (value == null) { + throw new IllegalArgumentException("Kafka data format must not be null."); + } + String normalized = value.trim().toLowerCase(Locale.ROOT); + for (KafkaDataFormat format : values()) { + if (format.value.equals(normalized)) { + return format; + } + } + throw new IllegalArgumentException( + "Unsupported Kafka data format '" + value + "'. Expected raw or string."); + } + + /** Returns the persisted table property value. */ + public String value() { + return value; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java new file mode 100644 index 00000000000..9f8ed63f543 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.Topic; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +@Internal +public final class KafkaTopicMapper { + + // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. + private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + + private final String databaseName; + + /** Creates a topic mapper for one Fluss database. */ + public KafkaTopicMapper(String databaseName) { + this.databaseName = checkNotNull(databaseName); + } + + /** Maps a Kafka topic name to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + Topic.validate(topicName); + return TablePath.of(databaseName, topicName); + } + + /** Returns whether a table belongs to this database and has a valid Kafka topic name. */ + public boolean isMappedTable(TablePath tablePath) { + return databaseName.equals(tablePath.getDatabaseName()) + && Topic.isValid(tablePath.getTableName()); + } + + /** Maps a Fluss table ID to a stable Kafka topic ID. */ + public Uuid toTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(TOPIC_ID_NAMESPACE, tableId); + } + + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ + public boolean isMappedTopicId(Uuid topicId) { + return topicId != null + && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && topicId.getLeastSignificantBits() >= 0L; + } + + /** Extracts the Fluss table ID encoded in a Kafka topic ID. */ + public long toTableId(Uuid topicId) { + checkArgument(isMappedTopicId(topicId), "Topic ID %s is not a Fluss topic ID.", topicId); + return topicId.getLeastSignificantBits(); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java new file mode 100644 index 00000000000..b1715758a9d --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Ordered physical Fluss fields populated by one Kafka record component. */ +@Internal +public final class KafkaFieldProjection { + + private final List positions; + private final List names; + private final List dataTypes; + + /** Creates a projection from physical row positions. */ + public KafkaFieldProjection(RowType rowType, List positions) { + checkNotNull(rowType); + checkNotNull(positions); + List positionCopy = new ArrayList<>(positions.size()); + List projectedNames = new ArrayList<>(positions.size()); + List projectedTypes = new ArrayList<>(positions.size()); + for (Integer position : positions) { + checkArgument( + position != null && position >= 0 && position < rowType.getFieldCount(), + "Invalid Kafka field projection position %s.", + position); + positionCopy.add(position); + projectedNames.add(rowType.getFieldNames().get(position)); + projectedTypes.add(rowType.getTypeAt(position)); + } + this.positions = Collections.unmodifiableList(positionCopy); + this.names = Collections.unmodifiableList(projectedNames); + this.dataTypes = Collections.unmodifiableList(projectedTypes); + } + + /** Returns the number of projected fields. */ + public int size() { + return positions.size(); + } + + /** Returns whether this projection owns no fields. */ + public boolean isEmpty() { + return positions.isEmpty(); + } + + /** Returns the physical row position at the projection position. */ + public int positionAt(int projectionPosition) { + return positions.get(projectionPosition); + } + + /** Returns the physical field name at the projection position. */ + public String nameAt(int projectionPosition) { + return names.get(projectionPosition); + } + + /** Returns the physical data type at the projection position. */ + public DataType dataTypeAt(int projectionPosition) { + return dataTypes.get(projectionPosition); + } + + /** Returns the projected physical positions. */ + public List positions() { + return positions; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaFieldProjection)) { + return false; + } + KafkaFieldProjection that = (KafkaFieldProjection) obj; + return Objects.equals(positions, that.positions) + && Objects.equals(names, that.names) + && Objects.equals(dataTypes, that.dataTypes); + } + + @Override + public int hashCode() { + return Objects.hash(positions, names, dataTypes); + } + + @Override + public String toString() { + return "KafkaFieldProjection{" + + "positions=" + + positions + + ", " + + "names=" + + names + + ", " + + "dataTypes=" + + dataTypes + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java new file mode 100644 index 00000000000..64263e69286 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.types.RowType; + +import javax.annotation.Nullable; + +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Resolved Kafka key, value, and metadata mapping for one Fluss table schema. */ +@Internal +public final class KafkaTopicSchema { + + private final RowType rowType; + private final @Nullable KafkaDataFormat keyFormat; + private final KafkaFieldProjection keyProjection; + private final KafkaDataFormat valueFormat; + private final KafkaFieldProjection valueProjection; + private final int timestampPosition; + private final int headersPosition; + + /** Creates a resolved Kafka topic schema. */ + public KafkaTopicSchema( + RowType rowType, + @Nullable KafkaDataFormat keyFormat, + KafkaFieldProjection keyProjection, + KafkaDataFormat valueFormat, + KafkaFieldProjection valueProjection, + int timestampPosition, + int headersPosition) { + this.rowType = checkNotNull(rowType); + this.keyFormat = keyFormat; + this.keyProjection = checkNotNull(keyProjection); + this.valueFormat = checkNotNull(valueFormat); + this.valueProjection = checkNotNull(valueProjection); + this.timestampPosition = timestampPosition; + this.headersPosition = headersPosition; + } + + /** Returns the physical Fluss row type. */ + public RowType rowType() { + return rowType; + } + + /** Returns the key format, or null when the Kafka key is not mapped. */ + public @Nullable KafkaDataFormat keyFormat() { + return keyFormat; + } + + /** Returns the key field projection. */ + public KafkaFieldProjection keyProjection() { + return keyProjection; + } + + /** Returns the value format. */ + public KafkaDataFormat valueFormat() { + return valueFormat; + } + + /** Returns the value field projection. */ + public KafkaFieldProjection valueProjection() { + return valueProjection; + } + + /** Returns the timestamp physical position, or -1 when it is not mapped. */ + public int timestampPosition() { + return timestampPosition; + } + + /** Returns the headers physical position, or -1 when they are not mapped. */ + public int headersPosition() { + return headersPosition; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaTopicSchema)) { + return false; + } + KafkaTopicSchema that = (KafkaTopicSchema) obj; + return Objects.equals(rowType, that.rowType) + && Objects.equals(keyFormat, that.keyFormat) + && Objects.equals(keyProjection, that.keyProjection) + && Objects.equals(valueFormat, that.valueFormat) + && Objects.equals(valueProjection, that.valueProjection) + && Objects.equals(timestampPosition, that.timestampPosition) + && Objects.equals(headersPosition, that.headersPosition); + } + + @Override + public int hashCode() { + return Objects.hash( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + @Override + public String toString() { + return "KafkaTopicSchema{" + + "rowType=" + + rowType + + ", " + + "keyFormat=" + + keyFormat + + ", " + + "keyProjection=" + + keyProjection + + ", " + + "valueFormat=" + + valueFormat + + ", " + + "valueProjection=" + + valueProjection + + ", " + + "timestampPosition=" + + timestampPosition + + ", " + + "headersPosition=" + + headersPosition + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java new file mode 100644 index 00000000000..0e6a294c4d5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that a Fluss table does not define a valid Kafka record mapping contract. */ +@Internal +public final class KafkaTopicSchemaException extends IllegalArgumentException { + + /** Creates a Kafka topic schema exception. */ + public KafkaTopicSchemaException(String message) { + super(message); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java new file mode 100644 index 00000000000..840ce8a4f58 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** Resolves and validates the Kafka record mapping stored in Fluss table custom properties. */ +@Internal +public final class KafkaTopicSchemaResolver { + + private static final String INCLUDE_ALL = "ALL"; + private static final String INCLUDE_EXCEPT_KEY = "EXCEPT_KEY"; + + private static final Set SUPPORTED_PROPERTIES = + new HashSet<>( + Arrays.asList( + KafkaDataFormat.KEY_FORMAT_CONFIG, + KafkaDataFormat.KEY_FIELDS_CONFIG, + KafkaDataFormat.VALUE_FORMAT_CONFIG, + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, + KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + + /** Resolves one table's Kafka record mapping contract. */ + public KafkaTopicSchema resolve(TableDescriptor table) { + validateTableKind(table); + RowType rowType = table.getSchema().getRowType(); + Map properties = table.getCustomProperties(); + + for (String property : properties.keySet()) { + if (property.startsWith("kafka.") && !SUPPORTED_PROPERTIES.contains(property)) { + throw invalid("Unsupported Kafka table property '" + property + "'."); + } + } + + int timestampPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG)); + int headersPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + if (timestampPosition >= 0 && timestampPosition == headersPosition) { + throw invalid("Kafka timestamp and headers cannot map to the same Fluss column."); + } + validateMetadataColumns(rowType, timestampPosition, headersPosition); + + String keyFormatValue = properties.get(KafkaDataFormat.KEY_FORMAT_CONFIG); + KafkaDataFormat keyFormat = keyFormatValue == null ? null : parseFormat(keyFormatValue); + List keyPositions = + resolveKeyPositions( + rowType, keyFormat, properties.get(KafkaDataFormat.KEY_FIELDS_CONFIG)); + for (Integer keyPosition : keyPositions) { + if (keyPosition == timestampPosition || keyPosition == headersPosition) { + throw invalid( + "Kafka key field '" + + rowType.getFieldNames().get(keyPosition) + + "' cannot be a Kafka metadata column."); + } + } + KafkaFieldProjection keyProjection = new KafkaFieldProjection(rowType, keyPositions); + + String valueFormatValue = properties.get(KafkaDataFormat.VALUE_FORMAT_CONFIG); + if (valueFormatValue == null) { + throw invalid( + "Missing required table property '" + + KafkaDataFormat.VALUE_FORMAT_CONFIG + + "'."); + } + KafkaDataFormat valueFormat = parseFormat(valueFormatValue); + String fieldsInclude = + normalizeFieldsInclude(properties.get(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG)); + if (!keyPositions.isEmpty() && INCLUDE_ALL.equals(fieldsInclude)) { + throw invalid( + "Mapping Kafka key fields requires " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + "=EXCEPT_KEY."); + } + + Set metadataPositions = new HashSet<>(); + if (timestampPosition >= 0) { + metadataPositions.add(timestampPosition); + } + if (headersPosition >= 0) { + metadataPositions.add(headersPosition); + } + List valuePositions = + resolveValuePositions(rowType, fieldsInclude, keyPositions, metadataPositions); + KafkaFieldProjection valueProjection = new KafkaFieldProjection(rowType, valuePositions); + if (valueProjection.isEmpty()) { + throw invalid("Kafka value projection must contain at least one Fluss column."); + } + + validateSingleFieldFormat(keyFormat, keyProjection, "key"); + validateSingleFieldFormat(valueFormat, valueProjection, "value"); + return new KafkaTopicSchema( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + private static void validateTableKind(TableDescriptor table) { + if (table.hasPrimaryKey()) { + throw invalid("Kafka topic table must be a log table."); + } + if (table.isPartitioned()) { + throw invalid("Partitioned Fluss tables are not supported."); + } + if (new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() + != LogFormat.ARROW) { + throw invalid("Kafka topic table must use the Arrow log format."); + } + } + + private static List resolveKeyPositions( + RowType rowType, KafkaDataFormat keyFormat, String keyFieldsValue) { + if (keyFormat == null) { + if (keyFieldsValue != null) { + throw invalid( + KafkaDataFormat.KEY_FIELDS_CONFIG + + " requires " + + KafkaDataFormat.KEY_FORMAT_CONFIG + + "."); + } + return Collections.emptyList(); + } + if (keyFieldsValue == null || keyFieldsValue.trim().isEmpty()) { + throw invalid( + "Missing required table property '" + KafkaDataFormat.KEY_FIELDS_CONFIG + "'."); + } + String[] fieldNames = keyFieldsValue.split(",", -1); + List positions = new ArrayList<>(fieldNames.length); + Set uniquePositions = new HashSet<>(); + for (String fieldNameValue : fieldNames) { + String fieldName = fieldNameValue.trim(); + if (fieldName.isEmpty()) { + throw invalid("Kafka key field names must not be empty."); + } + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka key field '" + fieldName + "' does not exist."); + } + if (!uniquePositions.add(position)) { + throw invalid("Duplicate Kafka key field '" + fieldName + "'."); + } + positions.add(position); + } + return positions; + } + + private static String normalizeFieldsInclude(String value) { + String normalized = value == null ? INCLUDE_ALL : value.trim().toUpperCase(Locale.ROOT); + if (!INCLUDE_ALL.equals(normalized) && !INCLUDE_EXCEPT_KEY.equals(normalized)) { + throw invalid( + "Invalid " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + " '" + + value + + "'. Expected ALL or EXCEPT_KEY."); + } + return normalized; + } + + private static List resolveValuePositions( + RowType rowType, + String fieldsInclude, + List keyPositions, + Set metadataPositions) { + Set excludedKeyPositions = + INCLUDE_EXCEPT_KEY.equals(fieldsInclude) + ? new HashSet<>(keyPositions) + : Collections.emptySet(); + List positions = new ArrayList<>(); + for (int position = 0; position < rowType.getFieldCount(); position++) { + if (!metadataPositions.contains(position) && !excludedKeyPositions.contains(position)) { + positions.add(position); + } + } + return positions; + } + + private static int resolveOptionalPosition(RowType rowType, String fieldNameValue) { + if (fieldNameValue == null) { + return -1; + } + if (fieldNameValue.trim().isEmpty()) { + throw invalid("Kafka metadata column name must not be empty."); + } + String fieldName = fieldNameValue.trim(); + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka metadata column '" + fieldName + "' does not exist."); + } + return position; + } + + private static void validateMetadataColumns( + RowType rowType, int timestampPosition, int headersPosition) { + if (timestampPosition >= 0) { + if (!(rowType.getTypeAt(timestampPosition) instanceof LocalZonedTimestampType) + || rowType.getTypeAt(timestampPosition).isNullable() + || ((LocalZonedTimestampType) rowType.getTypeAt(timestampPosition)) + .getPrecision() + != 3) { + throw invalid("Kafka timestamp column must be TIMESTAMP_LTZ(3) NOT NULL."); + } + } + if (headersPosition >= 0) { + validateHeadersType(rowType.getTypeAt(headersPosition)); + } + } + + private static void validateHeadersType(org.apache.fluss.types.DataType dataType) { + if (!(dataType instanceof ArrayType) || !dataType.isNullable()) { + throw invalid( + "Kafka headers column must be nullable " + + "ARRAY>."); + } + ArrayType arrayType = (ArrayType) dataType; + if (!(arrayType.getElementType() instanceof RowType)) { + throw invalid("Kafka headers elements must be rows."); + } + RowType headerType = (RowType) arrayType.getElementType(); + if (!headerType.getFieldNames().equals(Arrays.asList("name", "value")) + || !(headerType.getTypeAt(0) instanceof StringType) + || !(headerType.getTypeAt(1) instanceof BytesType) + || !headerType.getTypeAt(1).isNullable()) { + throw invalid("Kafka headers elements must be ROW."); + } + } + + private static void validateSingleFieldFormat( + KafkaDataFormat format, KafkaFieldProjection projection, String component) { + if (format == null) { + return; + } + if (format == KafkaDataFormat.RAW || format == KafkaDataFormat.STRING) { + if (projection.size() != 1) { + throw invalid( + "Kafka " + + component + + " format " + + format.value() + + " requires exactly one Fluss field."); + } + boolean validType = + format == KafkaDataFormat.RAW + ? projection.dataTypeAt(0) instanceof BytesType + : projection.dataTypeAt(0) instanceof StringType; + if (!validType) { + throw invalid( + "Kafka " + + component + + " field '" + + projection.nameAt(0) + + "' must be " + + (format == KafkaDataFormat.RAW ? "BYTES" : "STRING") + + " for format " + + format.value() + + "."); + } + } + } + + private static KafkaDataFormat parseFormat(String value) { + try { + return KafkaDataFormat.parse(value); + } catch (IllegalArgumentException e) { + throw invalid(e.getMessage()); + } + } + + private static KafkaTopicSchemaException invalid(String message) { + return new KafkaTopicSchemaException(message); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java new file mode 100644 index 00000000000..20c32ded941 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link KafkaTopicMapper}. */ +public class KafkaTopicMapperTest { + + @Test + public void testTopicNameAndIdMapping() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + + assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + Uuid topicId = mapper.toTopicId(123L); + assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); + assertThat(mapper.isMappedTopicId(topicId)).isTrue(); + assertThat(mapper.toTableId(topicId)).isEqualTo(123L); + + Uuid firstTableTopicId = mapper.toTopicId(0L); + assertThat(firstTableTopicId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(mapper.isMappedTopicId(firstTableTopicId)).isTrue(); + assertThat(mapper.toTableId(firstTableTopicId)).isZero(); + } + + @Test + public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue(); + assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse(); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath("invalid topic")) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); + assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java new file mode 100644 index 00000000000..0c92b8a7300 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies Kafka mapping properties survive the native Fluss table creation path. */ +public class KafkaTableMappingITCase { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder().setNumOfTabletServers(1).build(); + + @Test + public void testResolveMappingFromCreatedTableMetadata() throws Exception { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("event_key", DataTypes.STRING()) + .column("event_body", DataTypes.BYTES()) + .build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .build(); + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl"); + TablePath tablePath = mapper.toTablePath("events"); + long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor); + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + MetadataRequest request = new MetadataRequest(); + request.addTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()); + PbTableMetadata metadata = + FLUSS_CLUSTER_EXTENSION + .newCoordinatorClient() + .metadata(request) + .get() + .getTableMetadatasList() + .get(0); + TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); + + assertThat(metadata.getTableId()).isEqualTo(tableId); + assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId); + assertThat(metadata.getBucketMetadatasList()).hasSize(2); + assertThat(persisted.getCustomProperties()) + .containsAllEntriesOf(descriptor.getCustomProperties()); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java new file mode 100644 index 00000000000..4c698b5ad6b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the DDL contract independently of Kafka request handling and record decoding. */ +public class KafkaTopicSchemaResolverTest { + + private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); + + @Test + public void testRawMappingSurvivesTableMetadataSerialization() { + Schema schema = + Schema.newBuilder() + .column("message", DataTypes.BYTES()) + .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("attributes", headersType()) + .column("message_key", DataTypes.BYTES()) + .build(); + TableDescriptor table = + table(schema, "raw") + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .build(); + KafkaTopicSchema mapping = + resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes())); + + assertThat(mapping.rowType()).isEqualTo(schema.getRowType()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(3); + assertThat(mapping.keyProjection().nameAt(0)).isEqualTo("message_key"); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0)).isEqualTo(DataTypes.BYTES()); + assertThat(mapping.timestampPosition()).isEqualTo(1); + assertThat(mapping.headersPosition()).isEqualTo(2); + assertThat(mapping).isEqualTo(resolver.resolve(table)); + assertThatThrownBy(() -> mapping.keyProjection().positions().add(1)) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void testValueOnlyStringAndNullableColumns() { + for (boolean nullable : new boolean[] {true, false}) { + KafkaTopicSchema mapping = + resolver.resolve( + table( + Schema.newBuilder() + .column( + "body", + DataTypes.STRING().copy(nullable)) + .build(), + " STRING ") + .build()); + assertThat(mapping.keyFormat()).isNull(); + assertThat(mapping.keyProjection().isEmpty()).isTrue(); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0).isNullable()).isEqualTo(nullable); + assertThat(mapping.timestampPosition()).isEqualTo(-1); + assertThat(mapping.headersPosition()).isEqualTo(-1); + } + } + + @Test + public void testMixedFormatsAndMetadataAreOptional() { + KafkaTopicSchema mapping = + resolver.resolve( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, " except_key ") + .build()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + } + + @Test + public void testRejectsUnsupportedTableKinds() { + Schema primaryKeySchema = + Schema.newBuilder() + .column("id", DataTypes.STRING().copy(false)) + .primaryKey("id") + .build(); + assertInvalid(table(primaryKeySchema, "string"), "must be a log table"); + assertInvalid(keyValueTable().partitionedBy("id"), "Partitioned Fluss tables"); + assertInvalid(valueTable().logFormat(LogFormat.INDEXED), "Arrow log format"); + } + + @Test + public void testRequiresExplicitValueFormat() { + assertInvalid( + TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(1) + .customProperty("fluss.value.format", "raw"), + KafkaDataFormat.VALUE_FORMAT_CONFIG); + } + + @ParameterizedTest + @ValueSource(strings = {"json", "avro", ""}) + public void testRejectsUnavailableFormats(String format) { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + } + + @Test + public void testRejectsUnsupportedKafkaOptions() { + assertInvalid( + valueTable().customProperty("kafka.value.rescue-column", "body"), + "Unsupported Kafka table property"); + assertInvalid( + valueTable().customProperty("kafka.key.field", "body"), + "Unsupported Kafka table property"); + assertThat(resolver.resolve(valueTable().customProperty("owner", "team").build())) + .isNotNull(); + } + + @Test + public void testKeyFormatAndFieldMustBeSpecifiedTogether() { + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires kafka.key.format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"), + "kafka.key.fields"); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "missing", "id,id", "id,", ",id", "id,,body"}) + public void testRejectsInvalidKeyFields(String fields) { + assertThatThrownBy( + () -> + resolver.resolve( + keyValueTable() + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty( + KafkaDataFormat.KEY_FIELDS_CONFIG, fields) + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + "EXCEPT_KEY") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class); + } + + @Test + public void testRejectsAmbiguousAndEmptyValueProjections() { + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "bad"), + "Expected ALL or EXCEPT_KEY"); + assertInvalid( + valueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + assertInvalid(keyValueTable(), "exactly one Fluss field"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id,body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + } + + @Test + public void testRejectsWrongPhysicalTypes() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"), + "must be STRING"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "must be BYTES"); + } + + @Test + public void testRejectsWrongTimestampTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + DataTypes.TIMESTAMP_LTZ(3), + DataTypes.TIMESTAMP_LTZ(6).copy(false) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), + "TIMESTAMP_LTZ(3) NOT NULL"); + } + } + + @Test + public void testRejectsWrongHeaderTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + headersType().copy(false), + DataTypes.ARRAY(DataTypes.STRING()), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("key", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES().copy(false)))) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("attributes", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes"), + "headers"); + } + } + + @Test + public void testRejectsConflictingOrMissingMetadataColumns() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "missing"), + "does not exist"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, " "), + "must not be empty"); + TableDescriptor.Builder table = + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "ts"), + "same Fluss column"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "ts"), + "metadata column"); + } + + private void assertInvalid(TableDescriptor.Builder table, String message) { + assertThatThrownBy(() -> resolver.resolve(table.build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining(message); + } + + private static TableDescriptor.Builder keyValueTable() { + return table( + Schema.newBuilder() + .column("id", DataTypes.STRING()) + .column("body", DataTypes.BYTES()) + .build(), + "raw"); + } + + private static TableDescriptor.Builder valueTable() { + return table(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "raw"); + } + + private static TableDescriptor.Builder table(Schema schema, String valueFormat) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat); + } + + private static DataType headersType() { + return DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))); + } +} From 4361fa11b227f6d5573757b21499f692a289cc12 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 11:39:47 +0800 Subject: [PATCH 05/11] [kafka] Route qualified topics and streamline field projections Resolve database.table topic names directly to Fluss table paths and return qualified names for metadata. Validate name boundaries and cover same-named tables in different databases through native metadata round trips. Read projected field names directly from DataField to avoid rebuilding the complete field-name list for every projected column. Validated on Java 11: 67 Kafka unit tests and 2 integration tests passed, along with Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 52/52 AI-Contributed/UT: 167/167 --- .../fluss/kafka/mapping/KafkaTopicMapper.java | 50 ++++++--- .../kafka/schema/KafkaFieldProjection.java | 2 +- .../kafka/mapping/KafkaTopicMapperTest.java | 101 ++++++++++++++++-- .../kafka/schema/KafkaTableMappingITCase.java | 66 ++++++++---- 4 files changed, 172 insertions(+), 47 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index 9f8ed63f543..f8b5744f71d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -21,35 +21,57 @@ import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.internals.Topic; import static org.apache.fluss.utils.Preconditions.checkArgument; -import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +/** Maps fully qualified Kafka topic identities to Fluss tables. */ @Internal public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; - private final String databaseName; - - /** Creates a topic mapper for one Fluss database. */ - public KafkaTopicMapper(String databaseName) { - this.databaseName = checkNotNull(databaseName); + /** Maps a Kafka topic in database.table form to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + if (!isValidTopic(topicName)) { + throw new InvalidTopicException( + "Kafka topic must be a valid database.table name: " + topicName); + } + int separator = topicName.indexOf('.'); + return TablePath.of(topicName.substring(0, separator), topicName.substring(separator + 1)); } - /** Maps a Kafka topic name to its Fluss table path. */ - public TablePath toTablePath(String topicName) { - Topic.validate(topicName); - return TablePath.of(databaseName, topicName); + /** Returns the fully qualified Kafka name of a representable Fluss table. */ + public String toTopicName(TablePath tablePath) { + if (!isMappedTable(tablePath)) { + throw new InvalidTopicException( + "Fluss table cannot be represented as a Kafka database.table name: " + + tablePath); + } + return tablePath.toString(); } - /** Returns whether a table belongs to this database and has a valid Kafka topic name. */ + /** Returns whether a Fluss user table has a valid, fully qualified Kafka topic name. */ public boolean isMappedTable(TablePath tablePath) { - return databaseName.equals(tablePath.getDatabaseName()) - && Topic.isValid(tablePath.getTableName()); + return tablePath != null && tablePath.isValid() && isValidTopic(tablePath.toString()); + } + + /** Returns whether a name uniquely represents a user table in a Fluss database. */ + public static boolean isValidTopic(String topicName) { + if (topicName == null || !Topic.isValid(topicName)) { + return false; + } + int separator = topicName.indexOf('.'); + if (separator <= 0 || separator != topicName.lastIndexOf('.')) { + return false; + } + String database = topicName.substring(0, separator); + String table = topicName.substring(separator + 1); + return TablePath.of(database, table).isValid() + && TablePath.validatePrefix(database) == null + && TablePath.validatePrefix(table) == null; } /** Maps a Fluss table ID to a stable Kafka topic ID. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java index b1715758a9d..b6833475a20 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java @@ -50,7 +50,7 @@ public KafkaFieldProjection(RowType rowType, List positions) { "Invalid Kafka field projection position %s.", position); positionCopy.add(position); - projectedNames.add(rowType.getFieldNames().get(position)); + projectedNames.add(rowType.getFields().get(position).getName()); projectedTypes.add(rowType.getTypeAt(position)); } this.positions = Collections.unmodifiableList(positionCopy); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index 20c32ded941..c5d4a582a88 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -22,6 +22,12 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InvalidTopicException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -29,11 +35,88 @@ /** Tests for {@link KafkaTopicMapper}. */ public class KafkaTopicMapperTest { + private final KafkaTopicMapper mapper = new KafkaTopicMapper(); + + @Test + public void testQualifiedNamesAcrossDatabases() { + for (String database : Arrays.asList("sales", "archive", "sales_1-2026")) { + TablePath tablePath = TablePath.of(database, "orders_1-2026"); + String topicName = database + ".orders_1-2026"; + assertThat(mapper.toTablePath(topicName)).isEqualTo(tablePath); + assertThat(mapper.toTopicName(tablePath)).isEqualTo(topicName); + assertThat(mapper.isMappedTable(tablePath)).isTrue(); + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isTrue(); + } + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource( + strings = { + "orders", + ".orders", + "sales.", + "sales.orders.extra", + "sales..orders", + "sales.bad name", + " sales.orders", + "sales.orders ", + "sales/orders", + "__system.orders", + "sales.__internal", + "数据库.orders", + "sales.订单" + }) + public void testRejectsInvalidTopicNames(String topicName) { + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(topicName)) + .isInstanceOf(InvalidTopicException.class); + } + @Test - public void testTopicNameAndIdMapping() { - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + public void testNameLengthLimits() { + for (TablePath valid : + Arrays.asList( + TablePath.of(repeat('a', 200), "orders"), + TablePath.of("sales", repeat('a', 200)), + TablePath.of(repeat('a', 124), repeat('b', 124)))) { + assertThat(mapper.toTablePath(mapper.toTopicName(valid))).isEqualTo(valid); + } + for (TablePath invalid : + Arrays.asList( + TablePath.of(repeat('a', 201), "orders"), + TablePath.of("sales", repeat('a', 201)), + TablePath.of(repeat('a', 124), repeat('b', 125)))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(invalid.toString())) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } + + @Test + public void testRejectsUnrepresentableTablePaths() { + for (TablePath invalid : + Arrays.asList( + null, + TablePath.of(null, "orders"), + TablePath.of("sales", null), + TablePath.of("", "orders"), + TablePath.of("sales", ""), + TablePath.of("sales.region", "orders"), + TablePath.of("sales", "orders.v1"), + TablePath.of("__system", "orders"), + TablePath.of("sales", "__internal"), + TablePath.of("sales", "bad name"))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } - assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + @Test + public void testTopicIdMapping() { Uuid topicId = mapper.toTopicId(123L); assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); assertThat(mapper.isMappedTopicId(topicId)).isTrue(); @@ -46,17 +129,15 @@ public void testTopicNameAndIdMapping() { } @Test - public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() { - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); - assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue(); - assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse(); - assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse(); - assertThatThrownBy(() -> mapper.toTablePath("invalid topic")) - .isInstanceOf(InvalidTopicException.class); + public void testRejectsInvalidTopicIds() { assertThatThrownBy(() -> mapper.toTopicId(-1L)) .isInstanceOf(IllegalArgumentException.class); assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) .isInstanceOf(IllegalArgumentException.class); } + + private static String repeat(char character, int count) { + return String.join("", Collections.nCopies(count, String.valueOf(character))); + } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 0c92b8a7300..2ded7bd17f6 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -31,6 +31,11 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; import static org.assertj.core.api.Assertions.assertThat; @@ -42,7 +47,7 @@ public class KafkaTableMappingITCase { FlussClusterExtension.builder().setNumOfTabletServers(1).build(); @Test - public void testResolveMappingFromCreatedTableMetadata() throws Exception { + public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Exception { TableDescriptor descriptor = TableDescriptor.builder() .schema( @@ -57,32 +62,49 @@ public void testResolveMappingFromCreatedTableMetadata() throws Exception { .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl"); - TablePath tablePath = mapper.toTablePath("events"); - long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor); - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + KafkaTopicMapper mapper = new KafkaTopicMapper(); + Map tableIds = new LinkedHashMap<>(); MetadataRequest request = new MetadataRequest(); - request.addTablePath() - .setDatabaseName(tablePath.getDatabaseName()) - .setTableName(tablePath.getTableName()); - PbTableMetadata metadata = + for (String database : Arrays.asList("kafka_ddl", "kafka_ddl_archive")) { + TablePath tablePath = mapper.toTablePath(database + ".events"); + assertThat(tablePath).isEqualTo(TablePath.of(database, "events")); + tableIds.put(tablePath, createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor)); + request.addTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()); + } + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + List metadatas = FLUSS_CLUSTER_EXTENSION .newCoordinatorClient() .metadata(request) .get() - .getTableMetadatasList() - .get(0); - TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); + .getTableMetadatasList(); + assertThat(metadatas).hasSize(2); + assertThat(metadatas) + .extracting(PbTableMetadata::getTableId) + .containsExactlyInAnyOrderElementsOf(tableIds.values()) + .doesNotHaveDuplicates(); + for (PbTableMetadata metadata : metadatas) { + TablePath tablePath = + TablePath.of( + metadata.getTablePath().getDatabaseName(), + metadata.getTablePath().getTableName()); + TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); - assertThat(metadata.getTableId()).isEqualTo(tableId); - assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId); - assertThat(metadata.getBucketMetadatasList()).hasSize(2); - assertThat(persisted.getCustomProperties()) - .containsAllEntriesOf(descriptor.getCustomProperties()); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(1); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(metadata.getTableId()).isEqualTo(tableIds.get(tablePath)); + assertThat(mapper.toTopicName(tablePath)) + .isIn("kafka_ddl.events", "kafka_ddl_archive.events"); + assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))) + .isEqualTo(tableIds.get(tablePath)); + assertThat(metadata.getBucketMetadatasList()).hasSize(2); + assertThat(persisted.getCustomProperties()) + .containsAllEntriesOf(descriptor.getCustomProperties()); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + } } } From ce3759e87d331c928c54d7df45e8739fbe71bdf2 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 15:33:27 +0800 Subject: [PATCH 06/11] [kafka] Use the DDL table contract in Metadata Integrate the DDL mapping prerequisite and use its resolver for Metadata discovery. Omit unsupported tables from all-topic queries and return per-topic mapping errors for named queries. Preserve metadata for compatible tables in mixed requests. Validation: Java 11, mvn -o -pl fluss-kafka clean verify (62 unit tests and 2 integration tests); Checkstyle, Spotless and RAT passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 742/742 AI-Contributed/UT: 571/571 --- .../fluss/kafka/KafkaProtocolPlugin.java | 3 +- .../fluss/kafka/KafkaRequestHandler.java | 17 +- .../kafka/api/metadata/MetadataHandler.java | 190 ++++++ .../metadata/GatewayKafkaMetadataBackend.java | 322 ++++++++++ .../metadata/KafkaClusterMetadata.java | 210 ++++++ .../metadata/KafkaMetadataBackend.java | 30 + .../backend/metadata/KafkaMetadataQuery.java | 102 +++ .../fluss/kafka/KafkaMetadataHandlerTest.java | 602 ++++++++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 5 +- 9 files changed, 1477 insertions(+), 4 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 939c35a2d95..59ce1b2a0ed 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -66,6 +66,7 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " + service.getClass().getSimpleName()); } - return new KafkaRequestHandler(); + TabletServerGateway gateway = (TabletServerGateway) service; + return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 2df16a8bd7f..de561509b66 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,22 +17,35 @@ package org.apache.fluss.kafka; +import org.apache.fluss.kafka.api.metadata.MetadataHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { private final KafkaRequestDispatcher dispatcher; - /** Creates a Kafka request handler with the implemented server capabilities. */ - public KafkaRequestHandler() { + /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ + public KafkaRequestHandler( + RpcGatewayService service, TabletServerGateway gateway, String kafkaDatabase) { + checkNotNull(service); + checkNotNull(gateway); + checkNotNull(kafkaDatabase); KafkaApiRegistry registry = new KafkaApiRegistry(); registry.register(new ApiVersionsHandler(registry)); + registry.register( + new MetadataHandler( + new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase))); registry.freeze(); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java new file mode 100644 index 00000000000..ee8bfe7a5a8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataBackend; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements Kafka Metadata versions 0 through 11 using a narrow Fluss metadata backend. */ +@Internal +public final class MetadataHandler implements KafkaApiHandler { + + private static final short MAX_SUPPORTED_VERSION = 11; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.METADATA, + ApiKeys.METADATA.oldestVersion(), + (short) Math.min(ApiKeys.METADATA.latestVersion(), MAX_SUPPORTED_VERSION), + true); + + private final KafkaMetadataBackend backend; + + /** Creates a Metadata handler. */ + public MetadataHandler(KafkaMetadataBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, MetadataRequest request) { + List validTopics = new ArrayList<>(); + List invalidTopics = new ArrayList<>(); + if (!request.isAllTopics()) { + for (MetadataRequestTopic topic : request.data().topics()) { + if (topic.name() == null) { + throw new InvalidRequestException( + "Topic name must be set because topic ID lookup is not supported by " + + "Metadata versions 10 and 11."); + } else if (!Topic.isValid(topic.name())) { + invalidTopics.add( + new KafkaClusterMetadata.Topic( + topic.name(), + topic.topicId(), + TopicError.INVALID_TOPIC, + Collections.emptyList())); + } else { + // Kafka added the topic ID fields in v10, but ID-based Metadata lookup was not + // implemented until v12. This handler intentionally stops at v11. + validTopics.add(new TopicReference(topic.name(), Uuid.ZERO_UUID)); + } + } + } + + KafkaMetadataQuery query = + new KafkaMetadataQuery( + request.isAllTopics(), + validTopics, + context.listenerName(), + clientAddress(context.remoteAddress())); + return backend.getMetadata(query) + .thenApply( + metadata -> { + List topics = + new ArrayList<>(metadata.topics()); + topics.addAll(invalidTopics); + return toResponse( + request.version(), + new KafkaClusterMetadata(metadata.brokers(), topics)); + }); + } + + private static MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { + MetadataResponseData data = + new MetadataResponseData() + .setThrottleTimeMs(0) + .setControllerId(MetadataResponse.NO_CONTROLLER_ID) + .setClusterAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Broker broker : metadata.brokers()) { + MetadataResponseData.MetadataResponseBroker responseBroker = + new MetadataResponseData.MetadataResponseBroker() + .setNodeId(broker.id()) + .setHost(broker.host()) + .setPort(broker.port()); + if (broker.rack() != null) { + responseBroker.setRack(broker.rack()); + } + data.brokers().add(responseBroker); + } + for (KafkaClusterMetadata.Topic topic : metadata.topics()) { + MetadataResponseData.MetadataResponseTopic responseTopic = + new MetadataResponseData.MetadataResponseTopic() + .setName(topic.name()) + .setTopicId(topic.topicId()) + .setErrorCode(toKafkaError(topic.error()).code()) + .setIsInternal(topic.name() != null && Topic.isInternal(topic.name())) + .setTopicAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Partition partition : topic.partitions()) { + responseTopic + .partitions() + .add( + new MetadataResponseData.MetadataResponsePartition() + .setErrorCode( + partition.leaderAvailable() + ? Errors.NONE.code() + : Errors.LEADER_NOT_AVAILABLE.code()) + .setPartitionIndex(partition.partitionId()) + .setLeaderId(partition.leaderId()) + .setLeaderEpoch(partition.leaderEpoch()) + .setReplicaNodes(partition.replicas()) + .setIsrNodes(partition.isr()) + .setOfflineReplicas(partition.offlineReplicas())); + } + data.topics().add(responseTopic); + } + return new MetadataResponse(data, version); + } + + private static Errors toKafkaError(TopicError error) { + switch (error) { + case NONE: + return Errors.NONE; + case UNKNOWN_TOPIC_OR_PARTITION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case UNKNOWN_TOPIC_ID: + return Errors.UNKNOWN_TOPIC_ID; + case INVALID_TOPIC: + return Errors.INVALID_TOPIC_EXCEPTION; + default: + throw new IllegalArgumentException("Unsupported metadata error " + error); + } + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java new file mode 100644 index 00000000000..5905624654f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.MetadataResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.Uuid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */ +@Internal +public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { + + private static final Logger LOG = LoggerFactory.getLogger(GatewayKafkaMetadataBackend.class); + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final String databaseName; + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + + /** Creates a metadata backend backed by the local TabletServer gateway. */ + public GatewayKafkaMetadataBackend( + RpcGatewayService service, TabletServerGateway gateway, String databaseName) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.databaseName = checkNotNull(databaseName); + this.topicMapper = new KafkaTopicMapper(databaseName); + } + + @Override + public CompletableFuture getMetadata(KafkaMetadataQuery query) { + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenCompose( + response -> + requestFlussMetadata( + query, + new LinkedHashSet<>(response.getTableNamesList()))); + } + + Set topicNames = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + topicNames.add(topic.topicName()); + } + } + return requestFlussMetadata(query, topicNames); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames) { + return requestFlussMetadata(query, topicNames, true); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { + MetadataRequest request = new MetadataRequest(); + for (String topicName : topicNames) { + TablePath tablePath = TablePath.of(databaseName, topicName); + if (!topicMapper.isMappedTable(tablePath)) { + continue; + } + tablePath = topicMapper.toTablePath(topicName); + request.addAllTablePaths( + Collections.singletonList( + new PbTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()))); + } + setCurrentSession(query); + try { + return gateway.metadata(request) + .handle( + (response, failure) -> + failure == null + ? CompletableFuture.completedFuture( + toKafkaMetadata(query, response)) + : recoverMetadataFailure( + query, failure, refreshAndRetry)) + .thenCompose(future -> future); + } catch (Throwable failure) { + return recoverMetadataFailure(query, failure, refreshAndRetry); + } + } + + private CompletableFuture recoverMetadataFailure( + KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) { + if (refreshAndRetry) { + return currentTopicNames(query) + .thenCompose(currentNames -> requestFlussMetadata(query, currentNames, false)); + } + LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure)); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(unwrap(failure)); + return failed; + } + + private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenApply( + response -> { + Set currentNames = + new LinkedHashSet<>(response.getTableNamesList()); + if (!query.allTopics() && !containsTopicId(query.topics())) { + Set requestedNames = new HashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + requestedNames.add(topic.topicName()); + } + } + currentNames.retainAll(requestedNames); + } + return currentNames; + }); + } + + private KafkaClusterMetadata toKafkaMetadata( + KafkaMetadataQuery query, MetadataResponse response) { + List brokers = new ArrayList<>(); + Set aliveBrokerIds = new HashSet<>(); + for (PbServerNode server : response.getTabletServersList()) { + brokers.add( + new Broker( + server.getNodeId(), + server.getHost(), + server.getPort(), + server.hasRack() ? server.getRack() : null)); + aliveBrokerIds.add(server.getNodeId()); + } + Collections.sort(brokers, Comparator.comparingInt(Broker::id)); + + Map topicsByName = new HashMap<>(); + Map topicsById = new HashMap<>(); + for (PbTableMetadata table : response.getTableMetadatasList()) { + TablePath tablePath = + TablePath.of( + table.getTablePath().getDatabaseName(), + table.getTablePath().getTableName()); + if (!topicMapper.isMappedTable(tablePath)) { + continue; + } + Topic topic = toKafkaTopic(table, aliveBrokerIds); + topicsByName.put(topic.name(), topic); + topicsById.put(topic.topicId(), topic); + } + + List topics = new ArrayList<>(); + if (query.allTopics()) { + for (Topic topic : topicsByName.values()) { + if (topic.error() == TopicError.NONE) { + topics.add(topic); + } + } + Collections.sort(topics, Comparator.comparing(Topic::name)); + } else { + for (TopicReference reference : query.topics()) { + Topic topic = + reference.hasTopicId() + ? topicsById.get(reference.topicId()) + : topicsByName.get(reference.topicName()); + if (topic != null && matches(reference, topic)) { + topics.add(topic); + } else { + topics.add(missingTopic(reference)); + } + } + } + return new KafkaClusterMetadata(brokers, topics); + } + + private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { + try { + schemaResolver.resolve(TableDescriptor.fromJsonBytes(table.getTableJson())); + } catch (IllegalArgumentException e) { + LOG.debug( + "Table {} does not define a supported Kafka mapping: {}", + table.getTablePath().getTableName(), + e.getMessage()); + return new Topic( + table.getTablePath().getTableName(), + topicMapper.toTopicId(table.getTableId()), + TopicError.INVALID_TOPIC, + Collections.emptyList()); + } + List partitions = new ArrayList<>(); + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + boolean leaderAvailable = + bucket.hasLeaderId() && aliveBrokerIds.contains(bucket.getLeaderId()); + List replicas = new ArrayList<>(); + List isr = new ArrayList<>(); + List offlineReplicas = new ArrayList<>(); + Set isrIds = new HashSet<>(); + if (bucket.hasBucketEpoch()) { + for (int isrId : bucket.getIsrs()) { + isrIds.add(isrId); + } + } else if (leaderAvailable) { + // Legacy metadata has no authoritative ISR. Report only the available leader + // as a conservative fallback; live followers may still be out of sync. + isrIds.add(bucket.getLeaderId()); + } + for (int replicaId : bucket.getReplicaIds()) { + replicas.add(replicaId); + if (isrIds.contains(replicaId)) { + isr.add(replicaId); + } + if (!aliveBrokerIds.contains(replicaId)) { + offlineReplicas.add(replicaId); + } + } + partitions.add( + new Partition( + bucket.getBucketId(), + leaderAvailable ? bucket.getLeaderId() : -1, + bucket.hasLeaderEpoch() ? bucket.getLeaderEpoch() : -1, + replicas, + isr, + offlineReplicas, + leaderAvailable)); + } + Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); + return new Topic( + table.getTablePath().getTableName(), + topicMapper.toTopicId(table.getTableId()), + TopicError.NONE, + partitions); + } + + private static Topic missingTopic(TopicReference reference) { + TopicError error = + reference.hasTopicId() + ? TopicError.UNKNOWN_TOPIC_ID + : TopicError.UNKNOWN_TOPIC_OR_PARTITION; + return new Topic( + reference.topicName(), reference.topicId(), error, Collections.emptyList()); + } + + private static boolean matches(TopicReference reference, Topic topic) { + return (reference.topicName() == null || reference.topicName().equals(topic.name())) + && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId())); + } + + private void setCurrentSession(KafkaMetadataQuery query) { + service.setCurrentSession( + new Session( + (short) 0, + query.listenerName(), + false, + query.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static boolean containsTopicId(List topics) { + for (TopicReference topic : topics) { + if (topic.hasTopicId()) { + return true; + } + } + return false; + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java new file mode 100644 index 00000000000..bbc634af6f0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Kafka-domain cluster metadata returned by a Fluss metadata backend. */ +@Internal +public final class KafkaClusterMetadata { + + private final List brokers; + private final List topics; + + /** Creates cluster metadata. */ + public KafkaClusterMetadata(List brokers, List topics) { + this.brokers = immutableCopy(brokers); + this.topics = immutableCopy(topics); + } + + /** Returns Kafka-reachable brokers. */ + public List brokers() { + return brokers; + } + + /** Returns topic metadata and topic-level errors. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Kafka-reachable broker information. */ + @Internal + public static final class Broker { + + private final int id; + private final String host; + private final int port; + private final @Nullable String rack; + + /** Creates broker information. */ + public Broker(int id, String host, int port, @Nullable String rack) { + this.id = id; + this.host = checkNotNull(host); + this.port = port; + this.rack = rack; + } + + /** Returns the Kafka broker ID. */ + public int id() { + return id; + } + + /** Returns the Kafka listener host. */ + public String host() { + return host; + } + + /** Returns the Kafka listener port. */ + public int port() { + return port; + } + + /** Returns the broker rack, if configured. */ + public @Nullable String rack() { + return rack; + } + } + + /** Topic-level error independent of a Kafka response schema version. */ + @Internal + public enum TopicError { + NONE, + UNKNOWN_TOPIC_OR_PARTITION, + UNKNOWN_TOPIC_ID, + INVALID_TOPIC + } + + /** Metadata for one Kafka topic. */ + @Internal + public static final class Topic { + + private final @Nullable String name; + private final Uuid topicId; + private final TopicError error; + private final List partitions; + + /** Creates topic metadata. */ + public Topic( + @Nullable String name, Uuid topicId, TopicError error, List partitions) { + this.name = name; + this.topicId = checkNotNull(topicId); + this.error = checkNotNull(error); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name, if known. */ + public @Nullable String name() { + return name; + } + + /** Returns the stable Kafka topic ID. */ + public Uuid topicId() { + return topicId; + } + + /** Returns the topic-level domain error. */ + public TopicError error() { + return error; + } + + /** Returns the topic partitions. */ + public List partitions() { + return partitions; + } + } + + /** Metadata for one Kafka partition backed by a Fluss bucket. */ + @Internal + public static final class Partition { + + private final int partitionId; + private final int leaderId; + private final int leaderEpoch; + private final List replicas; + private final List isr; + private final List offlineReplicas; + private final boolean leaderAvailable; + + /** Creates partition metadata. */ + public Partition( + int partitionId, + int leaderId, + int leaderEpoch, + List replicas, + List isr, + List offlineReplicas, + boolean leaderAvailable) { + this.partitionId = partitionId; + this.leaderId = leaderId; + this.leaderEpoch = leaderEpoch; + this.replicas = immutableCopy(replicas); + this.isr = immutableCopy(isr); + this.offlineReplicas = immutableCopy(offlineReplicas); + this.leaderAvailable = leaderAvailable; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the current leader ID, or {@code -1} when unavailable. */ + public int leaderId() { + return leaderId; + } + + /** Returns the leader epoch, or {@code -1} when unavailable. */ + public int leaderEpoch() { + return leaderEpoch; + } + + /** Returns assigned replica IDs. */ + public List replicas() { + return replicas; + } + + /** Returns replica IDs currently visible as in-sync. */ + public List isr() { + return isr; + } + + /** Returns assigned replicas whose TabletServers are unavailable. */ + public List offlineReplicas() { + return offlineReplicas; + } + + /** Returns whether the partition has a reachable leader. */ + public boolean leaderAvailable() { + return leaderAvailable; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java new file mode 100644 index 00000000000..abb02a920ed --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Metadata API. */ +@Internal +public interface KafkaMetadataBackend { + + /** Resolves Kafka-domain metadata asynchronously. */ + CompletableFuture getMetadata(KafkaMetadataQuery query); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java new file mode 100644 index 00000000000..01e21fa4440 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Domain query used by the Metadata API to access the Fluss adapter layer. */ +@Internal +public final class KafkaMetadataQuery { + + private final boolean allTopics; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a metadata query. */ + public KafkaMetadataQuery( + boolean allTopics, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.allTopics = allTopics; + this.topics = Collections.unmodifiableList(new ArrayList<>(checkNotNull(topics))); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns whether all Kafka topics should be returned. */ + public boolean allTopics() { + return allTopics; + } + + /** Returns the explicitly requested topic identities. */ + public List topics() { + return topics; + } + + /** Returns the Kafka listener used by the client connection. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client address when it is available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + /** Kafka topic name and ID supplied by a Metadata request. */ + @Internal + public static final class TopicReference { + + private final @Nullable String topicName; + private final Uuid topicId; + + /** Creates a topic reference. */ + public TopicReference(@Nullable String topicName, Uuid topicId) { + this.topicName = topicName; + this.topicId = checkNotNull(topicId); + } + + /** Returns the requested topic name, if present. */ + public @Nullable String topicName() { + return topicName; + } + + /** Returns the requested topic ID, or {@link Uuid#ZERO_UUID} when absent. */ + public Uuid topicId() { + return topicId; + } + + /** Returns whether this reference identifies a topic by ID. */ + public boolean hasTopicId() { + return !Uuid.ZERO_UUID.equals(topicId); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java new file mode 100644 index 00000000000..2614b2df3fd --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -0,0 +1,602 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol compatibility tests for the Kafka Metadata API. */ +public class KafkaMetadataHandlerTest { + + private static final Uuid TOPIC_ID = new Uuid(0x466c757373000000L, 123L); + + @Test + public void testNamedTopicForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Collections.singletonList("topic"))), + version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(100, new byte[] {1, 2, 3})); + request.data() + .topics() + .get(0) + .unknownTaggedFields() + .add(new RawTaggedField(101, new byte[] {4, 5, 6})); + } + MetadataResponse response = handle(service, request, version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.controller()).isNull(); + Node broker = response.brokers().iterator().next(); + assertThat(broker.host()).isEqualTo("broker-1"); + assertThat(broker.port()).isEqualTo(9092); + assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.partitionIndex()).isZero(); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).isEmpty(); + } + assertThat(service.lastListenerName).isEqualTo("KAFKA"); + } + + @Test + public void testAllTopicsForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = allTopicsRequest(version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(102, new byte[] {7, 8, 9})); + } + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("other", "topic"); + } + } + + @Test + public void testAllTopicsAndMissingAndInvalidTopic() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse allTopics = + handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); + assertThat(allTopics.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactlyInAnyOrder("other", "topic"); + + MetadataRequest requestedTopics = + new MetadataRequest.Builder(Arrays.asList("missing", "invalid topic"), false) + .build((short) 9); + MetadataResponse errors = handle(service, requestedTopics, (short) 9); + assertThat(errors.errors()) + .containsEntry("missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("invalid topic", Errors.INVALID_TOPIC_EXCEPTION); + } + + @Test + public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName("topic") + .setTopicId( + new Uuid( + 0x466c757373000000L, + 999L)))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(response.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + } + } + + @Test + public void testV10AndV11RejectTopicIdOnlyLookup() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Uuid requestedTopicId = new Uuid(0x466c757373000000L, 999L); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(null) + .setTopicId(requestedTopicId))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()).hasSize(1); + MetadataResponseTopic responseTopic = response.data().topics().iterator().next(); + assertThat(responseTopic.name()).isEmpty(); + assertThat(responseTopic.topicId()).isEqualTo(requestedTopicId); + assertThat(responseTopic.errorCode()).isEqualTo(Errors.INVALID_REQUEST.code()); + } + } + + @Test + public void testTopicIdentityAcrossDeleteAndRecreate() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse initial = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(initial.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + + service.removeTable("topic"); + MetadataResponse deleted = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(deleted.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + + service.putTable("topic", 223L); + Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); + MetadataResponse recreatedByName = + handle( + service, + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11), + (short) 11); + assertThat(recreatedByName.data().topics().find("topic").topicId()) + .isEqualTo(recreatedTopicId) + .isNotEqualTo(TOPIC_ID); + } + + @Test + public void testDeleteRaceBecomesUnknownTopicResult() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.removeTable("topic"); + service.failNextMetadataAsMissing = true; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + } + + @Test + public void testUnavailableLeaderUsesPartitionError() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicLeaderAvailable = false; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.leaderId()).isEqualTo(-1); + assertThat(partition.replicaNodes()).containsExactly(1, 2, 3); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).containsExactly(3); + } + + @Test + public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicIsr = new int[] {1}; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataUsesLeaderOnlyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + service.topicLeaderAvailable = false; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.isrNodes()).isEmpty(); + } + + @Test + public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { + for (int bucketEpoch : new int[] {7, -1}) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = bucketEpoch; + service.topicIsr = new int[0]; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.isrNodes()).isEmpty(); + } + } + + @Test + public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failMetadata = true; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 1)); + assertThat(response.brokers()).isEmpty(); + } + + @Test + public void testMetadataUsesDdlContractForEveryVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable( + "no_mapping", + 125L, + TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(2) + .build()); + service.putTable( + "bad_format", + 126L, + TableDescriptor.builder(defaultDescriptor()) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") + .build()); + service.putTable( + "primary_key", + 127L, + TableDescriptor.builder(defaultDescriptor()) + .schema( + Schema.newBuilder() + .column("body", DataTypes.BYTES().copy(false)) + .primaryKey("body") + .build()) + .build()); + service.putTable( + "partitioned", + 128L, + TableDescriptor.builder(defaultDescriptor()).partitionedBy("body").build()); + service.putTable( + "indexed", + 129L, + TableDescriptor.builder(defaultDescriptor()).logFormat(LogFormat.INDEXED).build()); + service.putTable("invalid topic", 130L); + List invalidMappings = + Arrays.asList("no_mapping", "bad_format", "primary_key", "partitioned", "indexed"); + for (short version = 0; version <= 11; version++) { + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("other", "topic"); + List requested = new ArrayList<>(invalidMappings); + requested.add("topic"); + requested.add("missing"); + MetadataResponse named = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + requested)), + version), + version); + for (String name : invalidMappings) { + MetadataResponseTopic topic = named.data().topics().find(name); + assertThat(topic.errorCode()).isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + assertThat(topic.partitions()).isEmpty(); + } + assertThat(named.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(named.data().topics().find("missing").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + } + } + + @Test + public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable( + "topic", + 123L, + TableDescriptor.builder(defaultDescriptor()) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .build()); + MetadataResponse invalid = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(invalid.data().topics().find("topic").errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + service.putTable("topic", 123L); + MetadataResponse valid = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(valid.data().topics().find("topic").errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + } + + private static TableDescriptor defaultDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .build(); + } + + private static MetadataRequest namedTopicRequest(String topicName) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(topicName) + .setTopicId(Uuid.ZERO_UUID))), + (short) 11); + } + + private static MetadataRequest allTopicsRequest(short version) { + MetadataRequestData data = new MetadataRequestData(); + data.setTopics(version == 0 ? Collections.emptyList() : null); + return new MetadataRequest(data, version); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { + KafkaRequestHandler handler = new KafkaRequestHandler(service, service, "kafka"); + ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); + KafkaRequest request; + try { + request = + new KafkaRequest( + ApiKeys.METADATA, + version, + new RequestHeader(ApiKeys.METADATA, version, "client-id", 1), + requestBody, + "KAFKA", + requestBuffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + } finally { + // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest. + requestBuffer.release(); + } + handler.processRequest(request); + ByteBuf responseBuffer = request.responseBuffer(); + try { + return (MetadataResponse) + AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + assertThat(requestBuffer.refCnt()).isZero(); + } + } + + private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + + private final Map tables = new LinkedHashMap<>(); + private final Map descriptors = new LinkedHashMap<>(); + private String lastListenerName; + private boolean topicLeaderAvailable = true; + private int[] topicIsr = new int[] {1, 2}; + private Integer topicBucketEpoch = 7; + private boolean failMetadata; + private boolean failNextMetadataAsMissing; + + private TestingMetadataGatewayService() { + putTable("topic", 123L); + putTable("other", 124L); + } + + @Override + public CompletableFuture listTables(ListTablesRequest request) { + assertThat(request.getDatabaseName()).isEqualTo("kafka"); + return CompletableFuture.completedFuture( + new ListTablesResponse().addAllTableNames(new ArrayList<>(tables.keySet()))); + } + + @Override + public CompletableFuture metadata( + org.apache.fluss.rpc.messages.MetadataRequest request) { + lastListenerName = currentListenerName(); + if (failMetadata) { + CompletableFuture failure = + new CompletableFuture<>(); + failure.completeExceptionally(new IllegalStateException("metadata unavailable")); + return failure; + } + if (failNextMetadataAsMissing) { + failNextMetadataAsMissing = false; + throw new TableNotExistException("table was deleted"); + } + List topics = new ArrayList<>(); + for (PbTablePath tablePath : request.getTablePathsList()) { + Long tableId = tables.get(tablePath.getTableName()); + if (tableId != null) { + topics.add( + tableMetadata( + tablePath.getTableName(), + tableId, + !"topic".equals(tablePath.getTableName()) + || topicLeaderAvailable, + "topic".equals(tablePath.getTableName()) + ? topicIsr + : new int[] {1, 2}, + "topic".equals(tablePath.getTableName()) + ? topicBucketEpoch + : Integer.valueOf(7)) + .setTableJson( + descriptors + .get(tablePath.getTableName()) + .toJsonBytes())); + } + } + return CompletableFuture.completedFuture( + new org.apache.fluss.rpc.messages.MetadataResponse() + .addAllTabletServers( + Arrays.asList( + new PbServerNode() + .setNodeId(1) + .setHost("broker-1") + .setPort(9092) + .setRack("rack-a"), + new PbServerNode() + .setNodeId(2) + .setHost("broker-2") + .setPort(9093))) + .addAllTableMetadatas(topics)); + } + + private void putTable(String topic, long tableId) { + putTable(topic, tableId, defaultDescriptor()); + } + + private void putTable(String topic, long tableId, TableDescriptor descriptor) { + tables.put(topic, tableId); + descriptors.put(topic, descriptor); + } + + private void removeTable(String topic) { + tables.remove(topic); + descriptors.remove(topic); + } + + private static PbTableMetadata tableMetadata( + String topic, + long tableId, + boolean leaderAvailable, + int[] isr, + Integer bucketEpoch) { + PbTableMetadata table = + new PbTableMetadata() + .setTablePath( + new PbTablePath().setDatabaseName("kafka").setTableName(topic)) + .setTableId(tableId) + .addAllBucketMetadatas( + Arrays.asList( + new PbBucketMetadata() + .setBucketId(0) + .setLeaderId(leaderAvailable ? 1 : 3) + .setLeaderEpoch(5) + .setReplicaIds( + leaderAvailable + ? new int[] {1, 2} + : new int[] {1, 2, 3}) + .setIsrs( + bucketEpoch == null ? new int[0] : isr), + new PbBucketMetadata() + .setBucketId(1) + .setLeaderId(2) + .setLeaderEpoch(6) + .setReplicaIds(new int[] {1, 2}) + .setIsrs( + bucketEpoch == null + ? new int[0] + : new int[] {1, 2}))); + if (bucketEpoch != null) { + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + bucket.setBucketEpoch(bucketEpoch); + } + } + return table; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index e6b8e961128..c4476c69d59 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.kafka; +import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; @@ -113,6 +114,7 @@ private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.METADATA.id, ApiKeys.METADATA.oldestVersion(), (short) 11), tuple( ApiKeys.API_VERSIONS.id, ApiKeys.API_VERSIONS.oldestVersion(), @@ -208,6 +210,7 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + return new KafkaRequestHandler(service, service, "kafka"); } } From 988918e255c502c9b8030b00e68d0d0eed0e21c7 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 14:39:43 +0800 Subject: [PATCH 07/11] [kafka] Discover qualified topics and isolate missing databases Route Metadata queries through the database.table mapping from PR03 and enumerate compatible topics across databases. Preserve brokers and valid topics when a requested database is absent or disappears during discovery; propagate other gateway failures. Cover qualified names, same-named tables across databases, synchronous and asynchronous lookup failures, and Metadata v0-v11 over a real listener. Validated on Java 11: 90 unit tests and 4 integration tests passed, along with Checkstyle, Spotless, RAT, and git diff --check. Five additional interleaved asynchronous review probes passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 129/129 AI-Contributed/UT: 514/514 --- .../fluss/kafka/KafkaProtocolPlugin.java | 2 +- .../fluss/kafka/KafkaRequestHandler.java | 8 +- .../kafka/api/metadata/MetadataHandler.java | 3 +- .../metadata/GatewayKafkaMetadataBackend.java | 116 +++++-- .../fluss/kafka/KafkaMetadataHandlerTest.java | 312 ++++++++++++++---- .../fluss/kafka/KafkaMetadataITCase.java | 200 +++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 2 +- 7 files changed, 531 insertions(+), 112 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 59ce1b2a0ed..83eacd2c47c 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -67,6 +67,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { + service.getClass().getSimpleName()); } TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE)); + return new KafkaRequestHandler(service, gateway); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index de561509b66..151665929ff 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -36,16 +36,12 @@ public class KafkaRequestHandler implements RequestHandler { private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ - public KafkaRequestHandler( - RpcGatewayService service, TabletServerGateway gateway, String kafkaDatabase) { + public KafkaRequestHandler(RpcGatewayService service, TabletServerGateway gateway) { checkNotNull(service); checkNotNull(gateway); - checkNotNull(kafkaDatabase); KafkaApiRegistry registry = new KafkaApiRegistry(); registry.register(new ApiVersionsHandler(registry)); - registry.register( - new MetadataHandler( - new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase))); + registry.register(new MetadataHandler(new GatewayKafkaMetadataBackend(service, gateway))); registry.freeze(); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java index ee8bfe7a5a8..2ceca11fbe1 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -28,6 +28,7 @@ import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InvalidRequestException; @@ -85,7 +86,7 @@ public CompletableFuture handle( throw new InvalidRequestException( "Topic name must be set because topic ID lookup is not supported by " + "Metadata versions 10 and 11."); - } else if (!Topic.isValid(topic.name())) { + } else if (!KafkaTopicMapper.isValidTopic(topic.name())) { invalidTopics.add( new KafkaClusterMetadata.Topic( topic.name(), diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java index 5905624654f..202658b9f0f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -18,6 +18,7 @@ package org.apache.fluss.kafka.backend.metadata; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; @@ -29,7 +30,9 @@ import org.apache.fluss.metadata.TablePath; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; import org.apache.fluss.rpc.messages.MetadataRequest; import org.apache.fluss.rpc.messages.MetadataResponse; import org.apache.fluss.rpc.messages.PbBucketMetadata; @@ -38,6 +41,7 @@ import org.apache.fluss.rpc.messages.PbTablePath; import org.apache.fluss.rpc.netty.server.Session; import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.common.Uuid; import org.slf4j.Logger; @@ -65,29 +69,20 @@ public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { private final RpcGatewayService service; private final TabletServerGateway gateway; - private final String databaseName; - private final KafkaTopicMapper topicMapper; + private final KafkaTopicMapper topicMapper = new KafkaTopicMapper(); private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); /** Creates a metadata backend backed by the local TabletServer gateway. */ - public GatewayKafkaMetadataBackend( - RpcGatewayService service, TabletServerGateway gateway, String databaseName) { + public GatewayKafkaMetadataBackend(RpcGatewayService service, TabletServerGateway gateway) { this.service = checkNotNull(service); this.gateway = checkNotNull(gateway); - this.databaseName = checkNotNull(databaseName); - this.topicMapper = new KafkaTopicMapper(databaseName); } @Override public CompletableFuture getMetadata(KafkaMetadataQuery query) { if (query.allTopics() || containsTopicId(query.topics())) { - setCurrentSession(query); - return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) - .thenCompose( - response -> - requestFlussMetadata( - query, - new LinkedHashSet<>(response.getTableNamesList()))); + return currentTopicNames(query) + .thenCompose(topicNames -> requestFlussMetadata(query, topicNames)); } Set topicNames = new LinkedHashSet<>(); @@ -108,11 +103,7 @@ private CompletableFuture requestFlussMetadata( KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { MetadataRequest request = new MetadataRequest(); for (String topicName : topicNames) { - TablePath tablePath = TablePath.of(databaseName, topicName); - if (!topicMapper.isMappedTable(tablePath)) { - continue; - } - tablePath = topicMapper.toTablePath(topicName); + TablePath tablePath = topicMapper.toTablePath(topicName); request.addAllTablePaths( Collections.singletonList( new PbTablePath() @@ -148,12 +139,27 @@ private CompletableFuture recoverMetadataFailure( } private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { - setCurrentSession(query); - return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + CompletableFuture> databasesFuture; + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + databasesFuture = + gateway.listDatabases(new ListDatabasesRequest()) + .thenApply( + response -> + new LinkedHashSet<>(response.getDatabaseNamesList())); + } else { + Set databases = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + databases.add(topicMapper.toTablePath(topic.topicName()).getDatabaseName()); + } + } + databasesFuture = CompletableFuture.completedFuture(databases); + } + return databasesFuture + .thenCompose(databases -> listTopicNames(query, databases)) .thenApply( - response -> { - Set currentNames = - new LinkedHashSet<>(response.getTableNamesList()); + currentNames -> { if (!query.allTopics() && !containsTopicId(query.topics())) { Set requestedNames = new HashSet<>(); for (TopicReference topic : query.topics()) { @@ -167,6 +173,56 @@ private CompletableFuture> currentTopicNames(KafkaMetadataQuery quer }); } + private CompletableFuture> listTopicNames( + KafkaMetadataQuery query, Set databases) { + CompletableFuture> topicsFuture = + CompletableFuture.completedFuture(new LinkedHashSet<>()); + for (String database : databases) { + topicsFuture = + topicsFuture.thenCompose( + topicNames -> + listTables(query, database) + .thenApply( + response -> { + for (String tableName : + response.getTableNamesList()) { + TablePath tablePath = + TablePath.of( + database, tableName); + if (topicMapper.isMappedTable( + tablePath)) { + topicNames.add( + topicMapper.toTopicName( + tablePath)); + } + } + return topicNames; + })); + } + return topicsFuture; + } + + private CompletableFuture listTables( + KafkaMetadataQuery query, String database) { + setCurrentSession(query); + CompletableFuture tablesFuture; + try { + tablesFuture = gateway.listTables(new ListTablesRequest().setDatabaseName(database)); + } catch (Exception failure) { + tablesFuture = FutureUtils.completedExceptionally(failure); + } + return tablesFuture.exceptionally( + failure -> { + Throwable cause = unwrap(failure); + if (cause instanceof DatabaseNotExistException) { + // A database may be missing or deleted after listDatabases. Keep loading + // the remaining topics and brokers through the normal metadata request. + return new ListTablesResponse(); + } + throw new CompletionException(cause); + }); + } + private KafkaClusterMetadata toKafkaMetadata( KafkaMetadataQuery query, MetadataResponse response) { List brokers = new ArrayList<>(); @@ -192,7 +248,7 @@ private KafkaClusterMetadata toKafkaMetadata( if (!topicMapper.isMappedTable(tablePath)) { continue; } - Topic topic = toKafkaTopic(table, aliveBrokerIds); + Topic topic = toKafkaTopic(table, topicMapper.toTopicName(tablePath), aliveBrokerIds); topicsByName.put(topic.name(), topic); topicsById.put(topic.topicId(), topic); } @@ -221,16 +277,17 @@ private KafkaClusterMetadata toKafkaMetadata( return new KafkaClusterMetadata(brokers, topics); } - private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { + private Topic toKafkaTopic( + PbTableMetadata table, String topicName, Set aliveBrokerIds) { try { schemaResolver.resolve(TableDescriptor.fromJsonBytes(table.getTableJson())); } catch (IllegalArgumentException e) { LOG.debug( "Table {} does not define a supported Kafka mapping: {}", - table.getTablePath().getTableName(), + topicName, e.getMessage()); return new Topic( - table.getTablePath().getTableName(), + topicName, topicMapper.toTopicId(table.getTableId()), TopicError.INVALID_TOPIC, Collections.emptyList()); @@ -273,10 +330,7 @@ private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { } Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); return new Topic( - table.getTablePath().getTableName(), - topicMapper.toTopicId(table.getTableId()), - TopicError.NONE, - partitions); + topicName, topicMapper.toTopicId(table.getTableId()), TopicError.NONE, partitions); } private static Topic missingTopic(TopicReference reference) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java index 2614b2df3fd..4c33a51635a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -17,12 +17,15 @@ package org.apache.fluss.kafka; +import org.apache.fluss.exception.DatabaseNotExistException; import org.apache.fluss.exception.TableNotExistException; import org.apache.fluss.kafka.format.KafkaDataFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListDatabasesRequest; +import org.apache.fluss.rpc.messages.ListDatabasesResponse; import org.apache.fluss.rpc.messages.ListTablesRequest; import org.apache.fluss.rpc.messages.ListTablesResponse; import org.apache.fluss.rpc.messages.PbBucketMetadata; @@ -32,6 +35,7 @@ import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.kafka.common.Node; import org.apache.kafka.common.Uuid; @@ -47,14 +51,20 @@ import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; @@ -72,7 +82,7 @@ public void testNamedTopicForEverySupportedVersion() { new MetadataRequestData() .setTopics( MetadataRequest.convertToMetadataRequestTopic( - Collections.singletonList("topic"))), + Collections.singletonList("kafka.topic"))), version); if (version >= 9) { request.data() @@ -92,7 +102,7 @@ public void testNamedTopicForEverySupportedVersion() { assertThat(broker.host()).isEqualTo("broker-1"); assertThat(broker.port()).isEqualTo(9092); assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); - MetadataResponseTopic topic = response.data().topics().find("topic"); + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); assertThat(topic.partitions()).hasSize(2); assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); @@ -121,7 +131,7 @@ public void testAllTopicsForEverySupportedVersion() { assertThat(response.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactly("other", "topic"); + .containsExactly("kafka.other", "kafka.topic"); } } @@ -133,15 +143,16 @@ public void testAllTopicsAndMissingAndInvalidTopic() { handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); assertThat(allTopics.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactlyInAnyOrder("other", "topic"); + .containsExactlyInAnyOrder("kafka.other", "kafka.topic"); MetadataRequest requestedTopics = - new MetadataRequest.Builder(Arrays.asList("missing", "invalid topic"), false) + new MetadataRequest.Builder( + Arrays.asList("kafka.missing", "kafka.invalid topic"), false) .build((short) 9); MetadataResponse errors = handle(service, requestedTopics, (short) 9); assertThat(errors.errors()) - .containsEntry("missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) - .containsEntry("invalid topic", Errors.INVALID_TOPIC_EXCEPTION); + .containsEntry("kafka.missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("kafka.invalid topic", Errors.INVALID_TOPIC_EXCEPTION); } @Test @@ -154,7 +165,7 @@ public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { .setTopics( Collections.singletonList( new MetadataRequestTopic() - .setName("topic") + .setName("kafka.topic") .setTopicId( new Uuid( 0x466c757373000000L, @@ -164,7 +175,7 @@ public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { MetadataResponse response = handle(service, request, version); assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); - assertThat(response.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + assertThat(response.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); } } @@ -197,24 +208,24 @@ public void testV10AndV11RejectTopicIdOnlyLookup() { public void testTopicIdentityAcrossDeleteAndRecreate() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - MetadataResponse initial = handle(service, namedTopicRequest("topic"), (short) 11); - assertThat(initial.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + MetadataResponse initial = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + assertThat(initial.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); - service.removeTable("topic"); - MetadataResponse deleted = handle(service, namedTopicRequest("topic"), (short) 11); + service.removeTable("kafka.topic"); + MetadataResponse deleted = handle(service, namedTopicRequest("kafka.topic"), (short) 11); assertThat(deleted.errorCounts()) .containsExactlyEntriesOf( Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); - service.putTable("topic", 223L); + service.putTable("kafka.topic", 223L); Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); MetadataResponse recreatedByName = handle( service, - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11), (short) 11); - assertThat(recreatedByName.data().topics().find("topic").topicId()) + assertThat(recreatedByName.data().topics().find("kafka.topic").topicId()) .isEqualTo(recreatedTopicId) .isNotEqualTo(TOPIC_ID); } @@ -222,10 +233,10 @@ public void testTopicIdentityAcrossDeleteAndRecreate() { @Test public void testDeleteRaceBecomesUnknownTopicResult() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); - service.removeTable("topic"); + service.removeTable("kafka.topic"); service.failNextMetadataAsMissing = true; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); assertThat(response.errorCounts()) .containsExactlyEntriesOf( @@ -237,12 +248,12 @@ public void testUnavailableLeaderUsesPartitionError() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicLeaderAvailable = false; MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11); MetadataResponse response = handle(service, request, (short) 11); - MetadataResponseTopic topic = response.data().topics().find("topic"); + MetadataResponseTopic topic = response.data().topics().find("kafka.topic"); assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); MetadataResponsePartition partition = topic.partitions().get(0); assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); @@ -257,13 +268,13 @@ public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicIsr = new int[] {1}; MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11); MetadataResponse response = handle(service, request, (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.replicaNodes()).containsExactly(1, 2); assertThat(partition.isrNodes()).containsExactly(1); assertThat(partition.offlineReplicas()).isEmpty(); @@ -274,10 +285,10 @@ public void testLegacyMetadataUsesLeaderOnlyIsr() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.topicBucketEpoch = null; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.leaderId()).isEqualTo(1); assertThat(partition.replicaNodes()).containsExactly(1, 2); assertThat(partition.isrNodes()).containsExactly(1); @@ -290,10 +301,10 @@ public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() { service.topicBucketEpoch = null; service.topicLeaderAvailable = false; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); assertThat(partition.isrNodes()).isEmpty(); } @@ -305,10 +316,11 @@ public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { service.topicBucketEpoch = bucketEpoch; service.topicIsr = new int[0]; - MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + MetadataResponse response = + handle(service, namedTopicRequest("kafka.topic"), (short) 11); MetadataResponsePartition partition = - response.data().topics().find("topic").partitions().get(0); + response.data().topics().find("kafka.topic").partitions().get(0); assertThat(partition.isrNodes()).isEmpty(); } } @@ -318,7 +330,7 @@ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.failMetadata = true; MetadataRequest request = - new MetadataRequest.Builder(Collections.singletonList("topic"), false) + new MetadataRequest.Builder(Collections.singletonList("kafka.topic"), false) .build((short) 11); MetadataResponse response = handle(service, request, (short) 11); @@ -332,20 +344,20 @@ public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { public void testMetadataUsesDdlContractForEveryVersion() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.putTable( - "no_mapping", + "kafka.no_mapping", 125L, TableDescriptor.builder() .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) .distributedBy(2) .build()); service.putTable( - "bad_format", + "kafka.bad_format", 126L, TableDescriptor.builder(defaultDescriptor()) .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "json") .build()); service.putTable( - "primary_key", + "kafka.primary_key", 127L, TableDescriptor.builder(defaultDescriptor()) .schema( @@ -355,24 +367,29 @@ public void testMetadataUsesDdlContractForEveryVersion() { .build()) .build()); service.putTable( - "partitioned", + "kafka.partitioned", 128L, TableDescriptor.builder(defaultDescriptor()).partitionedBy("body").build()); service.putTable( - "indexed", + "kafka.indexed", 129L, TableDescriptor.builder(defaultDescriptor()).logFormat(LogFormat.INDEXED).build()); - service.putTable("invalid topic", 130L); + service.putTable("kafka.invalid topic", 130L); List invalidMappings = - Arrays.asList("no_mapping", "bad_format", "primary_key", "partitioned", "indexed"); + Arrays.asList( + "kafka.no_mapping", + "kafka.bad_format", + "kafka.primary_key", + "kafka.partitioned", + "kafka.indexed"); for (short version = 0; version <= 11; version++) { MetadataResponse all = handle(service, allTopicsRequest(version), version); assertThat(all.data().topics()) .extracting(MetadataResponseTopic::name) - .containsExactly("other", "topic"); + .containsExactly("kafka.other", "kafka.topic"); List requested = new ArrayList<>(invalidMappings); - requested.add("topic"); - requested.add("missing"); + requested.add("kafka.topic"); + requested.add("kafka.missing"); MetadataResponse named = handle( service, @@ -388,9 +405,9 @@ public void testMetadataUsesDdlContractForEveryVersion() { assertThat(topic.errorCode()).isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); assertThat(topic.partitions()).isEmpty(); } - assertThat(named.data().topics().find("topic").errorCode()) + assertThat(named.data().topics().find("kafka.topic").errorCode()) .isEqualTo(Errors.NONE.code()); - assertThat(named.data().topics().find("missing").errorCode()) + assertThat(named.data().topics().find("kafka.missing").errorCode()) .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); } } @@ -399,18 +416,135 @@ public void testMetadataUsesDdlContractForEveryVersion() { public void testMetadataReflectsMappingChangesWithoutChangingTopicIdentity() { TestingMetadataGatewayService service = new TestingMetadataGatewayService(); service.putTable( - "topic", + "kafka.topic", 123L, TableDescriptor.builder(defaultDescriptor()) .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") .build()); - MetadataResponse invalid = handle(service, namedTopicRequest("topic"), (short) 11); - assertThat(invalid.data().topics().find("topic").errorCode()) + MetadataResponse invalid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + assertThat(invalid.data().topics().find("kafka.topic").errorCode()) .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); - service.putTable("topic", 123L); - MetadataResponse valid = handle(service, namedTopicRequest("topic"), (short) 11); - assertThat(valid.data().topics().find("topic").errorCode()).isEqualTo(Errors.NONE.code()); - assertThat(valid.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + service.putTable("kafka.topic", 123L); + MetadataResponse valid = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + assertThat(valid.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(valid.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); + } + + @Test + public void testSameTableNameInDifferentDatabasesForEveryVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("sales.topic", 223L); + for (short version = 0; version <= 11; version++) { + MetadataResponse all = handle(service, allTopicsRequest(version), version); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic", "sales.topic"); + MetadataResponse named = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList( + "kafka.topic", "sales.topic"))), + version), + version); + assertThat(named.data().topics()).hasSize(2); + assertThat(named.errorCounts()).containsOnlyKeys(Errors.NONE); + if (version >= 10) { + assertThat(named.data().topics().find("kafka.topic").topicId()).isEqualTo(TOPIC_ID); + assertThat(named.data().topics().find("sales.topic").topicId()) + .isEqualTo(new Uuid(0x466c757373000000L, 223L)); + } + } + } + + @Test + public void testRejectUnqualifiedAndMalformedNamesWithoutHidingValidTopics() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + List invalidNames = Arrays.asList("topic", ".topic", "kafka.", "kafka.topic.extra"); + List names = new ArrayList<>(invalidNames); + names.add("kafka.topic"); + MetadataResponse response = + handle( + service, + new MetadataRequest.Builder(names, false).build((short) 11), + (short) 11); + for (String name : invalidNames) { + assertThat(response.data().topics().find(name).errorCode()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION.code()); + } + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.NONE.code()); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMissingDatabaseRetainsBrokersAndValidTopics(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failListTablesAsFuture = failedFuture; + for (short version = 0; version <= 11; version++) { + for (List names : + Arrays.asList( + Collections.singletonList("missing_db.topic"), + Arrays.asList("missing_db.topic", "kafka.topic"))) { + MetadataResponse response = + handle( + service, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest + .convertToMetadataRequestTopic( + names)), + version), + version); + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()).hasSize(names.size()); + assertThat(response.data().topics().find("missing_db.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + if (names.contains("kafka.topic")) { + MetadataResponseTopic valid = response.data().topics().find("kafka.topic"); + assertThat(valid.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(valid.partitions()).hasSize(2); + } + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testDatabaseDeletedDuringDiscoveryRetainsOtherTopics(boolean failedFuture) { + for (short version = 0; version <= 11; version++) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.putTable("deleted_db.topic", 223L); + service.deleteDatabaseBeforeListing = "deleted_db"; + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, allTopicsRequest(version), version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("kafka.other", "kafka.topic"); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testUnexpectedListTablesFailureIsNotTreatedAsMissingDatabase(boolean failedFuture) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failNextMetadataAsMissing = true; + service.listTablesFailure = new IllegalStateException("metadata unavailable"); + service.failListTablesAsFuture = failedFuture; + + MetadataResponse response = handle(service, namedTopicRequest("kafka.topic"), (short) 11); + + assertThat(response.data().topics().find("kafka.topic").errorCode()) + .isEqualTo(Errors.UNKNOWN_SERVER_ERROR.code()); } private static TableDescriptor defaultDescriptor() { @@ -441,7 +575,7 @@ private static MetadataRequest allTopicsRequest(short version) { private static MetadataResponse handle( TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { - KafkaRequestHandler handler = new KafkaRequestHandler(service, service, "kafka"); + KafkaRequestHandler handler = new KafkaRequestHandler(service, service); ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); KafkaRequest request; try { @@ -472,6 +606,7 @@ private static MetadataResponse handle( private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + private final Set databases = new LinkedHashSet<>(); private final Map tables = new LinkedHashMap<>(); private final Map descriptors = new LinkedHashMap<>(); private String lastListenerName; @@ -480,17 +615,49 @@ private static final class TestingMetadataGatewayService extends TestingTabletGa private Integer topicBucketEpoch = 7; private boolean failMetadata; private boolean failNextMetadataAsMissing; + private boolean failListTablesAsFuture; + private String deleteDatabaseBeforeListing; + private RuntimeException listTablesFailure; private TestingMetadataGatewayService() { - putTable("topic", 123L); - putTable("other", 124L); + putTable("kafka.topic", 123L); + putTable("kafka.other", 124L); + } + + @Override + public CompletableFuture listDatabases( + ListDatabasesRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + return CompletableFuture.completedFuture( + new ListDatabasesResponse().addAllDatabaseNames(databases)); } @Override public CompletableFuture listTables(ListTablesRequest request) { - assertThat(request.getDatabaseName()).isEqualTo("kafka"); + assertThat(currentListenerName()).isEqualTo("KAFKA"); + String database = request.getDatabaseName(); + if (database.equals(deleteDatabaseBeforeListing)) { + databases.remove(database); + tables.keySet().removeIf(name -> name.startsWith(database + ".")); + } + RuntimeException failure = listTablesFailure; + if (failure == null && !databases.contains(database)) { + failure = new DatabaseNotExistException("Database does not exist."); + } + if (failure != null) { + if (failListTablesAsFuture) { + return FutureUtils.completedExceptionally(new CompletionException(failure)); + } + throw failure; + } + String prefix = request.getDatabaseName() + "."; + List names = + tables.keySet().stream() + .filter(name -> name.startsWith(prefix)) + .map(name -> name.substring(prefix.length())) + .collect(Collectors.toList()); return CompletableFuture.completedFuture( - new ListTablesResponse().addAllTableNames(new ArrayList<>(tables.keySet()))); + new ListTablesResponse().addAllTableNames(names)); } @Override @@ -509,25 +676,23 @@ public CompletableFuture metadat } List topics = new ArrayList<>(); for (PbTablePath tablePath : request.getTablePathsList()) { - Long tableId = tables.get(tablePath.getTableName()); - if (tableId != null) { - topics.add( - tableMetadata( - tablePath.getTableName(), - tableId, - !"topic".equals(tablePath.getTableName()) - || topicLeaderAvailable, - "topic".equals(tablePath.getTableName()) - ? topicIsr - : new int[] {1, 2}, - "topic".equals(tablePath.getTableName()) - ? topicBucketEpoch - : Integer.valueOf(7)) - .setTableJson( - descriptors - .get(tablePath.getTableName()) - .toJsonBytes())); + String topicName = tablePath.getDatabaseName() + "." + tablePath.getTableName(); + Long tableId = tables.get(topicName); + if (tableId == null) { + throw new TableNotExistException("Table does not exist: " + topicName); } + topics.add( + tableMetadata( + topicName, + tableId, + !"kafka.topic".equals(topicName) || topicLeaderAvailable, + "kafka.topic".equals(topicName) + ? topicIsr + : new int[] {1, 2}, + "kafka.topic".equals(topicName) + ? topicBucketEpoch + : Integer.valueOf(7)) + .setTableJson(descriptors.get(topicName).toJsonBytes())); } return CompletableFuture.completedFuture( new org.apache.fluss.rpc.messages.MetadataResponse() @@ -550,6 +715,7 @@ private void putTable(String topic, long tableId) { } private void putTable(String topic, long tableId, TableDescriptor descriptor) { + databases.add(topic.substring(0, topic.indexOf('.'))); tables.put(topic, tableId); descriptors.put(topic, descriptor); } @@ -568,7 +734,9 @@ private static PbTableMetadata tableMetadata( PbTableMetadata table = new PbTableMetadata() .setTablePath( - new PbTablePath().setDatabaseName("kafka").setTableName(topic)) + new PbTablePath() + .setDatabaseName(topic.substring(0, topic.indexOf('.'))) + .setTableName(topic.substring(topic.indexOf('.') + 1))) .setTableId(tableId) .addAllBucketMetadatas( Arrays.asList( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java new file mode 100644 index 00000000000..2f18ffe495d --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataITCase.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.dropDatabase; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies qualified Metadata discovery over a real Kafka listener and TabletServer. */ +public class KafkaMetadataITCase { + + @RegisterExtension + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(1) + .setClusterConf(clusterConfiguration()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + public void testQualifiedMetadataAcrossDatabases() throws Exception { + TableDescriptor descriptor = topicDescriptor(); + long first = createTable(CLUSTER, TablePath.of("first_db", "events"), descriptor); + long second = createTable(CLUSTER, TablePath.of("second_db", "events"), descriptor); + CLUSTER.waitUntilTableReady(first); + CLUSTER.waitUntilTableReady(second); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + + for (short version = 0; version <= 11; version++) { + MetadataResponse named = + query( + node, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList( + "first_db.events", + "second_db.events", + "first_db.missing"))), + version)); + assertThat(named.data().topics().find("first_db.events").partitions()).hasSize(2); + assertThat(named.data().topics().find("second_db.events").partitions()).hasSize(2); + assertThat(named.data().topics().find("first_db.missing").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(named.data().topics().find("first_db.events").topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373000000L, first) : Uuid.ZERO_UUID); + assertThat(named.data().topics().find("second_db.events").topicId()) + .isEqualTo( + version >= 10 ? new Uuid(0x466c757373000000L, second) : Uuid.ZERO_UUID); + assertThat(named.brokers()).hasSize(1); + assertThat(named.brokers().iterator().next().port()).isEqualTo(node.port()); + + MetadataRequestData allTopics = new MetadataRequestData(); + allTopics.setTopics(version == 0 ? Collections.emptyList() : null); + MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("first_db.events", "second_db.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + @Test + public void testDeletedDatabaseDoesNotHideBrokersOrOtherTopics() throws Exception { + TableDescriptor descriptor = topicDescriptor(); + long existing = createTable(CLUSTER, TablePath.of("existing_db", "events"), descriptor); + long deleted = createTable(CLUSTER, TablePath.of("deleted_db", "events"), descriptor); + CLUSTER.waitUntilTableReady(existing); + CLUSTER.waitUntilTableReady(deleted); + dropDatabase(CLUSTER, "deleted_db"); + ServerNode node = CLUSTER.getTabletServerInfos().get(0).node("KAFKA"); + + for (short version = 0; version <= 11; version++) { + MetadataResponse missing = + query( + node, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Collections.singletonList( + "deleted_db.events"))), + version)); + assertThat(missing.brokers()).hasSize(1); + assertThat(missing.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + + MetadataResponse mixed = + query( + node, + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Arrays.asList( + "deleted_db.events", + "existing_db.events"))), + version)); + assertThat(mixed.brokers()).hasSize(1); + assertThat(mixed.brokers().iterator().next().port()).isEqualTo(node.port()); + assertThat(mixed.data().topics().find("deleted_db.events").errorCode()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()); + assertThat(mixed.data().topics().find("existing_db.events").errorCode()) + .isEqualTo(Errors.NONE.code()); + assertThat(mixed.data().topics().find("existing_db.events").partitions()).hasSize(2); + + MetadataRequestData allTopics = new MetadataRequestData(); + allTopics.setTopics(version == 0 ? Collections.emptyList() : null); + MetadataResponse all = query(node, new MetadataRequest(allTopics, version)); + assertThat(all.brokers()).hasSize(1); + assertThat(all.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("existing_db.events"); + assertThat(all.errorCounts()).containsOnlyKeys(Errors.NONE); + } + } + + private static TableDescriptor topicDescriptor() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty("kafka.value.format", "raw") + .build(); + } + + private static Configuration clusterConfiguration() { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.KAFKA_ENABLED, true); + conf.set(ConfigOptions.NETTY_SERVER_NUM_WORKER_THREADS, 3); + return conf; + } + + private static MetadataResponse query(ServerNode node, MetadataRequest request) + throws Exception { + RequestHeader header = + new RequestHeader(ApiKeys.METADATA, request.version(), "metadata-test", 1); + ByteBuffer encoded = request.serializeWithHeader(header); + byte[] requestBytes = new byte[encoded.remaining()]; + encoded.get(requestBytes); + try (Socket socket = new Socket(node.host(), node.port())) { + socket.setSoTimeout(10000); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + byte[] responseBytes = new byte[input.readInt()]; + input.readFully(responseBytes); + return (MetadataResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(responseBytes), header); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index c4476c69d59..8bf6cec2085 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -211,6 +211,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { private static KafkaRequestHandler createKafkaRequestHandler() { TestingTabletGatewayService service = new TestingTabletGatewayService(); - return new KafkaRequestHandler(service, service, "kafka"); + return new KafkaRequestHandler(service, service); } } From e27aff85e488c0f0fc38f441fc723b851a143a40 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:18:03 +0800 Subject: [PATCH 08/11] [kafka] Add independently testable Produce protocol handling Validate non-idempotent Produce v3-v11 and isolate invalid partitions before invoking a protocol-independent backend. Preserve partition order, acknowledgements and owned record data. Leave production registration to the append integration PR. Validation: Java 11, mvn -o -pl fluss-kafka spotless:apply clean verify; 69 unit tests and 2 integration tests passed, including Checkstyle, Spotless and RAT. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 621/621 AI-Contributed/UT: 366/366 --- .../kafka/api/produce/ProduceHandler.java | 284 ++++++++++++++ .../backend/produce/KafkaProduceBackend.java | 29 ++ .../backend/produce/KafkaProduceCommand.java | 197 ++++++++++ .../backend/produce/KafkaProduceResult.java | 111 ++++++ .../fluss/kafka/KafkaProduceProtocolTest.java | 366 ++++++++++++++++++ 5 files changed, 987 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java new file mode 100644 index 00000000000..0eb14b26ff2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidRequiredAcksException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.BaseRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements non-idempotent Kafka Produce versions 3 through 11. */ +@Internal +public final class ProduceHandler implements KafkaApiHandler { + + private static final short MIN_SUPPORTED_VERSION = 3; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec(ApiKeys.PRODUCE, MIN_SUPPORTED_VERSION, (short) 11, true); + + private final KafkaProduceBackend backend; + + /** Creates a non-idempotent Produce handler. */ + public ProduceHandler(KafkaProduceBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ProduceRequest request) { + validateRequest(request); + List topics = new ArrayList<>(); + Map failures = new HashMap<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + try { + if (!Topic.isValid(topic.name())) { + throw new InvalidTopicException("Invalid Kafka topic name " + topic.name()); + } + if (partition.index() < 0) { + throw new InvalidRequestException("Negative Kafka partition ID."); + } + partitions.add( + new PartitionWrite( + partition.index(), + copyRecords(request.version(), partition.records()))); + } catch (RuntimeException failure) { + failures.put( + new TopicPartition(topic.name(), partition.index()), + failedPartition(partition.index(), failure)); + } + } + if (!partitions.isEmpty()) { + topics.add(new TopicWrite(topic.name(), partitions)); + } + } + KafkaProduceCommand command = + new KafkaProduceCommand( + request.acks(), + request.timeout(), + topics, + context.listenerName(), + clientAddress(context.remoteAddress())); + CompletableFuture result; + try { + result = + topics.isEmpty() + ? CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList())) + : checkNotNull(backend.write(command)); + } catch (RuntimeException failure) { + result = new CompletableFuture<>(); + result.completeExceptionally(failure); + } + return result.handle( + (written, failure) -> { + Map results = new HashMap<>(failures); + if (failure == null && written != null) { + for (TopicResult topic : written.topics()) { + for (PartitionResult partition : topic.partitions()) { + results.putIfAbsent( + new TopicPartition( + topic.topicName(), partition.partitionId()), + partition); + } + } + } + List ordered = new ArrayList<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + TopicPartition key = + new TopicPartition(topic.name(), partition.index()); + PartitionResult value = results.get(key); + if (value == null) { + value = + failedPartition( + partition.index(), + failure == null + ? new IllegalStateException( + "Produce backend omitted this partition.") + : failure); + } + partitions.add(value); + } + ordered.add(new TopicResult(topic.name(), partitions)); + } + return toResponse(new KafkaProduceResult(ordered)); + }); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + while (failure instanceof CompletionException && failure.getCause() != null) { + failure = failure.getCause(); + } + return new PartitionResult( + partitionId, Errors.forException(failure), -1L, failure.getMessage()); + } + + private static void validateRequest(ProduceRequest request) { + Set names = new HashSet<>(); + for (TopicProduceData topic : request.data().topicData()) { + if (!names.add(topic.name())) { + throw new InvalidRequestException("Duplicate Kafka topic in Produce request."); + } + Set partitions = new HashSet<>(); + for (PartitionProduceData partition : topic.partitionData()) { + if (!partitions.add(partition.index())) { + throw new InvalidRequestException( + "Duplicate Kafka partition in Produce request."); + } + } + } + if (request.transactionalId() != null) { + throw new InvalidRequestException( + "Transactional Produce is not supported by the Fluss Kafka compatibility layer."); + } + if (request.acks() != -1 && request.acks() != 0 && request.acks() != 1) { + throw new InvalidRequiredAcksException("Invalid required acks " + request.acks()); + } + } + + private static List copyRecords( + short version, BaseRecords baseRecords) { + if (!(baseRecords instanceof Records)) { + throw new InvalidRequestException("Unsupported Kafka records representation."); + } + ProduceRequest.validateRecords(version, baseRecords); + Records records = (Records) baseRecords; + if (records.sizeInBytes() == 0) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + List copied = new ArrayList<>(); + int validBytes = 0; + for (RecordBatch batch : records.batches()) { + validBytes += batch.sizeInBytes(); + batch.ensureValid(); + if (batch.hasProducerId() || batch.isTransactional() || batch.isControlBatch()) { + throw new InvalidRequestException( + "Idempotent, transactional, and control record batches are not supported."); + } + for (org.apache.kafka.common.record.Record record : batch) { + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } + } + if (copied.isEmpty() || validBytes != records.sizeInBytes()) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + return copied; + } + + private static ProduceResponse toResponse(KafkaProduceResult result) { + ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0); + for (TopicResult topic : result.topics()) { + ProduceResponseData.TopicProduceResponse topicResponse = + new ProduceResponseData.TopicProduceResponse().setName(topic.topicName()); + for (PartitionResult partition : topic.partitions()) { + topicResponse + .partitionResponses() + .add( + new ProduceResponseData.PartitionProduceResponse() + .setIndex(partition.partitionId()) + .setErrorCode(partition.error().code()) + .setBaseOffset(partition.baseOffset()) + .setLogAppendTimeMs(-1L) + .setLogStartOffset(-1L) + .setErrorMessage(partition.errorMessage())); + } + data.responses().add(topicResponse); + } + return new ProduceResponse(data); + } + + private static List copyHeaders(Header[] headers) { + List copied = new ArrayList<>(headers.length); + for (Header header : headers) { + copied.add(new RecordHeader(header.key(), header.value())); + } + return copied; + } + + private static byte[] copyBuffer(ByteBuffer buffer) { + if (buffer == null) { + return null; + } + ByteBuffer duplicate = buffer.duplicate(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return bytes; + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java new file mode 100644 index 00000000000..3b8428c5670 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Produce API. */ +@Internal +public interface KafkaProduceBackend { + /** Writes copied Kafka records through the native Fluss write path. */ + CompletableFuture write(KafkaProduceCommand command); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java new file mode 100644 index 00000000000..722f3d3d245 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Protocol-independent write command used by the Kafka Produce backend. */ +@Internal +public final class KafkaProduceCommand { + + private final short acks; + private final int timeoutMs; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a Kafka write command. */ + public KafkaProduceCommand( + short acks, + int timeoutMs, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.acks = acks; + this.timeoutMs = timeoutMs; + this.topics = immutableCopy(topics); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns Kafka required acknowledgements. */ + public short acks() { + return acks; + } + + /** Returns the Produce timeout in milliseconds. */ + public int timeoutMs() { + return timeoutMs; + } + + /** Returns the topic writes in request order. */ + public List topics() { + return topics; + } + + /** Returns the listener that received the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client network address when available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Records addressed to one Kafka topic. */ + @Internal + public static final class TopicWrite { + private final String topicName; + private final List partitions; + + /** Creates the writes for one topic. */ + public TopicWrite(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns partition writes in request order. */ + public List partitions() { + return partitions; + } + } + + /** Records addressed to one Kafka partition. */ + @Internal + public static final class PartitionWrite { + private final int partitionId; + private final List records; + + /** Creates the writes for one partition. */ + public PartitionWrite(int partitionId, List records) { + this.partitionId = partitionId; + this.records = immutableCopy(records); + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns copied records in append order. */ + public List records() { + return records; + } + } + + /** A copied Kafka record whose lifetime is independent of the network request buffer. */ + @Internal + public static final class Record { + private final long timestamp; + private final @Nullable byte[] key; + private final @Nullable byte[] value; + private final List headers; + + /** Creates a copied Kafka record. */ + public Record( + long timestamp, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { + this.timestamp = timestamp; + this.key = copyNullable(key); + this.value = copyNullable(value); + this.headers = immutableCopy(headers); + } + + /** Returns the Kafka record timestamp. */ + public long timestamp() { + return timestamp; + } + + /** Returns a copy of the nullable Kafka record key. */ + public @Nullable byte[] key() { + return copyNullable(key); + } + + /** Returns a copy of the nullable Kafka record value. */ + public @Nullable byte[] value() { + return copyNullable(value); + } + + /** Returns the copied Kafka headers in record order. */ + public List headers() { + return headers; + } + + private static @Nullable byte[] copyNullable(@Nullable byte[] value) { + return value == null ? null : value.clone(); + } + } + + /** A copied Kafka record header. */ + @Internal + public static final class RecordHeader { + private final String name; + private final @Nullable byte[] value; + + /** Creates a copied Kafka record header. */ + public RecordHeader(String name, @Nullable byte[] value) { + this.name = checkNotNull(name); + this.value = value == null ? null : value.clone(); + } + + /** Returns the header name. */ + public String name() { + return name; + } + + /** Returns a copy of the nullable header value. */ + public @Nullable byte[] value() { + return value == null ? null : value.clone(); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java new file mode 100644 index 00000000000..218c566bde4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Result of a Kafka Produce backend invocation. */ +@Internal +public final class KafkaProduceResult { + private final List topics; + + /** Creates a Produce result. */ + public KafkaProduceResult(List topics) { + this.topics = immutableCopy(topics); + } + + /** Returns topic results in request order. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Results for one topic. */ + @Internal + public static final class TopicResult { + private final String topicName; + private final List partitions; + + /** Creates the result for one topic. */ + public TopicResult(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns the partition results in request order. */ + public List partitions() { + return partitions; + } + } + + /** Result for one partition. */ + @Internal + public static final class PartitionResult { + private final int partitionId; + private final Errors error; + private final long baseOffset; + private final @Nullable String errorMessage; + + /** Creates the result for one partition. */ + public PartitionResult( + int partitionId, Errors error, long baseOffset, @Nullable String errorMessage) { + this.partitionId = partitionId; + this.error = checkNotNull(error); + this.baseOffset = baseOffset; + this.errorMessage = errorMessage; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the Kafka protocol error. */ + public Errors error() { + return error; + } + + /** Returns the first appended offset, or {@code -1} on failure. */ + public long baseOffset() { + return baseOffset; + } + + /** Returns an optional diagnostic error message. */ + public @Nullable String errorMessage() { + return errorMessage; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java new file mode 100644 index 00000000000..bf043823689 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.kafka.api.produce.ProduceHandler; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; + +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol tests independent of the Fluss append and record conversion backends. */ +class KafkaProduceProtocolTest { + + @Test + void testSupportedVersionsCopyRecordsAndForwardContext() { + for (short version = 3; version <= 11; version++) { + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request(version, (short) -1, partition(0, records())), + command -> { + copied.set(command); + return successful(command); + }); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(42L); + KafkaProduceCommand command = copied.get(); + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(4321); + assertThat(command.listenerName()).isEqualTo("KAFKA"); + KafkaProduceCommand.Record record = + command.topics().get(0).partitions().get(0).records().get(0); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()).hasSize(1); + assertThat(record.headers().get(0).value()).isNull(); + byte[] key = record.key(); + key[0] = 9; + assertThat(record.key()).containsExactly((byte) 1); + } + } + + @Test + void testCorruptPartitionDoesNotSuppressValidPartition() { + MemoryRecords corrupt = records(); + corrupt.buffer().put(corrupt.sizeInBytes() - 1, (byte) 99); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, corrupt), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + assertThat(command.topics().get(0).partitions().get(0).partitionId()) + .isEqualTo(1); + return successful(command); + }); + assertErrors(response, Errors.CORRUPT_MESSAGE, Errors.NONE); + } + + @Test + void testInvalidTopicIsIsolated() { + ProduceRequest request = request((short) 11, (short) 1, partition(0, records())); + request.data() + .topicData() + .add( + new TopicProduceData() + .setName("bad/name") + .setPartitionData( + Collections.singletonList(partition(0, records())))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()).hasSize(1); + return successful(command); + }); + assertThat(response.errorCounts()) + .containsEntry(Errors.NONE, 1) + .containsEntry(Errors.INVALID_TOPIC_EXCEPTION, 1); + } + + @Test + void testEmptyNegativeAndIdempotentPartitionsAreRejectedLocally() { + MemoryRecords idempotent = + MemoryRecords.withIdempotentRecords( + Compression.NONE, 10L, (short) 0, 0, new SimpleRecord(new byte[] {1})); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(-1, records()), + partition(2, idempotent), + partition(3, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + return successful(command); + }); + assertErrors( + response, + Errors.INVALID_RECORD, + Errors.INVALID_REQUEST, + Errors.INVALID_REQUEST, + Errors.NONE); + } + + @Test + void testRequestValidationNeverCallsBackend() { + KafkaProduceBackend unused = + command -> { + throw new AssertionError("Backend must not be called"); + }; + assertErrors( + dispatch(request((short) 11, (short) 2, partition(0, records())), unused), + Errors.INVALID_REQUIRED_ACKS); + ProduceRequest transactional = request((short) 11, (short) 1, partition(0, records())); + transactional.data().setTransactionalId("transaction"); + assertErrors( + dispatch(new ProduceRequest(transactional.data(), transactional.version()), unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch( + request( + (short) 11, + (short) 1, + partition(0, records()), + partition(0, records())), + unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch(request((short) 2, (short) 1, partition(0, records())), unused), + Errors.UNSUPPORTED_VERSION); + } + + @Test + void testBackendFailuresAndMissingResultsPreserveValidationErrors() { + for (boolean synchronous : Arrays.asList(true, false)) { + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(1, records())), + command -> { + if (synchronous) { + throw new TimeoutException("append timed out"); + } + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally( + new TimeoutException("append timed out")); + return failed; + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.REQUEST_TIMED_OUT); + } + assertErrors( + dispatch( + request((short) 11, (short) 1, partition(0, records())), + command -> + CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList()))), + Errors.UNKNOWN_SERVER_ERROR); + } + + @Test + void testAcksZeroSuccessAndFailureReleaseBufferWithoutSendingResponse() { + for (boolean failure : Arrays.asList(false, true)) { + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + ProduceRequest body = request((short) 11, (short) 0, partition(0, records())); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, (short) 11, "producer", 1); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + CompletableFuture pending = new CompletableFuture<>(); + try { + channel.writeInbound(buffer); + KafkaRequest parsed = (KafkaRequest) requests.pollRequest(1000); + dispatcher(command -> pending) + .dispatch(parsed) + .whenComplete( + (response, error) -> { + if (error == null) { + parsed.complete(response); + } else { + parsed.fail(error); + } + }); + assertThat(parsed.future()).isNotDone(); + if (failure) { + pending.completeExceptionally(new TimeoutException("failure")); + } else { + pending.complete( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, Errors.NONE, 42L, null)))))); + } + channel.runPendingTasks(); + assertThat(parsed.future()).isDone(); + assertThat((Object) channel.readOutbound()).isNull(); + assertThat(buffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + } + + private static ProduceResponse dispatch(ProduceRequest body, KafkaProduceBackend backend) { + ByteBuf buffer = Unpooled.buffer(1); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, body.version(), "producer", 1); + KafkaRequest request = + new KafkaRequest( + ApiKeys.PRODUCE, + body.version(), + header, + body, + "KAFKA", + buffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + buffer.release(); + request.complete(dispatcher(backend).dispatch(request).join()); + ByteBuf response = request.responseBuffer(); + try { + ByteBuffer bytes = response.nioBuffer(); + ResponseHeader.parse(bytes, header.toResponseHeader().headerVersion()); + return (ProduceResponse) + AbstractResponse.parseResponse(ApiKeys.PRODUCE, bytes, body.version()); + } finally { + response.release(); + } + } + + private static KafkaRequestDispatcher dispatcher(KafkaProduceBackend backend) { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ProduceHandler(backend)); + registry.freeze(); + return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); + } + + private static CompletableFuture successful(KafkaProduceCommand command) { + List topics = new ArrayList<>(); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + List partitions = new ArrayList<>(); + for (KafkaProduceCommand.PartitionWrite partition : topic.partitions()) { + partitions.add( + new PartitionResult(partition.partitionId(), Errors.NONE, 42L, null)); + } + topics.add(new TopicResult(topic.topicName(), partitions)); + } + return CompletableFuture.completedFuture(new KafkaProduceResult(topics)); + } + + private static ProduceRequest request( + short version, short acks, PartitionProduceData... partitions) { + TopicProduceData topic = + new TopicProduceData().setName("topic").setPartitionData(Arrays.asList(partitions)); + return new ProduceRequest( + new ProduceRequestData() + .setAcks(acks) + .setTimeoutMs(4321) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topic).iterator())), + version); + } + + private static PartitionProduceData partition(int index, MemoryRecords records) { + return new PartitionProduceData().setIndex(index).setRecords(records); + } + + private static MemoryRecords records() { + return MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.NONE, + new SimpleRecord( + 123L, + new byte[] {1}, + new byte[] {2}, + new Header[] {new RecordHeader("header", null)})); + } + + private static void assertErrors(ProduceResponse response, Errors... errors) { + assertThat(response.data().responses().find("topic").partitionResponses()) + .extracting(PartitionProduceResponse::errorCode) + .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); + } +} From ff27c55c709bf1386b6f45d549d7de2b8525b064 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 17:04:41 +0800 Subject: [PATCH 09/11] [kafka] Validate Produce batch offsets and stream compressed records Read compressed records through a closeable streaming iterator to avoid preallocating a record list from the declared batch count. Reject incoming Kafka batches whose offset range disagrees with their record count before copying records or invoking the backend. Keep Fluss-assigned offsets independent of this input validation and preserve partition error isolation. Cover all five codecs, owned record copies, malformed counts and offset ranges, and backend-assigned offsets. Java 11 validation passed: 117 unit tests and 4 integration tests, plus Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 42/42 AI-Contributed/UT: 168/168 --- .../kafka/api/produce/ProduceHandler.java | 42 ++++- .../fluss/kafka/KafkaProduceProtocolTest.java | 168 ++++++++++++++++++ 2 files changed, 202 insertions(+), 8 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java index 0eb14b26ff2..a598c0d6cfa 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -30,6 +30,7 @@ import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; +import org.apache.kafka.common.InvalidRecordException; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.InvalidRequiredAcksException; @@ -47,6 +48,8 @@ import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ProduceRequest; import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.CloseableIterator; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -219,14 +222,37 @@ private static List copyRecords( throw new InvalidRequestException( "Idempotent, transactional, and control record batches are not supported."); } - for (org.apache.kafka.common.record.Record record : batch) { - record.ensureValid(); - copied.add( - new KafkaProduceCommand.Record( - record.timestamp(), - copyBuffer(record.hasKey() ? record.key() : null), - copyBuffer(record.hasValue() ? record.value() : null), - copyHeaders(record.headers()))); + // Validate the incoming Kafka batch before copying away its offset metadata. Fluss + // assigns its own storage offsets later, independently of this input validation. + long countFromOffsets = batch.lastOffset() - batch.baseOffset() + 1; + Integer recordCount = batch.countOrNull(); + if (countFromOffsets <= 0 + || recordCount == null + || recordCount <= 0 + || countFromOffsets != recordCount) { + throw new InvalidRecordException( + "Invalid Kafka record batch offset range [" + + batch.baseOffset() + + ", " + + batch.lastOffset() + + "] for record count " + + recordCount + + "."); + } + // The ordinary compressed iterator preallocates a list using the untrusted record + // count. Stream records instead and close the decompressor on success and failure. + try (CloseableIterator iterator = + batch.streamingIterator(BufferSupplier.NO_CACHING)) { + while (iterator.hasNext()) { + org.apache.kafka.common.record.Record record = iterator.next(); + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } } } if (copied.isEmpty() || validBytes != records.sizeInBytes()) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java index bf043823689..c2160805510 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -41,6 +41,8 @@ import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.DefaultRecordBatch; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.SimpleRecord; @@ -50,7 +52,10 @@ import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.requests.ResponseHeader; +import org.apache.kafka.common.utils.Crc32C; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -102,6 +107,161 @@ void testSupportedVersionsCopyRecordsAndForwardContext() { } } + @ParameterizedTest + @EnumSource(CompressionType.class) + void testStreamedRecordsRemainIndependentOfInput(CompressionType compressionType) { + byte[] value = new byte[32 * 1024]; + Arrays.fill(value, (byte) 7); + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord( + 123L, + new byte[] {1}, + value, + new Header[] { + new RecordHeader("header", new byte[] {2, 3}), + new RecordHeader("header", null) + }), + new SimpleRecord(456L, null, new byte[0])); + AtomicReference copied = new AtomicReference<>(); + + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + copied.set(command); + return successful(command); + }); + assertErrors(response, Errors.NONE); + ByteBuffer input = records.buffer().duplicate(); + while (input.hasRemaining()) { + input.put((byte) 0); + } + + List result = + copied.get().topics().get(0).partitions().get(0).records(); + assertThat(result).hasSize(2); + assertThat(result.get(0).timestamp()).isEqualTo(123L); + assertThat(result.get(0).key()).containsExactly((byte) 1); + assertThat(result.get(0).value()).containsExactly(value); + assertThat(result.get(0).headers()) + .extracting(KafkaProduceCommand.RecordHeader::name) + .containsExactly("header", "header"); + assertThat(result.get(0).headers().get(0).value()).containsExactly((byte) 2, (byte) 3); + assertThat(result.get(0).headers().get(1).value()).isNull(); + assertThat(result.get(1).timestamp()).isEqualTo(456L); + assertThat(result.get(1).key()).isNull(); + assertThat(result.get(1).value()).isEmpty(); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testIncorrectRecordCountDoesNotSuppressValidPartition(CompressionType compressionType) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + ByteBuffer bytes = malformed.buffer(); + bytes.putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, 32); + bytes.putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, 31); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testInvalidBatchOffsetRangeDoesNotSuppressValidPartition(CompressionType compressionType) { + for (int[] invalidHeader : + new int[][] {{1, -1}, {1, 9}, {1, Integer.MAX_VALUE}, {0, 0}, {-1, 0}}) { + MemoryRecords malformed = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1})); + int recordCount = invalidHeader[0]; + int lastOffsetDelta = invalidHeader[1]; + malformed.buffer().putInt(DefaultRecordBatch.RECORDS_COUNT_OFFSET, recordCount); + malformed.buffer().putInt(DefaultRecordBatch.LAST_OFFSET_DELTA_OFFSET, lastOffsetDelta); + updateBatchChecksum(malformed); + + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, malformed), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()) + .extracting(KafkaProduceCommand.PartitionWrite::partitionId) + .containsExactly(1); + return successful(command); + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.NONE); + } + } + + @ParameterizedTest + @EnumSource(CompressionType.class) + void testValidBatchUsesOffsetAssignedByBackend(CompressionType compressionType) { + MemoryRecords records = + MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.of(compressionType).build(), + new SimpleRecord(new byte[] {1}), + new SimpleRecord(new byte[] {2}), + new SimpleRecord(new byte[] {3})); + long backendBaseOffset = 1L << 40; + ProduceResponse response = + dispatch( + request((short) 11, (short) 1, partition(0, records)), + command -> { + assertThat(command.topics().get(0).partitions().get(0).records()) + .hasSize(3); + return CompletableFuture.completedFuture( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, + Errors.NONE, + backendBaseOffset, + null)))))); + }); + assertErrors(response, Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(backendBaseOffset); + } + @Test void testCorruptPartitionDoesNotSuppressValidPartition() { MemoryRecords corrupt = records(); @@ -363,4 +523,12 @@ private static void assertErrors(ProduceResponse response, Errors... errors) { .extracting(PartitionProduceResponse::errorCode) .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); } + + private static void updateBatchChecksum(MemoryRecords records) { + ByteBuffer bytes = records.buffer(); + int checksumStart = DefaultRecordBatch.CRC_OFFSET + Integer.BYTES; + bytes.putInt( + DefaultRecordBatch.CRC_OFFSET, + (int) Crc32C.compute(bytes, checksumStart, bytes.limit() - checksumStart)); + } } From 21276e0728461ba8bf5ecae08d9749c7db043773 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:21:35 +0800 Subject: [PATCH 10/11] [kafka] Transcode mapped raw and string records to Arrow Reuse the DDL contract to assemble key, value, timestamp and ordered headers into physical rows. Decode strings with strict UTF-8 validation and preserve nullable values. Return owned heap log bytes and recycle Arrow writers on success and failure. Validation: Java 11, mvn -o -pl fluss-kafka spotless:apply clean verify; 75 unit tests and 2 integration tests passed, including Checkstyle, Spotless and RAT. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 314/314 AI-Contributed/UT: 326/326 --- .../transcode/ArrowKafkaRecordTranscoder.java | 87 +++++ .../transcode/FlussArrowRecordEncoder.java | 71 ++++ .../KafkaRecordEncodingException.java | 35 ++ .../transcode/KafkaRecordTranscoder.java | 32 ++ .../kafka/transcode/KafkaRowAssembler.java | 89 +++++ .../ArrowKafkaRecordTranscoderTest.java | 326 ++++++++++++++++++ 6 files changed, 640 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java new file mode 100644 index 00000000000..717f5cd33ff --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; + +import javax.annotation.Nullable; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Converts raw/string Kafka records using the DDL mapping into owned Fluss Arrow log bytes. */ +@Internal +public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); + + @Override + public BytesView transcode(List records, TableInfo tableInfo) throws Exception { + checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); + KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaRowAssembler assembler = new KafkaRowAssembler(schema); + List rows = new ArrayList<>(records.size()); + for (Record record : records) { + Object[] key = + schema.keyFormat() == null + ? new Object[0] + : new Object[] {decode(schema.keyFormat(), record.key())}; + rows.add( + assembler.assemble( + key, + new Object[] {decode(schema.valueFormat(), record.value())}, + record.timestamp(), + record.headers())); + } + return arrowRecordEncoder.encode(rows, tableInfo); + } + + private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) { + if (bytes == null || format == KafkaDataFormat.RAW) { + return bytes; + } + if (format != KafkaDataFormat.STRING) { + throw new KafkaRecordEncodingException("Unsupported Kafka data format: " + format); + } + try { + return BinaryString.fromString( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString()); + } catch (CharacterCodingException e) { + throw new KafkaRecordEncodingException("Kafka string field is not valid UTF-8.", e); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java new file mode 100644 index 00000000000..f74680ccaa2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.memory.UnmanagedPagedOutputView; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.MemoryLogRecordsArrowBuilder; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.arrow.ArrowWriter; +import org.apache.fluss.row.arrow.ArrowWriterPool; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator; + +import java.util.List; + +/** Encodes assembled physical Fluss rows into one native Arrow log batch. */ +@Internal +public final class FlussArrowRecordEncoder { + + private static final int INITIAL_PAGE_SIZE = 4096; + + /** Encodes all rows using the table's current schema ID and Arrow compression settings. */ + public BytesView encode(List rows, TableInfo tableInfo) throws Exception { + try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + ArrowWriterPool provider = new ArrowWriterPool(allocator)) { + ArrowWriter writer = + provider.getOrCreateWriter( + tableInfo.getTableId(), + tableInfo.getSchemaId(), + Integer.MAX_VALUE, + tableInfo.getRowType(), + tableInfo.getTableConfig().getArrowCompressionInfo()); + long epoch = writer.getEpoch(); + try { + MemoryLogRecordsArrowBuilder builder = + MemoryLogRecordsArrowBuilder.builder( + tableInfo.getSchemaId(), + writer, + new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), + true, + null); + for (GenericRow row : rows) { + builder.append(ChangeType.APPEND_ONLY, row); + } + // The output view owns heap pages. The result remains valid after the Arrow + // writer and allocator close, including across asynchronous append completion. + return builder.build(); + } finally { + writer.recycle(epoch); + } + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java new file mode 100644 index 00000000000..b48dd9d894f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that Kafka record bytes cannot be decoded using the configured data format. */ +@Internal +public final class KafkaRecordEncodingException extends IllegalArgumentException { + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message) { + super(message); + } + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java new file mode 100644 index 00000000000..eeff2f3d14f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; + +import java.util.List; + +/** Converts copied Kafka records into the native Fluss log representation. */ +@Internal +public interface KafkaRecordTranscoder { + /** Transcodes records according to the target Fluss table schema and log format. */ + BytesView transcode(List records, TableInfo tableInfo) throws Exception; +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java new file mode 100644 index 00000000000..4ebd506dc3c --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.schema.KafkaFieldProjection; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; + +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Assembles decoded Kafka key, value, and metadata into a physical Fluss row. */ +@Internal +public final class KafkaRowAssembler { + + private final KafkaTopicSchema topicSchema; + + /** Creates an assembler for the resolved topic schema. */ + public KafkaRowAssembler(KafkaTopicSchema topicSchema) { + this.topicSchema = checkNotNull(topicSchema); + } + + /** Assembles one Fluss row. */ + public GenericRow assemble( + Object[] keyValues, Object[] valueValues, long timestamp, List headers) { + GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); + setProjectedFields(row, topicSchema.keyProjection(), keyValues); + setProjectedFields(row, topicSchema.valueProjection(), valueValues); + if (topicSchema.timestampPosition() >= 0) { + row.setField(topicSchema.timestampPosition(), TimestampLtz.fromEpochMillis(timestamp)); + } + if (topicSchema.headersPosition() >= 0) { + row.setField(topicSchema.headersPosition(), toHeaders(headers)); + } + return row; + } + + private static void setProjectedFields( + GenericRow row, KafkaFieldProjection projection, Object[] values) { + if (values.length != projection.size()) { + throw new IllegalArgumentException( + "Kafka decoder returned " + + values.length + + " fields for a projection of " + + projection.size() + + "."); + } + for (int i = 0; i < values.length; i++) { + Object value = values[i]; + if (value == null && !projection.dataTypeAt(i).isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka record cannot populate NOT NULL field '" + + projection.nameAt(i) + + "' with null."); + } + row.setField(projection.positionAt(i), value); + } + } + + private static GenericArray toHeaders(List headers) { + Object[] rows = new Object[headers.size()]; + for (int i = 0; i < headers.size(); i++) { + RecordHeader header = headers.get(i); + rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.value()); + } + return new GenericArray(rows); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java new file mode 100644 index 00000000000..6936b5c8d61 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.CloseableIterator; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; + +/** Verifies mapped Arrow records after transcoding resources have closed. */ +class ArrowKafkaRecordTranscoderTest { + private final ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + + @Test + void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { + for (boolean string : new boolean[] {false, true}) { + TableInfo table = envelope(string); + Record record = + new Record( + 123L, + bytes("键"), + bytes("message"), + Arrays.asList( + new RecordHeader("duplicate", bytes("first")), + new RecordHeader("duplicate", null))); + BytesView encoded = transcoder.transcode(Arrays.asList(record, record), table); + read( + encoded, + table, + 2, + row -> { + if (string) { + assertThat(row.getString(0).toString()).isEqualTo("message"); + assertThat(row.getString(3).toString()).isEqualTo("键"); + } else { + assertThat(row.getBytes(0)).isEqualTo(bytes("message")); + assertThat(row.getBytes(3)).isEqualTo(bytes("键")); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + InternalArray headers = row.getArray(2); + assertThat(headers.size()).isEqualTo(2); + assertThat(headers.getRow(0, 2).getString(0).toString()) + .isEqualTo("duplicate"); + assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("first")); + assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); + }); + } + } + + @Test + void testNullsAndEmptyBytesRemainDistinct() throws Exception { + TableInfo table = envelope(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, null, null, Collections.emptyList())), + table), + table, + 1, + row -> { + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.isNullAt(3)).isTrue(); + assertThat(row.getArray(2).size()).isZero(); + }); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, new byte[0], new byte[0], Collections.emptyList())), + table), + table, + 1, + row -> { + assertThat(row.getBytes(0)).isEmpty(); + assertThat(row.getBytes(3)).isEmpty(); + }); + } + + @Test + void testUnmappedKeyIsIgnoredAndMixedFormatsWork() throws Exception { + TableInfo valueOnly = valueTable(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + new byte[] {(byte) 0xff}, + bytes("value"), + Collections.emptyList())), + valueOnly), + valueOnly, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("value")); + Schema schema = + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("id", DataTypes.STRING()) + .build(); + TableInfo mixed = + table( + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .build()); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + bytes("key"), + new byte[] {(byte) 0xff}, + Collections.emptyList())), + mixed), + mixed, + 1, + row -> { + assertThat(row.getString(1).toString()).isEqualTo("key"); + assertThat(row.getBytes(0)).containsExactly((byte) 0xff); + }); + } + + @Test + void testRejectsMalformedUtf8AndNotNullViolationsThenRemainsUsable() throws Exception { + TableInfo table = valueTable(false); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + new byte[] {(byte) 0xc3, 0x28}, + Collections.emptyList())), + table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("UTF-8"); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, null, null, Collections.emptyList())), + valueTable(true))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL"); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, null, bytes("ok"), Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("ok")); + } + + @Test + void testRejectsInvalidMappingAndEmptyPartition() { + TableInfo invalid = + table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING()) + .build()) + .distributedBy(1) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + bytes("value"), + Collections.emptyList())), + invalid)) + .isInstanceOf(KafkaTopicSchemaException.class); + assertThatThrownBy(() -> transcoder.transcode(Collections.emptyList(), valueTable(false))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("empty"); + } + + @Test + void testArrowWriterFailureClosesAllocatorWithoutLeaking() { + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encode( + Collections.singletonList(GenericRow.of(123)), + valueTable(false))); + assertThat(failure).isInstanceOf(ClassCastException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + + private static TableInfo valueTable(boolean notNull) { + return table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING().copy(!notNull)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .build()); + } + + private static TableInfo envelope(boolean string) { + Schema schema = + Schema.newBuilder() + .column("body", string ? DataTypes.STRING() : DataTypes.BYTES()) + .column("time", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "attrs", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES())))) + .column("id", string ? DataTypes.STRING() : DataTypes.BYTES()) + .build(); + return table( + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty( + KafkaDataFormat.VALUE_FORMAT_CONFIG, string ? "string" : "raw") + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, string ? "string" : "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "time") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attrs") + .build()); + } + + private static TableInfo table(TableDescriptor descriptor) { + return TableInfo.of(TablePath.of("kafka", "topic"), 42L, 3, descriptor, null, 1L, 1L); + } + + private static void read( + BytesView encoded, TableInfo table, int count, Consumer verify) + throws Exception { + ByteBuf buffer = encoded.getByteBuf(); + try { + LogRecordBatch batch = + MemoryLogRecords.pointToByteBuffer(buffer.nioBuffer()) + .batches() + .iterator() + .next(); + batch.ensureValid(); + assertThat(batch.schemaId()).isEqualTo((short) table.getSchemaId()); + assertThat(batch.getRecordCount()).isEqualTo(count); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + table.getRowType(), + table.getSchemaId(), + new TestingSchemaGetter( + new SchemaInfo( + table.getSchema(), table.getSchemaId()))); + CloseableIterator records = batch.records(context)) { + for (int i = 0; i < count; i++) { + assertThat(records.hasNext()).isTrue(); + verify.accept(records.next().getRow()); + } + assertThat(records.hasNext()).isFalse(); + } + } finally { + buffer.release(); + } + } + + private static byte[] bytes(String text) { + return text.getBytes(StandardCharsets.UTF_8); + } +} From 04a9093b331cfb1835493627ef0ab0adb969b8c2 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 17:19:52 +0800 Subject: [PATCH 11/11] [kafka] Stream record transcoding and reuse owned payloads Decode and append one record at a time instead of retaining an entire row list. Borrow command-owned key, value and header bytes during synchronous conversion while preserving defensive-copy accessors. Cover row and payload reuse, batches beyond initial Arrow capacity, and resource cleanup after partial conversion failures. Validation on Java 11: reactor clean verify passed with 130 unit tests and 4 integration tests, plus Checkstyle, Spotless, RAT and git diff --check. An additional 25 boundary scenarios passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 101/101 AI-Contributed/UT: 224/224 --- .../backend/produce/KafkaProduceCommand.java | 30 ++++ .../transcode/ArrowKafkaRecordTranscoder.java | 36 +++-- .../transcode/FlussArrowRecordEncoder.java | 26 +++- .../kafka/transcode/KafkaRowAssembler.java | 9 +- .../produce/KafkaProduceCommandTest.java | 80 ++++++++++ .../ArrowKafkaRecordTranscoderTest.java | 144 ++++++++++++++++++ 6 files changed, 304 insertions(+), 21 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java index 722f3d3d245..52157c6d4ef 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -157,11 +157,31 @@ public long timestamp() { return copyNullable(key); } + /** + * Borrows the nullable key owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #key()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedKey() { + return key; + } + /** Returns a copy of the nullable Kafka record value. */ public @Nullable byte[] value() { return copyNullable(value); } + /** + * Borrows the nullable value owned by this record for synchronous conversion. + * + *

The returned array must not be modified or retained after conversion. Use {@link + * #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } + /** Returns the copied Kafka headers in record order. */ public List headers() { return headers; @@ -193,5 +213,15 @@ public String name() { public @Nullable byte[] value() { return value == null ? null : value.clone(); } + + /** + * Borrows the nullable header value for synchronous conversion. + * + *

The returned array is owned by this header and must not be modified or retained after + * conversion. Use {@link #value()} when an independently owned copy is needed. + */ + public @Nullable byte[] borrowedValue() { + return value; + } } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java index 717f5cd33ff..9fcaaaabf31 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -25,7 +25,6 @@ import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.record.bytesview.BytesView; import org.apache.fluss.row.BinaryString; -import org.apache.fluss.row.GenericRow; import javax.annotation.Nullable; @@ -33,7 +32,6 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; import static org.apache.fluss.utils.Preconditions.checkArgument; @@ -49,20 +47,26 @@ public BytesView transcode(List records, TableInfo tableInfo) throws Exc checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); KafkaRowAssembler assembler = new KafkaRowAssembler(schema); - List rows = new ArrayList<>(records.size()); - for (Record record : records) { - Object[] key = - schema.keyFormat() == null - ? new Object[0] - : new Object[] {decode(schema.keyFormat(), record.key())}; - rows.add( - assembler.assemble( - key, - new Object[] {decode(schema.valueFormat(), record.value())}, - record.timestamp(), - record.headers())); - } - return arrowRecordEncoder.encode(rows, tableInfo); + return arrowRecordEncoder.encodeStreaming( + consumer -> { + for (Record record : records) { + Object[] key = + schema.keyFormat() == null + ? new Object[0] + : new Object[] { + decode(schema.keyFormat(), record.borrowedKey()) + }; + consumer.append( + assembler.assemble( + key, + new Object[] { + decode(schema.valueFormat(), record.borrowedValue()) + }, + record.timestamp(), + record.headers())); + } + }, + tableInfo); } private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java index f74680ccaa2..c84dd20d690 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java @@ -39,6 +39,16 @@ public final class FlussArrowRecordEncoder { /** Encodes all rows using the table's current schema ID and Arrow compression settings. */ public BytesView encode(List rows, TableInfo tableInfo) throws Exception { + return encodeStreaming( + consumer -> { + for (GenericRow row : rows) { + consumer.append(row); + } + }, + tableInfo); + } + + BytesView encodeStreaming(RowSource rowSource, TableInfo tableInfo) throws Exception { try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); ArrowWriterPool provider = new ArrowWriterPool(allocator)) { ArrowWriter writer = @@ -57,9 +67,7 @@ public BytesView encode(List rows, TableInfo tableInfo) throws Excep new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), true, null); - for (GenericRow row : rows) { - builder.append(ChangeType.APPEND_ONLY, row); - } + rowSource.produce(row -> builder.append(ChangeType.APPEND_ONLY, row)); // The output view owns heap pages. The result remains valid after the Arrow // writer and allocator close, including across asynchronous append completion. return builder.build(); @@ -68,4 +76,16 @@ public BytesView encode(List rows, TableInfo tableInfo) throws Excep } } } + + @FunctionalInterface + interface RowSource { + /** Produces rows synchronously and never retains the supplied consumer. */ + void produce(RowConsumer consumer) throws Exception; + } + + @FunctionalInterface + interface RowConsumer { + /** Serializes one row before returning and never retains the row. */ + void append(GenericRow row) throws Exception; + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java index 4ebd506dc3c..cac8ac3e841 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -41,7 +41,12 @@ public KafkaRowAssembler(KafkaTopicSchema topicSchema) { this.topicSchema = checkNotNull(topicSchema); } - /** Assembles one Fluss row. */ + /** + * Assembles one Fluss row for immediate synchronous encoding. + * + *

The row borrows decoded values and header payloads; it must not modify them or retain them + * after conversion. + */ public GenericRow assemble( Object[] keyValues, Object[] valueValues, long timestamp, List headers) { GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); @@ -82,7 +87,7 @@ private static GenericArray toHeaders(List headers) { Object[] rows = new Object[headers.size()]; for (int i = 0; i < headers.size(); i++) { RecordHeader header = headers.get(i); - rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.value()); + rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.borrowedValue()); } return new GenericArray(rows); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java new file mode 100644 index 00000000000..09b161d5f24 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommandTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies borrowed payloads preserve command ownership and defensive copy accessors. */ +class KafkaProduceCommandTest { + + @Test + void testBorrowedPayloadsReuseOwnedArraysWhileCopyAccessorsRemainDefensive() { + byte[] key = {1}; + byte[] value = {2}; + byte[] headerValue = {3}; + RecordHeader header = new RecordHeader("header", headerValue); + List headers = new ArrayList<>(); + headers.add(header); + Record record = new Record(1L, key, value, headers); + + assertThat(record.borrowedKey()).isNotSameAs(key); + assertThat(record.borrowedValue()).isNotSameAs(value); + assertThat(header.borrowedValue()).isNotSameAs(headerValue); + key[0] = 10; + value[0] = 20; + headerValue[0] = 30; + headers.clear(); + + assertThat(record.borrowedKey()).containsExactly((byte) 1); + assertThat(record.borrowedValue()).containsExactly((byte) 2); + assertThat(header.borrowedValue()).containsExactly((byte) 3); + assertThat(record.headers()).containsExactly(header); + assertThat(record.borrowedKey()).isSameAs(record.borrowedKey()); + assertThat(record.borrowedValue()).isSameAs(record.borrowedValue()); + assertThat(header.borrowedValue()).isSameAs(header.borrowedValue()); + + record.key()[0] = 40; + record.value()[0] = 50; + header.value()[0] = 60; + assertThat(record.key()).containsExactly((byte) 1).isNotSameAs(record.borrowedKey()); + assertThat(record.value()).containsExactly((byte) 2).isNotSameAs(record.borrowedValue()); + assertThat(header.value()).containsExactly((byte) 3).isNotSameAs(header.borrowedValue()); + } + + @Test + void testBorrowedPayloadsPreserveNullAndEmptyBytes() { + Record nullRecord = new Record(1L, null, null, Collections.emptyList()); + assertThat(nullRecord.borrowedKey()).isNull(); + assertThat(nullRecord.borrowedValue()).isNull(); + assertThat(new RecordHeader("header", null).borrowedValue()).isNull(); + + Record emptyRecord = new Record(1L, new byte[0], new byte[0], Collections.emptyList()); + assertThat(emptyRecord.borrowedKey()).isEmpty(); + assertThat(emptyRecord.borrowedValue()).isEmpty(); + assertThat(new RecordHeader("header", new byte[0]).borrowedValue()).isEmpty(); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java index 6936b5c8d61..f1df39504d2 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -33,18 +33,27 @@ import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.record.TestingSchemaGetter; import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; import org.apache.fluss.row.GenericRow; import org.apache.fluss.row.InternalArray; import org.apache.fluss.row.InternalRow; +import org.apache.fluss.row.TimestampLtz; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.CloseableIterator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; @@ -242,6 +251,141 @@ void testArrowWriterFailureClosesAllocatorWithoutLeaking() { assertThat(failure.getSuppressed()).isEmpty(); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testStreamingBatchPreservesDistinctRecordsBeyondInitialCapacity(boolean string) + throws Exception { + TableInfo table = envelope(string); + List records = new ArrayList<>(); + for (int i = 0; i < 2050; i++) { + records.add( + new Record( + i, + bytes("key-" + i), + bytes("消息-" + i), + Arrays.asList( + new RecordHeader("duplicate", bytes("header-" + i)), + new RecordHeader("duplicate", null)))); + } + BytesView encoded = transcoder.transcode(records, table); + AtomicInteger nextRecord = new AtomicInteger(); + read( + encoded, + table, + records.size(), + row -> { + int i = nextRecord.getAndIncrement(); + if (string) { + assertThat(row.getString(0).toString()).isEqualTo("消息-" + i); + assertThat(row.getString(3).toString()).isEqualTo("key-" + i); + } else { + assertThat(row.getBytes(0)).isEqualTo(bytes("消息-" + i)); + assertThat(row.getBytes(3)).isEqualTo(bytes("key-" + i)); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + InternalArray headers = row.getArray(2); + assertThat(headers.size()).isEqualTo(2); + assertThat(headers.getRow(0, 2).getString(0).toString()).isEqualTo("duplicate"); + assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("header-" + i)); + assertThat(headers.getRow(1, 2).getString(0).toString()).isEqualTo("duplicate"); + assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); + }); + } + + @Test + void testStreamingEncoderCopiesRowAndPayloadBeforeSourceReusesThem() throws Exception { + TableInfo table = envelope(false); + byte[] key = new byte[1]; + byte[] value = new byte[1]; + byte[] headerValue = new byte[1]; + GenericRow row = + GenericRow.of( + value, + TimestampLtz.fromEpochMillis(1L), + new GenericArray( + new Object[] { + GenericRow.of(BinaryString.fromString("header"), headerValue) + }), + key); + BytesView encoded = + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + for (int i = 0; i < 3; i++) { + key[0] = (byte) i; + value[0] = (byte) (i + 10); + headerValue[0] = (byte) (i + 20); + row.setField(1, TimestampLtz.fromEpochMillis(i)); + consumer.append(row); + } + Arrays.fill(key, (byte) -1); + Arrays.fill(value, (byte) -1); + Arrays.fill(headerValue, (byte) -1); + row.setField(0, null); + }, + table); + AtomicInteger nextRecord = new AtomicInteger(); + read( + encoded, + table, + 3, + actual -> { + int i = nextRecord.getAndIncrement(); + assertThat(actual.getBytes(3)).containsExactly((byte) i); + assertThat(actual.getBytes(0)).containsExactly((byte) (i + 10)); + assertThat(actual.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(i); + assertThat(actual.getArray(2).getRow(0, 2).getBytes(1)) + .containsExactly((byte) (i + 20)); + }); + } + + @Test + void testDecodeFailureAfterValidRecordClosesAllocatorAndRemainsUsable() throws Exception { + TableInfo table = valueTable(true); + Record valid = new Record(1L, null, bytes("valid"), Collections.emptyList()); + for (byte[] invalidValue : new byte[][] {null, {(byte) 0xc3, 0x28}}) { + Throwable failure = + catchThrowable( + () -> + transcoder.transcode( + Arrays.asList( + valid, + new Record( + 2L, + null, + invalidValue, + Collections.emptyList())), + table)); + assertThat(failure).isInstanceOf(KafkaRecordEncodingException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + read( + transcoder.transcode(Collections.singletonList(valid), table), + table, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("valid")); + } + + @Test + void testStreamingSourceFailureClosesAllocator() { + IOException expected = new IOException("source failed after the first row"); + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encodeStreaming( + consumer -> { + consumer.append( + GenericRow.of( + BinaryString.fromString( + "valid"))); + throw expected; + }, + valueTable(false))); + assertThat(failure).isSameAs(expected); + assertThat(failure.getSuppressed()).isEmpty(); + } + private static TableInfo valueTable(boolean notNull) { return table( TableDescriptor.builder()