From 49b95cef3e0e876cff3e6ef9683d0a55bfcaa25a Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 8 Sep 2026 11:29:44 +0800 Subject: [PATCH 01/10] [kafka] Add request dispatch and transport framework Introduce API registration, request context, version validation, and asynchronous error mapping. Fix request buffer ownership and response serialization cleanup while preserving the existing ApiVersions entry point. Validated with mvn -o -pl fluss-rpc,fluss-kafka verify. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 555/555 AI-Contributed/UT: 525/525 --- .../fluss/kafka/KafkaChannelInitializer.java | 5 +- .../fluss/kafka/KafkaCommandDecoder.java | 45 +++-- .../fluss/kafka/KafkaProtocolPlugin.java | 1 + .../org/apache/fluss/kafka/KafkaRequest.java | 43 ++++- .../fluss/kafka/KafkaRequestContext.java | 96 ++++++++++ .../kafka/dispatcher/KafkaApiHandler.java | 37 ++++ .../kafka/dispatcher/KafkaApiRegistry.java | 80 +++++++++ .../fluss/kafka/dispatcher/KafkaApiSpec.java | 82 +++++++++ .../dispatcher/KafkaRequestDispatcher.java | 108 ++++++++++++ .../fluss/kafka/error/KafkaErrorMapper.java | 45 +++++ .../fluss/kafka/KafkaCommandDecoderTest.java | 118 +++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 56 ++++-- .../apache/fluss/kafka/KafkaRequestTest.java | 58 ++++++ .../dispatcher/KafkaApiRegistryTest.java | 127 ++++++++++++++ .../KafkaRequestDispatcherTest.java | 166 ++++++++++++++++++ .../fluss/rpc/netty/server/NettyServer.java | 13 +- 16 files changed, 1039 insertions(+), 41 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java index 5e7551a9af7..29bdc745ca9 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaChannelInitializer.java @@ -31,17 +31,20 @@ public class KafkaChannelInitializer extends NettyChannelInitializer { private final RequestChannel[] requestChannels; + private final String listenerName; private final int maxRequestSize; private final LengthFieldPrepender prepender = new LengthFieldPrepender(4); private final boolean preferHeap; public KafkaChannelInitializer( RequestChannel[] requestChannels, + String listenerName, long maxIdleTimeSeconds, int maxRequestSize, boolean preferHeap) { super(maxIdleTimeSeconds); this.requestChannels = requestChannels; + this.listenerName = listenerName; this.maxRequestSize = maxRequestSize; this.preferHeap = preferHeap; } @@ -53,6 +56,6 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast(prepender); addFrameDecoder(ch, maxRequestSize, 4, preferHeap); ch.pipeline().addLast("flowController", new FlowControlHandler()); - ch.pipeline().addLast(new KafkaCommandDecoder(requestChannels)); + ch.pipeline().addLast(new KafkaCommandDecoder(requestChannels, listenerName)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java index 43a0533b2d3..637a1aaa1fe 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaCommandDecoder.java @@ -27,6 +27,7 @@ import org.apache.fluss.utils.MathUtils; import org.apache.kafka.common.errors.LeaderNotAvailableException; +import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; @@ -55,6 +56,7 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler { private final RequestChannel[] requestChannels; private final int numChannels; + private final String listenerName; // Need to use a Queue to store the inflight responses, because Kafka clients require the // responses to be sent in order. @@ -65,18 +67,18 @@ public class KafkaCommandDecoder extends SimpleChannelInboundHandler { protected volatile ChannelHandlerContext ctx; protected SocketAddress remoteAddress; - public KafkaCommandDecoder(RequestChannel[] requestChannels) { + public KafkaCommandDecoder(RequestChannel[] requestChannels, String listenerName) { super(false); this.requestChannels = requestChannels; this.numChannels = requestChannels.length; + this.listenerName = listenerName; } @Override public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Exception { CompletableFuture future = new CompletableFuture<>(); - boolean needRelease = false; try { - KafkaRequest request = parseRequest(ctx, future, buffer); + KafkaRequest request = parseRequest(ctx, future, buffer, listenerName); inflightResponses.addLast(request); future.whenCompleteAsync((r, t) -> sendResponse(ctx), ctx.executor()); int channelIndex = @@ -86,16 +88,15 @@ public void channelRead0(ChannelHandlerContext ctx, ByteBuf buffer) throws Excep if (!isActive.get()) { LOG.warn("Received a request on an inactive channel: {}", remoteAddress); request.fail(new LeaderNotAvailableException("Channel is inactive")); - needRelease = true; } } catch (Throwable t) { - needRelease = true; LOG.error("Error handling request", t); future.completeExceptionally(t); } finally { - if (needRelease) { - ReferenceCountUtil.release(buffer); - } + // KafkaRequest retains the buffer to transfer ownership to request processing. Release + // the decoder's ownership on every path. KafkaRequest.releaseBuffer() is idempotent + // because worker cleanup and response completion can both release that ownership. + ReferenceCountUtil.release(buffer); } } @@ -184,19 +185,39 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E } private static KafkaRequest parseRequest( - ChannelHandlerContext ctx, CompletableFuture future, ByteBuf buffer) { + ChannelHandlerContext ctx, + CompletableFuture future, + ByteBuf buffer, + String listenerName) { ByteBuffer nioBuffer = buffer.nioBuffer(); RequestHeader header = RequestHeader.parse(nioBuffer); if (isUnsupportedApiVersionRequest(header)) { ApiVersionsRequest request = - new ApiVersionsRequest.Builder(header.apiVersion()).build(); + new ApiVersionsRequest( + new ApiVersionsRequestData(), + API_VERSIONS.oldestVersion(), + header.apiVersion()); return new KafkaRequest( - API_VERSIONS, header.apiVersion(), header, request, buffer, ctx, future); + API_VERSIONS, + header.apiVersion(), + header, + request, + listenerName, + buffer, + ctx, + future); } RequestAndSize request = AbstractRequest.parseRequest(header.apiKey(), header.apiVersion(), nioBuffer); return new KafkaRequest( - header.apiKey(), header.apiVersion(), header, request.request, buffer, ctx, future); + header.apiKey(), + header.apiVersion(), + header, + request.request, + listenerName, + buffer, + ctx, + future); } private static boolean isUnsupportedApiVersionRequest(RequestHeader header) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index d92ba5e68fc..c966f745b8c 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -53,6 +53,7 @@ public ChannelHandler createChannelHandler( RequestChannel[] requestChannels, String listenerName) { return new KafkaChannelInitializer( requestChannels, + listenerName, conf.get(ConfigOptions.KAFKA_CONNECTION_MAX_IDLE_TIME).getSeconds(), (int) conf.get(ConfigOptions.NETTY_SERVER_MAX_REQUEST_SIZE).getBytes(), conf.getBoolean(ConfigOptions.NETTY_CLIENT_ALLOCATOR_HEAP_BUFFER_FIRST)); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java index 25e409a7455..0d2799a7a18 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequest.java @@ -35,6 +35,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; /** Represents a request received from Kafka protocol channel. */ @@ -46,10 +47,12 @@ public class KafkaRequest implements RpcRequest { private final long requestId = ID_GENERATOR.getAndIncrement(); private final RequestHeader header; private final AbstractRequest request; + private final String listenerName; private final ByteBuf buffer; private final ChannelHandlerContext ctx; private final long startTimeMs; private final CompletableFuture future; + private final AtomicBoolean bufferReleased = new AtomicBoolean(); private volatile boolean cancelled = false; protected KafkaRequest( @@ -60,10 +63,23 @@ protected KafkaRequest( ByteBuf buffer, ChannelHandlerContext ctx, CompletableFuture future) { + this(apiKey, apiVersion, header, request, "UNKNOWN", buffer, ctx, future); + } + + protected KafkaRequest( + ApiKeys apiKey, + short apiVersion, + RequestHeader header, + AbstractRequest request, + String listenerName, + ByteBuf buffer, + ChannelHandlerContext ctx, + CompletableFuture future) { this.apiKey = apiKey; this.apiVersion = apiVersion; this.header = header; this.request = request; + this.listenerName = listenerName; this.buffer = buffer.retain(); this.ctx = ctx; this.startTimeMs = System.currentTimeMillis(); @@ -77,7 +93,9 @@ public RequestType getRequestType() { @Override public void releaseBuffer() { - ReferenceCountUtil.safeRelease(buffer); + if (bufferReleased.compareAndSet(false, true)) { + ReferenceCountUtil.safeRelease(buffer); + } } public ApiKeys apiKey() { @@ -100,6 +118,10 @@ public T request() { return (T) request; } + public String listenerName() { + return listenerName; + } + public ChannelHandlerContext ctx() { return ctx; } @@ -149,12 +171,17 @@ private ByteBuf serialize(AbstractResponse response) { int headerSize = headerData.size(cache, headerVersion); ApiMessage apiMessage = response.data(); int messageSize = apiMessage.size(cache, apiVersion); - final ByteBuf buffer = ctx.alloc().buffer(headerSize + messageSize); - buffer.writerIndex(headerSize + messageSize); - final ByteBuffer nioBuffer = buffer.nioBuffer(); - final ByteBufferAccessor writable = new ByteBufferAccessor(nioBuffer); - headerData.write(writable, cache, headerVersion); - apiMessage.write(writable, cache, apiVersion); - return buffer; + final ByteBuf responseBuffer = ctx.alloc().buffer(headerSize + messageSize); + try { + responseBuffer.writerIndex(headerSize + messageSize); + final ByteBuffer nioBuffer = responseBuffer.nioBuffer(); + final ByteBufferAccessor writable = new ByteBufferAccessor(nioBuffer); + headerData.write(writable, cache, headerVersion); + apiMessage.write(writable, cache, apiVersion); + return responseBuffer; + } catch (Throwable t) { + ReferenceCountUtil.safeRelease(responseBuffer); + throw t; + } } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java new file mode 100644 index 00000000000..e75a20babcc --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestContext.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.shaded.netty4.io.netty.channel.Channel; + +import org.apache.kafka.common.protocol.ApiKeys; + +import java.net.SocketAddress; + +/** Immutable wire-level context made available to Kafka API handlers. */ +@Internal +public final class KafkaRequestContext { + + private final int correlationId; + private final String clientId; + private final ApiKeys apiKey; + private final short apiVersion; + private final String listenerName; + private final SocketAddress localAddress; + private final SocketAddress remoteAddress; + private final long receivedTimeMs; + + private KafkaRequestContext(KafkaRequest request) { + this.correlationId = request.header().correlationId(); + this.clientId = request.header().clientId(); + this.apiKey = request.apiKey(); + this.apiVersion = request.apiVersion(); + this.listenerName = request.listenerName(); + Channel channel = request.ctx().channel(); + this.localAddress = channel == null ? null : channel.localAddress(); + this.remoteAddress = channel == null ? null : channel.remoteAddress(); + this.receivedTimeMs = request.startTimeMs(); + } + + /** Creates a context from a network request. */ + public static KafkaRequestContext fromRequest(KafkaRequest request) { + return new KafkaRequestContext(request); + } + + /** Returns the request correlation ID. */ + public int correlationId() { + return correlationId; + } + + /** Returns the client ID, or {@code null} when the request did not provide one. */ + public String clientId() { + return clientId; + } + + /** Returns the Kafka API key. */ + public ApiKeys apiKey() { + return apiKey; + } + + /** Returns the Kafka request version. */ + public short apiVersion() { + return apiVersion; + } + + /** Returns the listener that accepted the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the local socket address. */ + public SocketAddress localAddress() { + return localAddress; + } + + /** Returns the remote socket address. */ + public SocketAddress remoteAddress() { + return remoteAddress; + } + + /** Returns the wall-clock time at which the request was received. */ + public long receivedTimeMs() { + return receivedTimeMs; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java new file mode 100644 index 00000000000..36995b8e27f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiHandler.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; + +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; + +import java.util.concurrent.CompletableFuture; + +/** Handles one Kafka API without blocking the request processor thread. */ +@Internal +public interface KafkaApiHandler { + + /** Returns the capability implemented by this handler. */ + KafkaApiSpec apiSpec(); + + /** Handles a parsed request asynchronously. */ + CompletableFuture handle(KafkaRequestContext context, R request); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java new file mode 100644 index 00000000000..b4a1dfd8ea3 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistry.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.ApiKeys; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkState; + +/** Registry and single source of truth for Kafka APIs exposed by one server. */ +@Internal +public final class KafkaApiRegistry { + + private final Map> handlers = new HashMap<>(); + private boolean frozen; + + /** Creates an empty API registry. */ + public KafkaApiRegistry() {} + + /** Registers a handler. Registrations are rejected after {@link #freeze()} is called. */ + public void register(KafkaApiHandler handler) { + checkNotNull(handler); + checkState(!frozen, "Kafka API registry is already frozen."); + ApiKeys apiKey = handler.apiSpec().apiKey(); + checkArgument(!handlers.containsKey(apiKey), "Kafka API %s is already registered.", apiKey); + handlers.put(apiKey, handler); + } + + /** Prevents further registrations. */ + public void freeze() { + frozen = true; + } + + /** Returns a routable handler, or {@code null} when the API is not exposed by this server. */ + public KafkaApiHandler lookup(ApiKeys apiKey) { + KafkaApiHandler handler = handlers.get(apiKey); + if (handler == null || !handler.apiSpec().advertised()) { + return null; + } + return handler; + } + + /** Returns the sorted API specifications advertised by this server. */ + public List advertisedApiSpecs() { + List specs = new ArrayList<>(); + for (KafkaApiHandler handler : handlers.values()) { + KafkaApiSpec spec = handler.apiSpec(); + if (spec.advertised()) { + specs.add(spec); + } + } + Collections.sort(specs, Comparator.comparingInt(spec -> spec.apiKey().id)); + return Collections.unmodifiableList(specs); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java new file mode 100644 index 00000000000..50d6a7ebab2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaApiSpec.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.ApiKeys; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Describes the versions actually supported by a Kafka API handler. */ +@Internal +public final class KafkaApiSpec { + + private final ApiKeys apiKey; + private final short minVersion; + private final short maxVersion; + private final boolean advertised; + + /** Creates an API specification. */ + public KafkaApiSpec(ApiKeys apiKey, short minVersion, short maxVersion, boolean advertised) { + this.apiKey = checkNotNull(apiKey); + checkArgument(minVersion >= 0, "Minimum version must not be negative."); + checkArgument( + minVersion <= maxVersion, + "Minimum version %s must not exceed maximum version %s.", + minVersion, + maxVersion); + checkArgument( + minVersion >= apiKey.oldestVersion() && maxVersion <= apiKey.latestVersion(), + "Version range [%s, %s] is outside the Kafka library range [%s, %s] for %s.", + minVersion, + maxVersion, + apiKey.oldestVersion(), + apiKey.latestVersion(), + apiKey); + this.minVersion = minVersion; + this.maxVersion = maxVersion; + this.advertised = advertised; + } + + /** Returns the Kafka API key. */ + public ApiKeys apiKey() { + return apiKey; + } + + /** Returns the oldest supported request version. */ + public short minVersion() { + return minVersion; + } + + /** Returns the newest supported request version. */ + public short maxVersion() { + return maxVersion; + } + + /** Returns whether this API is allowed to be routed and advertised. */ + public boolean advertised() { + return advertised; + } + + /** Returns whether the supplied request version is supported. */ + public boolean supportsVersion(short version) { + return version >= minVersion && version <= maxVersion; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java new file mode 100644 index 00000000000..efdfe33dd66 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcher.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequest; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.error.KafkaErrorMapper; + +import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Validates and dispatches parsed Kafka requests to independently registered API handlers. */ +@Internal +public final class KafkaRequestDispatcher { + + private final KafkaApiRegistry registry; + private final KafkaErrorMapper errorMapper; + + /** Creates a dispatcher backed by the supplied registry and error mapper. */ + public KafkaRequestDispatcher(KafkaApiRegistry registry, KafkaErrorMapper errorMapper) { + this.registry = checkNotNull(registry); + this.errorMapper = checkNotNull(errorMapper); + } + + /** Dispatches a request and always completes with a Kafka protocol response. */ + public CompletableFuture dispatch(KafkaRequest request) { + AbstractRequest abstractRequest = request.request(); + KafkaApiHandler handler = registry.lookup(request.apiKey()); + if (handler == null) { + return completedErrorResponse( + abstractRequest, + new UnsupportedVersionException( + "Kafka API " + request.apiKey() + " is not supported by this server.")); + } + + KafkaApiSpec spec = handler.apiSpec(); + if (!spec.supportsVersion(request.apiVersion())) { + return completedErrorResponse( + abstractRequest, + new UnsupportedVersionException( + String.format( + "Version %s is not supported for %s. Supported versions are [%s, %s].", + request.apiVersion(), + request.apiKey(), + spec.minVersion(), + spec.maxVersion()))); + } + + CompletableFuture responseFuture; + try { + responseFuture = + invoke(handler, KafkaRequestContext.fromRequest(request), abstractRequest); + if (responseFuture == null) { + throw new NullPointerException("Kafka API handler returned a null future."); + } + } catch (Throwable t) { + return completedErrorResponse(abstractRequest, t); + } + + CompletableFuture result = new CompletableFuture<>(); + responseFuture.whenComplete( + (response, failure) -> { + if (failure == null && response != null) { + result.complete(response); + } else { + Throwable responseFailure = + failure == null + ? new NullPointerException( + "Kafka API handler returned a null response.") + : failure; + result.complete(errorMapper.toResponse(abstractRequest, responseFailure)); + } + }); + return result; + } + + @SuppressWarnings("unchecked") + private static CompletableFuture invoke( + KafkaApiHandler handler, KafkaRequestContext context, AbstractRequest request) { + return ((KafkaApiHandler) handler).handle(context, request); + } + + private CompletableFuture completedErrorResponse( + AbstractRequest request, Throwable failure) { + return CompletableFuture.completedFuture(errorMapper.toResponse(request, failure)); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java new file mode 100644 index 00000000000..4396566396d --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/error/KafkaErrorMapper.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.error; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; + +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; + +/** Maps failures from the compatibility layer to version-aware Kafka responses. */ +@Internal +public final class KafkaErrorMapper { + + /** Converts a failure to the error response defined by the parsed Kafka request. */ + public AbstractResponse toResponse(AbstractRequest request, Throwable failure) { + return request.getErrorResponse(unwrap(failure)); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while ((current instanceof CompletionException || current instanceof ExecutionException) + && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java new file mode 100644 index 00000000000..a2b8a0dc60b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; + +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests response ordering and ownership in {@link KafkaCommandDecoder}. */ +public class KafkaCommandDecoderTest { + + @Test + public void testAcksZeroSuppressesResponseAndUnblocksFollowingResponse() { + RequestChannel requestChannel = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requestChannel}, "KAFKA")); + short produceVersion = ApiKeys.PRODUCE.latestVersion(); + ProduceRequest produceRequest = + new ProduceRequest( + new ProduceRequestData().setAcks((short) 0).setTimeoutMs(1000), + produceVersion); + RequestHeader produceHeader = + new RequestHeader(ApiKeys.PRODUCE, produceVersion, "client", 1); + ByteBuf produceBuffer = serialize(produceHeader, produceRequest); + + short apiVersionsVersion = ApiKeys.API_VERSIONS.latestVersion(); + ApiVersionsRequest apiVersionsRequest = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData(), + apiVersionsVersion, + apiVersionsVersion) + .build(); + RequestHeader apiVersionsHeader = + new RequestHeader(ApiKeys.API_VERSIONS, apiVersionsVersion, "client", 2); + ByteBuf apiVersionsBuffer = serialize(apiVersionsHeader, apiVersionsRequest); + + try { + channel.writeInbound(produceBuffer); + channel.writeInbound(apiVersionsBuffer); + KafkaRequest first = (KafkaRequest) requestChannel.pollRequest(1000); + KafkaRequest second = (KafkaRequest) requestChannel.pollRequest(1000); + assertThat(first).isNotNull(); + assertThat(second).isNotNull(); + + second.complete(new ApiVersionsResponse(new ApiVersionsResponseData())); + channel.runPendingTasks(); + Object blockedResponse = channel.readOutbound(); + assertThat(blockedResponse).isNull(); + + first.complete(new ProduceResponse(new ProduceResponseData())); + channel.runPendingTasks(); + + ByteBuf response = channel.readOutbound(); + try { + assertThat(response).isNotNull(); + ResponseHeader responseHeader = + ResponseHeader.parse( + response.nioBuffer(), + apiVersionsHeader.toResponseHeader().headerVersion()); + assertThat(responseHeader.correlationId()).isEqualTo(2); + Object additionalResponse = channel.readOutbound(); + assertThat(additionalResponse).isNull(); + } finally { + if (response != null) { + response.release(); + } + } + + assertThat(produceBuffer.refCnt()).isZero(); + assertThat(apiVersionsBuffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + + private static ByteBuf serialize(RequestHeader header, AbstractRequest request) { + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + return Unpooled.wrappedBuffer(serialized); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 24e4ce8a6ce..8613d83df32 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; @@ -46,21 +47,15 @@ public void testKafkaApiVersionsNotSupported() { new ApiVersionsRequest.Builder().build(latestVersion); ChannelHandlerContext ctx = new TestingChannelHandlerContext(); KafkaRequest request = - new KafkaRequest( + newRequest( ApiKeys.API_VERSIONS, (short) (latestVersion + 1), // unsupported version new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), apiVersionsRequest, - ByteBufAllocator.DEFAULT.buffer(), - ctx, - new CompletableFuture<>()); + ctx); handler.handleApiVersionsRequest(request); - ByteBuf responseBuffer = request.responseBuffer(); - ApiVersionsResponse response = - (ApiVersionsResponse) - AbstractResponse.parseResponse( - responseBuffer.nioBuffer(), request.header()); + ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); Map errorCounts = response.errorCounts(); assertThat(1).isEqualTo(errorCounts.size()); assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); @@ -74,21 +69,15 @@ public void testKafkaApiVersionsRequest() { new ApiVersionsRequest.Builder().build(latestVersion); ChannelHandlerContext ctx = new TestingChannelHandlerContext(); KafkaRequest request = - new KafkaRequest( + newRequest( ApiKeys.API_VERSIONS, latestVersion, new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), apiVersionsRequest, - ByteBufAllocator.DEFAULT.buffer(), - ctx, - new CompletableFuture<>()); + ctx); handler.handleApiVersionsRequest(request); - ByteBuf responseBuffer = request.responseBuffer(); - ApiVersionsResponse response = - (ApiVersionsResponse) - AbstractResponse.parseResponse( - responseBuffer.nioBuffer(), request.header()); + ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); Map errorCounts = response.errorCounts(); assertThat(1).isEqualTo(errorCounts.size()); assertThat(1).isEqualTo(errorCounts.get(Errors.NONE)); @@ -112,6 +101,37 @@ public void testKafkaApiVersionsRequest() { }); } + private static KafkaRequest newRequest( + ApiKeys apiKey, + short apiVersion, + RequestHeader header, + AbstractRequest requestBody, + ChannelHandlerContext context) { + ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); + try { + return new KafkaRequest( + apiKey, + apiVersion, + header, + requestBody, + requestBuffer, + context, + new CompletableFuture<>()); + } finally { + // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest. + requestBuffer.release(); + } + } + + private static AbstractResponse parseResponse(KafkaRequest request) { + ByteBuf responseBuffer = request.responseBuffer(); + try { + return AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + } + } + private static KafkaRequestHandler createKafkaRequestHandler() { return new KafkaRequestHandler(new TestingTabletGatewayService()); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java new file mode 100644 index 00000000000..2aa9caf9ba9 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; + +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Tests for {@link KafkaRequest}. */ +public class KafkaRequestTest { + + @Test + public void testReleaseBufferIsIdempotent() { + short version = ApiKeys.API_VERSIONS.oldestVersion(); + ByteBuf buffer = mock(ByteBuf.class); + when(buffer.retain()).thenReturn(buffer); + KafkaRequest request = + new KafkaRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 1), + new ApiVersionsRequest.Builder().build(version), + buffer, + mock(ChannelHandlerContext.class), + new CompletableFuture<>()); + + request.releaseBuffer(); + request.releaseBuffer(); + + verify(buffer).retain(); + verify(buffer).release(); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java new file mode 100644 index 00000000000..84a889c16bd --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaApiRegistryTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.kafka.KafkaRequestContext; + +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link KafkaApiRegistry}. */ +public class KafkaApiRegistryTest { + + @Test + public void testRejectDuplicateRegistrationAndRegistrationAfterFreeze() { + KafkaApiRegistry registry = brokerRegistry(); + TestingApiVersionsHandler handler = new TestingApiVersionsHandler(true); + registry.register(handler); + + assertThatThrownBy(() -> registry.register(handler)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("already registered"); + + registry.freeze(); + assertThatThrownBy(() -> registry.register(handler)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already frozen"); + } + + @Test + public void testOnlyAdvertiseEnabledHandlers() { + KafkaApiRegistry registry = brokerRegistry(); + registry.register(new TestingApiVersionsHandler(true)); + assertThat(registry.advertisedApiSpecs()).hasSize(1); + + KafkaApiRegistry hiddenRegistry = brokerRegistry(); + hiddenRegistry.register(new TestingApiVersionsHandler(false)); + assertThat(hiddenRegistry.advertisedApiSpecs()).isEmpty(); + assertThat(hiddenRegistry.lookup(ApiKeys.API_VERSIONS)).isNull(); + } + + @Test + public void testAdvertisedSpecIsSameSpecUsedForRouting() { + KafkaApiRegistry registry = brokerRegistry(); + TestingApiVersionsHandler handler = new TestingApiVersionsHandler(true); + registry.register(handler); + registry.freeze(); + + KafkaApiSpec advertisedSpec = registry.advertisedApiSpecs().get(0); + KafkaApiHandler routedHandler = registry.lookup(ApiKeys.API_VERSIONS); + + assertThat(routedHandler).isSameAs(handler); + assertThat(routedHandler.apiSpec()).isSameAs(advertisedSpec); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + assertThat(advertisedSpec.supportsVersion(version)).isTrue(); + } + assertThat( + advertisedSpec.supportsVersion( + (short) (ApiKeys.API_VERSIONS.latestVersion() + 1))) + .isFalse(); + } + + @Test + public void testRejectInvalidVersionRange() { + assertThatThrownBy(() -> new KafkaApiSpec(ApiKeys.API_VERSIONS, (short) 1, (short) 0, true)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy( + () -> + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + (short) (ApiKeys.API_VERSIONS.latestVersion() + 1), + true)) + .isInstanceOf(IllegalArgumentException.class); + } + + private static KafkaApiRegistry brokerRegistry() { + return new KafkaApiRegistry(); + } + + private static final class TestingApiVersionsHandler + implements KafkaApiHandler { + + private final KafkaApiSpec spec; + + private TestingApiVersionsHandler(boolean advertised) { + this.spec = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + advertised); + } + + @Override + public KafkaApiSpec apiSpec() { + return spec; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java new file mode 100644 index 00000000000..97f42350198 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/dispatcher/KafkaRequestDispatcherTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.dispatcher; + +import org.apache.fluss.kafka.KafkaRequest; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; + +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Tests request routing and failure handling without a concrete API implementation. */ +class KafkaRequestDispatcherTest { + + @Test + void testUnregisteredApiReturnsUnsupportedVersion() { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.freeze(); + + AbstractResponse response = + new KafkaRequestDispatcher(registry, new KafkaErrorMapper()) + .dispatch(request((short) 0)) + .join(); + + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); + } + + @Test + void testUnsupportedVersionDoesNotInvokeHandler() { + KafkaRequestDispatcher dispatcher = + dispatcher( + (context, request) -> { + throw new AssertionError( + "Unsupported versions must not be dispatched."); + }); + + AbstractResponse response = dispatcher.dispatch(request((short) 1)).join(); + + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); + } + + @Test + void testDispatchWaitsForHandlerAndPreservesContext() { + CompletableFuture handlerResult = new CompletableFuture<>(); + AtomicReference receivedContext = new AtomicReference<>(); + KafkaRequestDispatcher dispatcher = + dispatcher( + (context, request) -> { + receivedContext.set(context); + return handlerResult; + }); + KafkaRequest request = request((short) 0); + + CompletableFuture result = dispatcher.dispatch(request); + + assertThat(result).isNotDone(); + assertThat(receivedContext.get().clientId()).isEqualTo("client"); + assertThat(receivedContext.get().correlationId()).isEqualTo(42); + assertThat(receivedContext.get().listenerName()).isEqualTo("KAFKA"); + assertThat(receivedContext.get().apiKey()).isEqualTo(ApiKeys.API_VERSIONS); + assertThat(receivedContext.get().apiVersion()).isZero(); + AbstractResponse response = new ApiVersionsResponse(new ApiVersionsResponseData()); + handlerResult.complete(response); + assertThat(result.join()).isSameAs(response); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testSynchronousAndAsynchronousFailuresBecomeErrorResponses(boolean synchronous) { + InvalidRequestException failure = new InvalidRequestException("invalid request"); + KafkaRequestDispatcher dispatcher = + dispatcher( + (context, request) -> { + if (synchronous) { + throw failure; + } + CompletableFuture result = new CompletableFuture<>(); + result.completeExceptionally(new CompletionException(failure)); + return result; + }); + + AbstractResponse response = dispatcher.dispatch(request((short) 0)).join(); + + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testNullFutureAndNullResponseBecomeErrorResponses(boolean nullFuture) { + KafkaRequestDispatcher dispatcher = + dispatcher( + (context, request) -> + nullFuture ? null : CompletableFuture.completedFuture(null)); + + AbstractResponse response = dispatcher.dispatch(request((short) 0)).join(); + + assertThat(response.errorCounts()).containsEntry(Errors.UNKNOWN_SERVER_ERROR, 1); + } + + private static KafkaRequestDispatcher dispatcher( + BiFunction> + action) { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register( + new KafkaApiHandler() { + @Override + public KafkaApiSpec apiSpec() { + return new KafkaApiSpec(ApiKeys.API_VERSIONS, (short) 0, (short) 0, true); + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + return action.apply(context, request); + } + }); + registry.freeze(); + return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); + } + + private static KafkaRequest request(short version) { + KafkaRequest request = mock(KafkaRequest.class); + when(request.apiKey()).thenReturn(ApiKeys.API_VERSIONS); + when(request.apiVersion()).thenReturn(version); + when(request.request()).thenReturn(new ApiVersionsRequest.Builder().build(version)); + when(request.header()) + .thenReturn(new RequestHeader(ApiKeys.API_VERSIONS, version, "client", 42)); + when(request.listenerName()).thenReturn("KAFKA"); + when(request.ctx()).thenReturn(mock(ChannelHandlerContext.class)); + return request; + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java index 03d798fb371..df2256985c2 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/NettyServer.java @@ -232,8 +232,17 @@ private static List loadProtocols( NetworkProtocolPlugin kafkaPlugin = loadProtocolPlugin(NetworkProtocolPlugin.KAFKA_PROTOCOL_NAME); kafkaPlugin.setup(conf); - listeners.removeAll(kafkaPlugin.listenerNames()); - protocolPlugins.add(kafkaPlugin); + List kafkaListenerNames = kafkaPlugin.listenerNames(); + boolean hasKafkaEndpoint = + endpoints.stream() + .anyMatch( + endpoint -> + kafkaListenerNames.contains( + endpoint.getListenerName())); + if (hasKafkaEndpoint) { + listeners.removeAll(kafkaListenerNames); + protocolPlugins.add(kafkaPlugin); + } } // Add the Fluss protocol plugin in the end to allow other protocol From 0ea02c70ac875b97728f910e2135e2bb6a53a8b5 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 8 Sep 2026 11:31:07 +0800 Subject: [PATCH 02/10] [kafka] Serve ApiVersions from registered capabilities Route requests through the dispatcher and advertise only implemented APIs. Return version-aware errors for unsupported APIs and invalid requests. Validated with mvn -o -pl fluss-kafka verify (23 unit tests and 1 IT). Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 313/313 AI-Contributed/UT: 137/137 --- .../fluss/kafka/KafkaProtocolPlugin.java | 3 +- .../fluss/kafka/KafkaRequestHandler.java | 232 ++---------------- .../api/versions/ApiVersionsHandler.java | 78 ++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 137 ++++++++--- 4 files changed, 207 insertions(+), 243 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index c966f745b8c..939c35a2d95 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -66,7 +66,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " + service.getClass().getSimpleName()); } - TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(gateway); + return new KafkaRequestHandler(); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 73555093ff0..2df16a8bd7f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,27 +17,24 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import org.apache.kafka.common.message.ApiVersionsResponseData; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.requests.AbstractRequest; -import org.apache.kafka.common.requests.AbstractResponse; -import org.apache.kafka.common.requests.ApiVersionsResponse; - -/** Kafka protocol implementation for request handler. */ +/** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { - // TODO: we may need a new abstraction between TabletService and ReplicaManager to avoid - // affecting Fluss protocol when supporting compatibility with Kafka. - private final TabletServerGateway gateway; + private final KafkaRequestDispatcher dispatcher; - public KafkaRequestHandler(TabletServerGateway gateway) { - this.gateway = gateway; + /** Creates a Kafka request handler with the implemented server capabilities. */ + public KafkaRequestHandler() { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ApiVersionsHandler(registry)); + registry.freeze(); + this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @Override @@ -47,200 +44,15 @@ public RequestType requestType() { @Override public void processRequest(KafkaRequest request) { - // See kafka.server.KafkaApis#handle - switch (request.apiKey()) { - case API_VERSIONS: - handleApiVersionsRequest(request); - break; - case METADATA: - handleMetadataRequest(request); - break; - case PRODUCE: - handleProducerRequest(request); - break; - case FIND_COORDINATOR: - handleFindCoordinatorRequest(request); - break; - case LIST_OFFSETS: - handleListOffsetRequest(request); - break; - case OFFSET_FETCH: - handleOffsetFetchRequest(request); - break; - case OFFSET_COMMIT: - handleOffsetCommitRequest(request); - break; - case FETCH: - handleFetchRequest(request); - break; - case JOIN_GROUP: - handleJoinGroupRequest(request); - break; - case SYNC_GROUP: - handleSyncGroupRequest(request); - break; - case HEARTBEAT: - handleHeartbeatRequest(request); - break; - case LEAVE_GROUP: - handleLeaveGroupRequest(request); - break; - case DESCRIBE_GROUPS: - handleDescribeGroupsRequest(request); - break; - case LIST_GROUPS: - handleListGroupsRequest(request); - break; - case DELETE_GROUPS: - handleDeleteGroupsRequest(request); - break; - case SASL_HANDSHAKE: - handleSaslHandshakeRequest(request); - break; - case SASL_AUTHENTICATE: - handleSaslAuthenticateRequest(request); - break; - case CREATE_TOPICS: - handleCreateTopicsRequest(request); - break; - case INIT_PRODUCER_ID: - handleInitProducerIdRequest(request); - break; - case ADD_PARTITIONS_TO_TXN: - handleAddPartitionsToTxnRequest(request); - break; - case ADD_OFFSETS_TO_TXN: - handleAddOffsetsToTxnRequest(request); - break; - case TXN_OFFSET_COMMIT: - handleTxnOffsetCommitRequest(request); - break; - case END_TXN: - handleEndTxnRequest(request); - break; - case WRITE_TXN_MARKERS: - handleWriteTxnMarkersRequest(request); - break; - case DESCRIBE_CONFIGS: - handleDescribeConfigsRequest(request); - break; - case ALTER_CONFIGS: - handleAlterConfigsRequest(request); - break; - case DELETE_TOPICS: - handleDeleteTopicsRequest(request); - break; - case DELETE_RECORDS: - handleDeleteRecordsRequest(request); - break; - case OFFSET_DELETE: - handleOffsetDeleteRequest(request); - break; - case CREATE_PARTITIONS: - handleCreatePartitionsRequest(request); - break; - case DESCRIBE_CLUSTER: - handleDescribeClusterRequest(request); - break; - default: - handleUnsupportedRequest(request); - } - } - - private void handleUnsupportedRequest(KafkaRequest request) { - String message = String.format("Unsupported request with api key %s", request.apiKey()); - AbstractRequest abstractRequest = request.request(); - AbstractResponse response = - abstractRequest.getErrorResponse(new UnsupportedOperationException(message)); - request.complete(response); - } - - void handleApiVersionsRequest(KafkaRequest request) { - short apiVersion = request.apiVersion(); - if (!ApiKeys.API_VERSIONS.isVersionSupported(apiVersion)) { - request.fail(Errors.UNSUPPORTED_VERSION.exception()); - return; - } - ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (ApiKeys apiKey : ApiKeys.values()) { - if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { - ApiVersionsResponseData.ApiVersion apiVersionData = - new ApiVersionsResponseData.ApiVersion() - .setApiKey(apiKey.id) - .setMinVersion(apiKey.oldestVersion()) - .setMaxVersion(apiKey.latestVersion()); - if (apiKey.equals(ApiKeys.METADATA)) { - // Not support TopicId - short v = apiKey.latestVersion() > 11 ? 11 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } else if (apiKey.equals(ApiKeys.FETCH)) { - // Not support TopicId - short v = apiKey.latestVersion() > 12 ? 12 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } - data.apiKeys().add(apiVersionData); - } - } - request.complete(new ApiVersionsResponse(data)); + dispatcher + .dispatch(request) + .whenComplete( + (response, failure) -> { + if (failure == null) { + request.complete(response); + } else { + request.fail(failure); + } + }); } - - void handleProducerRequest(KafkaRequest request) {} - - void handleMetadataRequest(KafkaRequest request) {} - - void handleFindCoordinatorRequest(KafkaRequest request) {} - - void handleListOffsetRequest(KafkaRequest request) {} - - void handleOffsetFetchRequest(KafkaRequest request) {} - - void handleOffsetCommitRequest(KafkaRequest request) {} - - void handleFetchRequest(KafkaRequest request) {} - - void handleJoinGroupRequest(KafkaRequest request) {} - - void handleSyncGroupRequest(KafkaRequest request) {} - - void handleHeartbeatRequest(KafkaRequest request) {} - - void handleLeaveGroupRequest(KafkaRequest request) {} - - void handleDescribeGroupsRequest(KafkaRequest request) {} - - void handleListGroupsRequest(KafkaRequest request) {} - - void handleDeleteGroupsRequest(KafkaRequest request) {} - - void handleSaslHandshakeRequest(KafkaRequest request) {} - - void handleSaslAuthenticateRequest(KafkaRequest request) {} - - void handleCreateTopicsRequest(KafkaRequest request) {} - - void handleInitProducerIdRequest(KafkaRequest request) {} - - void handleAddPartitionsToTxnRequest(KafkaRequest request) {} - - void handleAddOffsetsToTxnRequest(KafkaRequest request) {} - - void handleTxnOffsetCommitRequest(KafkaRequest request) {} - - void handleEndTxnRequest(KafkaRequest request) {} - - void handleWriteTxnMarkersRequest(KafkaRequest request) {} - - void handleDescribeConfigsRequest(KafkaRequest request) {} - - void handleAlterConfigsRequest(KafkaRequest request) {} - - void handleDeleteTopicsRequest(KafkaRequest request) {} - - void handleDeleteRecordsRequest(KafkaRequest request) {} - - void handleOffsetDeleteRequest(KafkaRequest request) {} - - void handleCreatePartitionsRequest(KafkaRequest request) {} - - void handleDescribeClusterRequest(KafkaRequest request) {} } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java new file mode 100644 index 00000000000..c38d7bc6cb2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements ApiVersions from the capabilities actually registered on this server. */ +@Internal +public final class ApiVersionsHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + true); + + private final KafkaApiRegistry registry; + + /** Creates an ApiVersions handler backed by the server capability registry. */ + public ApiVersionsHandler(KafkaApiRegistry registry) { + this.registry = checkNotNull(registry); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + if (!request.isValid()) { + return CompletableFuture.completedFuture( + request.getErrorResponse(Errors.INVALID_REQUEST.exception())); + } + ApiVersionsResponseData data = new ApiVersionsResponseData(); + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + data.apiKeys() + .add( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(spec.apiKey().id) + .setMinVersion(spec.minVersion()) + .setMaxVersion(spec.maxVersion())); + } + return CompletableFuture.completedFuture(new ApiVersionsResponse(data)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index 8613d83df32..e6b8e961128 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,24 +17,32 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; /** Tests for {@link KafkaRequestHandler}. */ public class KafkaRequestHandlerTest { @@ -53,7 +61,7 @@ public void testKafkaApiVersionsNotSupported() { new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), apiVersionsRequest, ctx); - handler.handleApiVersionsRequest(request); + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); Map errorCounts = response.errorCounts(); @@ -61,44 +69,111 @@ public void testKafkaApiVersionsNotSupported() { assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); } + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + public void testKafkaApiVersionsRequest(short version) { + KafkaRequestHandler handler = createKafkaRequestHandler(); + ApiVersionsResponse response = requestApiVersions(handler, version); + + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + } + + private static ApiVersionsResponse requestApiVersions( + KafkaRequestHandler handler, short version) { + ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build(version); + ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + KafkaRequest request = + newRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), + apiVersionsRequest, + ctx); + handler.processRequest(request); + + return parseApiVersionsResponse(request); + } + + private static ApiVersionsResponse parseApiVersionsResponse(KafkaRequest request) { + return (ApiVersionsResponse) parseResponse(request); + } + + private static void assertSuccessfulResponseDefaults(ApiVersionsResponse response) { + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1)); + assertThat(response.data().throttleTimeMs()).isZero(); + assertThat(response.data().supportedFeatures()).isEmpty(); + assertThat(response.data().finalizedFeaturesEpoch()).isEqualTo(-1L); + assertThat(response.data().finalizedFeatures()).isEmpty(); + assertThat(response.data().zkMigrationReady()).isFalse(); + } + + private static void assertBrokerCapabilities(ApiVersionsResponse response) { + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion())); + } + @Test - public void testKafkaApiVersionsRequest() { + public void testInvalidApiVersionsRequest() { KafkaRequestHandler handler = createKafkaRequestHandler(); short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + ApiVersionsRequest requestBody = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData() + .setClientSoftwareName("invalid client name") + .setClientSoftwareVersion("1.0"), + latestVersion, + latestVersion) + .build(latestVersion); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, latestVersion, new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, - ctx); - handler.handleApiVersionsRequest(request); + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.NONE)); - response.data() - .apiKeys() - .forEach( - apiVersion -> { - if (ApiKeys.METADATA.id == apiVersion.apiKey()) { - assertThat((short) 11) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else if (ApiKeys.FETCH.id == apiVersion.apiKey()) { - assertThat((short) 12) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else { - ApiKeys apiKeys = ApiKeys.forId(apiVersion.apiKey()); - assertThat(apiVersion.minVersion()) - .isEqualTo(apiKeys.oldestVersion()); - assertThat(apiVersion.maxVersion()) - .isEqualTo(apiKeys.latestVersion()); - } - }); + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + } + + @Test + public void testUnregisteredApiIsNotRouted() { + KafkaRequestHandler handler = createKafkaRequestHandler(); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + CreateTopicsRequestData requestData = + new CreateTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + new CreateTopicsRequestData.CreatableTopicCollection( + Collections.singletonList( + new CreateTopicsRequestData.CreatableTopic() + .setName("topic") + .setNumPartitions(1) + .setReplicationFactor((short) 1)) + .iterator())); + CreateTopicsRequest requestBody = + new CreateTopicsRequest.Builder(requestData).build(version); + KafkaRequest request = + newRequest( + ApiKeys.CREATE_TOPICS, + version, + new RequestHeader(ApiKeys.CREATE_TOPICS, version, "client-id", 0), + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); + + CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request); + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); } private static KafkaRequest newRequest( @@ -133,6 +208,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(new TestingTabletGatewayService()); + return new KafkaRequestHandler(); } } From 2212f5ae79990a827afbf23e28d81ef4507f19c4 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 8 Sep 2026 11:46:38 +0800 Subject: [PATCH 03/10] [kafka] Implement Metadata API over Fluss table metadata Add Metadata v0-v11 handling over pre-created Fluss tables, with listener-aware gateway access, table/topic mapping, partition metadata, and error handling. Do not implement CreateTopics, DeleteTopics, or automatic table creation. Honor authoritative ISR values when bucket epochs are present. For legacy metadata without an epoch, conservatively report only an available leader. Verified fluss-kafka: 37 unit tests and 1 integration test, with Spotless and Checkstyle passing. Rebuilt the matching RPC dependency from the same base. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 909/909 AI-Contributed/UT: 540/540 --- .../fluss/kafka/KafkaProtocolPlugin.java | 3 +- .../fluss/kafka/KafkaRequestHandler.java | 17 +- .../kafka/api/metadata/MetadataHandler.java | 190 +++++++ .../metadata/GatewayKafkaMetadataBackend.java | 292 +++++++++++ .../metadata/KafkaClusterMetadata.java | 210 ++++++++ .../metadata/KafkaMetadataBackend.java | 30 ++ .../backend/metadata/KafkaMetadataQuery.java | 102 ++++ .../fluss/kafka/mapping/KafkaTopicMapper.java | 65 +++ .../fluss/kafka/KafkaMetadataHandlerTest.java | 492 ++++++++++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 5 +- .../kafka/mapping/KafkaTopicMapperTest.java | 43 ++ 11 files changed, 1445 insertions(+), 4 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 939c35a2d95..59ce1b2a0ed 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -66,6 +66,7 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " + service.getClass().getSimpleName()); } - return new KafkaRequestHandler(); + TabletServerGateway gateway = (TabletServerGateway) service; + return new KafkaRequestHandler(service, gateway, conf.get(ConfigOptions.KAFKA_DATABASE)); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 2df16a8bd7f..de561509b66 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,22 +17,35 @@ package org.apache.fluss.kafka; +import org.apache.fluss.kafka.api.metadata.MetadataHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { private final KafkaRequestDispatcher dispatcher; - /** Creates a Kafka request handler with the implemented server capabilities. */ - public KafkaRequestHandler() { + /** Creates a Kafka request handler with the capabilities provided by a TabletServer. */ + public KafkaRequestHandler( + RpcGatewayService service, TabletServerGateway gateway, String kafkaDatabase) { + checkNotNull(service); + checkNotNull(gateway); + checkNotNull(kafkaDatabase); KafkaApiRegistry registry = new KafkaApiRegistry(); registry.register(new ApiVersionsHandler(registry)); + registry.register( + new MetadataHandler( + new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase))); registry.freeze(); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java new file mode 100644 index 00000000000..ee8bfe7a5a8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/metadata/MetadataHandler.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataBackend; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements Kafka Metadata versions 0 through 11 using a narrow Fluss metadata backend. */ +@Internal +public final class MetadataHandler implements KafkaApiHandler { + + private static final short MAX_SUPPORTED_VERSION = 11; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.METADATA, + ApiKeys.METADATA.oldestVersion(), + (short) Math.min(ApiKeys.METADATA.latestVersion(), MAX_SUPPORTED_VERSION), + true); + + private final KafkaMetadataBackend backend; + + /** Creates a Metadata handler. */ + public MetadataHandler(KafkaMetadataBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, MetadataRequest request) { + List validTopics = new ArrayList<>(); + List invalidTopics = new ArrayList<>(); + if (!request.isAllTopics()) { + for (MetadataRequestTopic topic : request.data().topics()) { + if (topic.name() == null) { + throw new InvalidRequestException( + "Topic name must be set because topic ID lookup is not supported by " + + "Metadata versions 10 and 11."); + } else if (!Topic.isValid(topic.name())) { + invalidTopics.add( + new KafkaClusterMetadata.Topic( + topic.name(), + topic.topicId(), + TopicError.INVALID_TOPIC, + Collections.emptyList())); + } else { + // Kafka added the topic ID fields in v10, but ID-based Metadata lookup was not + // implemented until v12. This handler intentionally stops at v11. + validTopics.add(new TopicReference(topic.name(), Uuid.ZERO_UUID)); + } + } + } + + KafkaMetadataQuery query = + new KafkaMetadataQuery( + request.isAllTopics(), + validTopics, + context.listenerName(), + clientAddress(context.remoteAddress())); + return backend.getMetadata(query) + .thenApply( + metadata -> { + List topics = + new ArrayList<>(metadata.topics()); + topics.addAll(invalidTopics); + return toResponse( + request.version(), + new KafkaClusterMetadata(metadata.brokers(), topics)); + }); + } + + private static MetadataResponse toResponse(short version, KafkaClusterMetadata metadata) { + MetadataResponseData data = + new MetadataResponseData() + .setThrottleTimeMs(0) + .setControllerId(MetadataResponse.NO_CONTROLLER_ID) + .setClusterAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Broker broker : metadata.brokers()) { + MetadataResponseData.MetadataResponseBroker responseBroker = + new MetadataResponseData.MetadataResponseBroker() + .setNodeId(broker.id()) + .setHost(broker.host()) + .setPort(broker.port()); + if (broker.rack() != null) { + responseBroker.setRack(broker.rack()); + } + data.brokers().add(responseBroker); + } + for (KafkaClusterMetadata.Topic topic : metadata.topics()) { + MetadataResponseData.MetadataResponseTopic responseTopic = + new MetadataResponseData.MetadataResponseTopic() + .setName(topic.name()) + .setTopicId(topic.topicId()) + .setErrorCode(toKafkaError(topic.error()).code()) + .setIsInternal(topic.name() != null && Topic.isInternal(topic.name())) + .setTopicAuthorizedOperations( + MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + for (Partition partition : topic.partitions()) { + responseTopic + .partitions() + .add( + new MetadataResponseData.MetadataResponsePartition() + .setErrorCode( + partition.leaderAvailable() + ? Errors.NONE.code() + : Errors.LEADER_NOT_AVAILABLE.code()) + .setPartitionIndex(partition.partitionId()) + .setLeaderId(partition.leaderId()) + .setLeaderEpoch(partition.leaderEpoch()) + .setReplicaNodes(partition.replicas()) + .setIsrNodes(partition.isr()) + .setOfflineReplicas(partition.offlineReplicas())); + } + data.topics().add(responseTopic); + } + return new MetadataResponse(data, version); + } + + private static Errors toKafkaError(TopicError error) { + switch (error) { + case NONE: + return Errors.NONE; + case UNKNOWN_TOPIC_OR_PARTITION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case UNKNOWN_TOPIC_ID: + return Errors.UNKNOWN_TOPIC_ID; + case INVALID_TOPIC: + return Errors.INVALID_TOPIC_EXCEPTION; + default: + throw new IllegalArgumentException("Unsupported metadata error " + error); + } + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java new file mode 100644 index 00000000000..8e5450a04e1 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/GatewayKafkaMetadataBackend.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Broker; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Partition; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.Topic; +import org.apache.fluss.kafka.backend.metadata.KafkaClusterMetadata.TopicError; +import org.apache.fluss.kafka.backend.metadata.KafkaMetadataQuery.TopicReference; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.MetadataResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.Uuid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the existing Fluss metadata RPC to the Kafka Metadata backend contract. */ +@Internal +public final class GatewayKafkaMetadataBackend implements KafkaMetadataBackend { + + private static final Logger LOG = LoggerFactory.getLogger(GatewayKafkaMetadataBackend.class); + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final String databaseName; + private final KafkaTopicMapper topicMapper; + + /** Creates a metadata backend backed by the local TabletServer gateway. */ + public GatewayKafkaMetadataBackend( + RpcGatewayService service, TabletServerGateway gateway, String databaseName) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.databaseName = checkNotNull(databaseName); + this.topicMapper = new KafkaTopicMapper(databaseName); + } + + @Override + public CompletableFuture getMetadata(KafkaMetadataQuery query) { + if (query.allTopics() || containsTopicId(query.topics())) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenCompose( + response -> + requestFlussMetadata( + query, + new LinkedHashSet<>(response.getTableNamesList()))); + } + + Set topicNames = new LinkedHashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + topicNames.add(topic.topicName()); + } + } + return requestFlussMetadata(query, topicNames); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames) { + return requestFlussMetadata(query, topicNames, true); + } + + private CompletableFuture requestFlussMetadata( + KafkaMetadataQuery query, Set topicNames, boolean refreshAndRetry) { + MetadataRequest request = new MetadataRequest(); + for (String topicName : topicNames) { + request.addAllTablePaths( + Collections.singletonList( + new PbTablePath() + .setDatabaseName(databaseName) + .setTableName(topicName))); + } + setCurrentSession(query); + try { + return gateway.metadata(request) + .handle( + (response, failure) -> + failure == null + ? CompletableFuture.completedFuture( + toKafkaMetadata(query, response)) + : recoverMetadataFailure( + query, failure, refreshAndRetry)) + .thenCompose(future -> future); + } catch (Throwable failure) { + return recoverMetadataFailure(query, failure, refreshAndRetry); + } + } + + private CompletableFuture recoverMetadataFailure( + KafkaMetadataQuery query, Throwable failure, boolean refreshAndRetry) { + if (refreshAndRetry) { + return currentTopicNames(query) + .thenCompose(currentNames -> requestFlussMetadata(query, currentNames, false)); + } + LOG.warn("Failed to load Kafka metadata from Fluss.", unwrap(failure)); + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(unwrap(failure)); + return failed; + } + + private CompletableFuture> currentTopicNames(KafkaMetadataQuery query) { + setCurrentSession(query); + return gateway.listTables(new ListTablesRequest().setDatabaseName(databaseName)) + .thenApply( + response -> { + Set currentNames = + new LinkedHashSet<>(response.getTableNamesList()); + if (!query.allTopics() && !containsTopicId(query.topics())) { + Set requestedNames = new HashSet<>(); + for (TopicReference topic : query.topics()) { + if (topic.topicName() != null) { + requestedNames.add(topic.topicName()); + } + } + currentNames.retainAll(requestedNames); + } + return currentNames; + }); + } + + private KafkaClusterMetadata toKafkaMetadata( + KafkaMetadataQuery query, MetadataResponse response) { + List brokers = new ArrayList<>(); + Set aliveBrokerIds = new HashSet<>(); + for (PbServerNode server : response.getTabletServersList()) { + brokers.add( + new Broker( + server.getNodeId(), + server.getHost(), + server.getPort(), + server.hasRack() ? server.getRack() : null)); + aliveBrokerIds.add(server.getNodeId()); + } + Collections.sort(brokers, Comparator.comparingInt(Broker::id)); + + Map topicsByName = new HashMap<>(); + Map topicsById = new HashMap<>(); + for (PbTableMetadata table : response.getTableMetadatasList()) { + if (!databaseName.equals(table.getTablePath().getDatabaseName())) { + continue; + } + Topic topic = toKafkaTopic(table, aliveBrokerIds); + topicsByName.put(topic.name(), topic); + topicsById.put(topic.topicId(), topic); + } + + List topics = new ArrayList<>(); + if (query.allTopics()) { + topics.addAll(topicsByName.values()); + Collections.sort(topics, Comparator.comparing(Topic::name)); + } else { + for (TopicReference reference : query.topics()) { + Topic topic = + reference.hasTopicId() + ? topicsById.get(reference.topicId()) + : topicsByName.get(reference.topicName()); + if (topic != null && matches(reference, topic)) { + topics.add(topic); + } else { + topics.add(missingTopic(reference)); + } + } + } + return new KafkaClusterMetadata(brokers, topics); + } + + private Topic toKafkaTopic(PbTableMetadata table, Set aliveBrokerIds) { + List partitions = new ArrayList<>(); + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + boolean leaderAvailable = + bucket.hasLeaderId() && aliveBrokerIds.contains(bucket.getLeaderId()); + List replicas = new ArrayList<>(); + List isr = new ArrayList<>(); + List offlineReplicas = new ArrayList<>(); + Set isrIds = new HashSet<>(); + if (bucket.hasBucketEpoch()) { + for (int isrId : bucket.getIsrs()) { + isrIds.add(isrId); + } + } else if (leaderAvailable) { + // Legacy metadata has no authoritative ISR. Report only the available leader + // as a conservative fallback; live followers may still be out of sync. + isrIds.add(bucket.getLeaderId()); + } + for (int replicaId : bucket.getReplicaIds()) { + replicas.add(replicaId); + if (isrIds.contains(replicaId)) { + isr.add(replicaId); + } + if (!aliveBrokerIds.contains(replicaId)) { + offlineReplicas.add(replicaId); + } + } + partitions.add( + new Partition( + bucket.getBucketId(), + leaderAvailable ? bucket.getLeaderId() : -1, + bucket.hasLeaderEpoch() ? bucket.getLeaderEpoch() : -1, + replicas, + isr, + offlineReplicas, + leaderAvailable)); + } + Collections.sort(partitions, Comparator.comparingInt(Partition::partitionId)); + return new Topic( + table.getTablePath().getTableName(), + topicMapper.toTopicId(table.getTableId()), + TopicError.NONE, + partitions); + } + + private static Topic missingTopic(TopicReference reference) { + TopicError error = + reference.hasTopicId() + ? TopicError.UNKNOWN_TOPIC_ID + : TopicError.UNKNOWN_TOPIC_OR_PARTITION; + return new Topic( + reference.topicName(), reference.topicId(), error, Collections.emptyList()); + } + + private static boolean matches(TopicReference reference, Topic topic) { + return (reference.topicName() == null || reference.topicName().equals(topic.name())) + && (!reference.hasTopicId() || reference.topicId().equals(topic.topicId())); + } + + private void setCurrentSession(KafkaMetadataQuery query) { + service.setCurrentSession( + new Session( + (short) 0, + query.listenerName(), + false, + query.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static boolean containsTopicId(List topics) { + for (TopicReference topic : topics) { + if (topic.hasTopicId()) { + return true; + } + } + return false; + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java new file mode 100644 index 00000000000..bbc634af6f0 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaClusterMetadata.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Kafka-domain cluster metadata returned by a Fluss metadata backend. */ +@Internal +public final class KafkaClusterMetadata { + + private final List brokers; + private final List topics; + + /** Creates cluster metadata. */ + public KafkaClusterMetadata(List brokers, List topics) { + this.brokers = immutableCopy(brokers); + this.topics = immutableCopy(topics); + } + + /** Returns Kafka-reachable brokers. */ + public List brokers() { + return brokers; + } + + /** Returns topic metadata and topic-level errors. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Kafka-reachable broker information. */ + @Internal + public static final class Broker { + + private final int id; + private final String host; + private final int port; + private final @Nullable String rack; + + /** Creates broker information. */ + public Broker(int id, String host, int port, @Nullable String rack) { + this.id = id; + this.host = checkNotNull(host); + this.port = port; + this.rack = rack; + } + + /** Returns the Kafka broker ID. */ + public int id() { + return id; + } + + /** Returns the Kafka listener host. */ + public String host() { + return host; + } + + /** Returns the Kafka listener port. */ + public int port() { + return port; + } + + /** Returns the broker rack, if configured. */ + public @Nullable String rack() { + return rack; + } + } + + /** Topic-level error independent of a Kafka response schema version. */ + @Internal + public enum TopicError { + NONE, + UNKNOWN_TOPIC_OR_PARTITION, + UNKNOWN_TOPIC_ID, + INVALID_TOPIC + } + + /** Metadata for one Kafka topic. */ + @Internal + public static final class Topic { + + private final @Nullable String name; + private final Uuid topicId; + private final TopicError error; + private final List partitions; + + /** Creates topic metadata. */ + public Topic( + @Nullable String name, Uuid topicId, TopicError error, List partitions) { + this.name = name; + this.topicId = checkNotNull(topicId); + this.error = checkNotNull(error); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name, if known. */ + public @Nullable String name() { + return name; + } + + /** Returns the stable Kafka topic ID. */ + public Uuid topicId() { + return topicId; + } + + /** Returns the topic-level domain error. */ + public TopicError error() { + return error; + } + + /** Returns the topic partitions. */ + public List partitions() { + return partitions; + } + } + + /** Metadata for one Kafka partition backed by a Fluss bucket. */ + @Internal + public static final class Partition { + + private final int partitionId; + private final int leaderId; + private final int leaderEpoch; + private final List replicas; + private final List isr; + private final List offlineReplicas; + private final boolean leaderAvailable; + + /** Creates partition metadata. */ + public Partition( + int partitionId, + int leaderId, + int leaderEpoch, + List replicas, + List isr, + List offlineReplicas, + boolean leaderAvailable) { + this.partitionId = partitionId; + this.leaderId = leaderId; + this.leaderEpoch = leaderEpoch; + this.replicas = immutableCopy(replicas); + this.isr = immutableCopy(isr); + this.offlineReplicas = immutableCopy(offlineReplicas); + this.leaderAvailable = leaderAvailable; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the current leader ID, or {@code -1} when unavailable. */ + public int leaderId() { + return leaderId; + } + + /** Returns the leader epoch, or {@code -1} when unavailable. */ + public int leaderEpoch() { + return leaderEpoch; + } + + /** Returns assigned replica IDs. */ + public List replicas() { + return replicas; + } + + /** Returns replica IDs currently visible as in-sync. */ + public List isr() { + return isr; + } + + /** Returns assigned replicas whose TabletServers are unavailable. */ + public List offlineReplicas() { + return offlineReplicas; + } + + /** Returns whether the partition has a reachable leader. */ + public boolean leaderAvailable() { + return leaderAvailable; + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java new file mode 100644 index 00000000000..abb02a920ed --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataBackend.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Metadata API. */ +@Internal +public interface KafkaMetadataBackend { + + /** Resolves Kafka-domain metadata asynchronously. */ + CompletableFuture getMetadata(KafkaMetadataQuery query); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java new file mode 100644 index 00000000000..01e21fa4440 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/metadata/KafkaMetadataQuery.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.metadata; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.Uuid; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Domain query used by the Metadata API to access the Fluss adapter layer. */ +@Internal +public final class KafkaMetadataQuery { + + private final boolean allTopics; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a metadata query. */ + public KafkaMetadataQuery( + boolean allTopics, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.allTopics = allTopics; + this.topics = Collections.unmodifiableList(new ArrayList<>(checkNotNull(topics))); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns whether all Kafka topics should be returned. */ + public boolean allTopics() { + return allTopics; + } + + /** Returns the explicitly requested topic identities. */ + public List topics() { + return topics; + } + + /** Returns the Kafka listener used by the client connection. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client address when it is available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + /** Kafka topic name and ID supplied by a Metadata request. */ + @Internal + public static final class TopicReference { + + private final @Nullable String topicName; + private final Uuid topicId; + + /** Creates a topic reference. */ + public TopicReference(@Nullable String topicName, Uuid topicId) { + this.topicName = topicName; + this.topicId = checkNotNull(topicId); + } + + /** Returns the requested topic name, if present. */ + public @Nullable String topicName() { + return topicName; + } + + /** Returns the requested topic ID, or {@link Uuid#ZERO_UUID} when absent. */ + public Uuid topicId() { + return topicId; + } + + /** Returns whether this reference identifies a topic by ID. */ + public boolean hasTopicId() { + return !Uuid.ZERO_UUID.equals(topicId); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java new file mode 100644 index 00000000000..6b75bdfd0d1 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +@Internal +public final class KafkaTopicMapper { + + // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. + private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + + private final String databaseName; + + /** Creates a topic mapper for one Fluss database. */ + public KafkaTopicMapper(String databaseName) { + this.databaseName = checkNotNull(databaseName); + } + + /** Maps a Kafka topic name to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + return TablePath.of(databaseName, topicName); + } + + /** Maps a Fluss table ID to a stable Kafka topic ID. */ + public Uuid toTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(TOPIC_ID_NAMESPACE, tableId); + } + + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ + public boolean isMappedTopicId(Uuid topicId) { + return topicId != null + && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && topicId.getLeastSignificantBits() >= 0L; + } + + /** Extracts the Fluss table ID encoded in a Kafka topic ID. */ + public long toTableId(Uuid topicId) { + checkArgument(isMappedTopicId(topicId), "Topic ID %s is not a Fluss topic ID.", topicId); + return topicId.getLeastSignificantBits(); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java new file mode 100644 index 00000000000..8e1670155e9 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaMetadataHandlerTest.java @@ -0,0 +1,492 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.ListTablesRequest; +import org.apache.fluss.rpc.messages.ListTablesResponse; +import org.apache.fluss.rpc.messages.PbBucketMetadata; +import org.apache.fluss.rpc.messages.PbServerNode; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.messages.PbTablePath; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; + +import org.apache.kafka.common.Node; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.MetadataRequestData; +import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol compatibility tests for the Kafka Metadata API. */ +public class KafkaMetadataHandlerTest { + + private static final Uuid TOPIC_ID = new Uuid(0x466c757373000000L, 123L); + + @Test + public void testNamedTopicForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + MetadataRequest.convertToMetadataRequestTopic( + Collections.singletonList("topic"))), + version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(100, new byte[] {1, 2, 3})); + request.data() + .topics() + .get(0) + .unknownTaggedFields() + .add(new RawTaggedField(101, new byte[] {4, 5, 6})); + } + MetadataResponse response = handle(service, request, version); + + assertThat(response.brokers()).hasSize(2); + assertThat(response.controller()).isNull(); + Node broker = response.brokers().iterator().next(); + assertThat(broker.host()).isEqualTo("broker-1"); + assertThat(broker.port()).isEqualTo(9092); + assertThat(broker.rack()).isEqualTo(version >= 1 ? "rack-a" : null); + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(topic.partitions()).hasSize(2); + assertThat(topic.topicId()).isEqualTo(version >= 10 ? TOPIC_ID : Uuid.ZERO_UUID); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.partitionIndex()).isZero(); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).isEmpty(); + } + assertThat(service.lastListenerName).isEqualTo("KAFKA"); + } + + @Test + public void testAllTopicsForEverySupportedVersion() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = ApiKeys.METADATA.oldestVersion(); version <= 11; version++) { + MetadataRequest request = allTopicsRequest(version); + if (version >= 9) { + request.data() + .unknownTaggedFields() + .add(new RawTaggedField(102, new byte[] {7, 8, 9})); + } + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactly("other", "topic"); + } + } + + @Test + public void testAllTopicsAndMissingAndInvalidTopic() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse allTopics = + handle(service, MetadataRequest.Builder.allTopics().build((short) 9), (short) 9); + assertThat(allTopics.data().topics()) + .extracting(MetadataResponseTopic::name) + .containsExactlyInAnyOrder("other", "topic"); + + MetadataRequest requestedTopics = + new MetadataRequest.Builder(Arrays.asList("missing", "invalid topic"), false) + .build((short) 9); + MetadataResponse errors = handle(service, requestedTopics, (short) 9); + assertThat(errors.errors()) + .containsEntry("missing", Errors.UNKNOWN_TOPIC_OR_PARTITION) + .containsEntry("invalid topic", Errors.INVALID_TOPIC_EXCEPTION); + } + + @Test + public void testV10AndV11IgnoreRequestTopicIdAndLookupByName() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName("topic") + .setTopicId( + new Uuid( + 0x466c757373000000L, + 999L)))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat(response.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + } + } + + @Test + public void testV10AndV11RejectTopicIdOnlyLookup() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + Uuid requestedTopicId = new Uuid(0x466c757373000000L, 999L); + for (short version = 10; version <= 11; version++) { + MetadataRequest request = + new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(null) + .setTopicId(requestedTopicId))), + version); + + MetadataResponse response = handle(service, request, version); + + assertThat(response.data().topics()).hasSize(1); + MetadataResponseTopic responseTopic = response.data().topics().iterator().next(); + assertThat(responseTopic.name()).isEmpty(); + assertThat(responseTopic.topicId()).isEqualTo(requestedTopicId); + assertThat(responseTopic.errorCode()).isEqualTo(Errors.INVALID_REQUEST.code()); + } + } + + @Test + public void testTopicIdentityAcrossDeleteAndRecreate() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + + MetadataResponse initial = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(initial.data().topics().find("topic").topicId()).isEqualTo(TOPIC_ID); + + service.removeTable("topic"); + MetadataResponse deleted = handle(service, namedTopicRequest("topic"), (short) 11); + assertThat(deleted.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + + service.putTable("topic", 223L); + Uuid recreatedTopicId = new Uuid(0x466c757373000000L, 223L); + MetadataResponse recreatedByName = + handle( + service, + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11), + (short) 11); + assertThat(recreatedByName.data().topics().find("topic").topicId()) + .isEqualTo(recreatedTopicId) + .isNotEqualTo(TOPIC_ID); + } + + @Test + public void testDeleteRaceBecomesUnknownTopicResult() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.removeTable("topic"); + service.failNextMetadataAsMissing = true; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf( + Collections.singletonMap(Errors.UNKNOWN_TOPIC_OR_PARTITION, 1)); + } + + @Test + public void testUnavailableLeaderUsesPartitionError() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicLeaderAvailable = false; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponseTopic topic = response.data().topics().find("topic"); + assertThat(topic.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = topic.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.leaderId()).isEqualTo(-1); + assertThat(partition.replicaNodes()).containsExactly(1, 2, 3); + assertThat(partition.isrNodes()).containsExactly(1, 2); + assertThat(partition.offlineReplicas()).containsExactly(3); + } + + @Test + public void testAliveReplicaOutsideIsrIsNotReportedAsInSync() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicIsr = new int[] {1}; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataUsesLeaderOnlyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.leaderId()).isEqualTo(1); + assertThat(partition.replicaNodes()).containsExactly(1, 2); + assertThat(partition.isrNodes()).containsExactly(1); + assertThat(partition.offlineReplicas()).isEmpty(); + } + + @Test + public void testLegacyMetadataWithoutAvailableLeaderHasEmptyIsr() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = null; + service.topicLeaderAvailable = false; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.LEADER_NOT_AVAILABLE.code()); + assertThat(partition.isrNodes()).isEmpty(); + } + + @Test + public void testAuthoritativeEmptyIsrDoesNotUseLegacyFallback() { + for (int bucketEpoch : new int[] {7, -1}) { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.topicBucketEpoch = bucketEpoch; + service.topicIsr = new int[0]; + + MetadataResponse response = handle(service, namedTopicRequest("topic"), (short) 11); + + MetadataResponsePartition partition = + response.data().topics().find("topic").partitions().get(0); + assertThat(partition.isrNodes()).isEmpty(); + } + } + + @Test + public void testUnexpectedGatewayFailureUsesRequestErrorResponse() { + TestingMetadataGatewayService service = new TestingMetadataGatewayService(); + service.failMetadata = true; + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList("topic"), false) + .build((short) 11); + + MetadataResponse response = handle(service, request, (short) 11); + + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.UNKNOWN_SERVER_ERROR, 1)); + assertThat(response.brokers()).isEmpty(); + } + + private static MetadataRequest namedTopicRequest(String topicName) { + return new MetadataRequest( + new MetadataRequestData() + .setTopics( + Collections.singletonList( + new MetadataRequestTopic() + .setName(topicName) + .setTopicId(Uuid.ZERO_UUID))), + (short) 11); + } + + private static MetadataRequest allTopicsRequest(short version) { + MetadataRequestData data = new MetadataRequestData(); + data.setTopics(version == 0 ? Collections.emptyList() : null); + return new MetadataRequest(data, version); + } + + private static MetadataResponse handle( + TestingMetadataGatewayService service, MetadataRequest requestBody, short version) { + KafkaRequestHandler handler = new KafkaRequestHandler(service, service, "kafka"); + ByteBuf requestBuffer = ByteBufAllocator.DEFAULT.buffer(); + KafkaRequest request; + try { + request = + new KafkaRequest( + ApiKeys.METADATA, + version, + new RequestHeader(ApiKeys.METADATA, version, "client-id", 1), + requestBody, + "KAFKA", + requestBuffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + } finally { + // Mirror KafkaCommandDecoder's ownership transfer to KafkaRequest. + requestBuffer.release(); + } + handler.processRequest(request); + ByteBuf responseBuffer = request.responseBuffer(); + try { + return (MetadataResponse) + AbstractResponse.parseResponse(responseBuffer.nioBuffer(), request.header()); + } finally { + responseBuffer.release(); + assertThat(requestBuffer.refCnt()).isZero(); + } + } + + private static final class TestingMetadataGatewayService extends TestingTabletGatewayService { + + private final Map tables = new LinkedHashMap<>(); + private String lastListenerName; + private boolean topicLeaderAvailable = true; + private int[] topicIsr = new int[] {1, 2}; + private Integer topicBucketEpoch = 7; + private boolean failMetadata; + private boolean failNextMetadataAsMissing; + + private TestingMetadataGatewayService() { + tables.put("topic", 123L); + tables.put("other", 124L); + } + + @Override + public CompletableFuture listTables(ListTablesRequest request) { + assertThat(request.getDatabaseName()).isEqualTo("kafka"); + return CompletableFuture.completedFuture( + new ListTablesResponse().addAllTableNames(new ArrayList<>(tables.keySet()))); + } + + @Override + public CompletableFuture metadata( + org.apache.fluss.rpc.messages.MetadataRequest request) { + lastListenerName = currentListenerName(); + if (failMetadata) { + CompletableFuture failure = + new CompletableFuture<>(); + failure.completeExceptionally(new IllegalStateException("metadata unavailable")); + return failure; + } + if (failNextMetadataAsMissing) { + failNextMetadataAsMissing = false; + throw new TableNotExistException("table was deleted"); + } + List topics = new ArrayList<>(); + for (PbTablePath tablePath : request.getTablePathsList()) { + Long tableId = tables.get(tablePath.getTableName()); + if (tableId != null) { + topics.add( + tableMetadata( + tablePath.getTableName(), + tableId, + !"topic".equals(tablePath.getTableName()) + || topicLeaderAvailable, + "topic".equals(tablePath.getTableName()) + ? topicIsr + : new int[] {1, 2}, + "topic".equals(tablePath.getTableName()) + ? topicBucketEpoch + : Integer.valueOf(7))); + } + } + return CompletableFuture.completedFuture( + new org.apache.fluss.rpc.messages.MetadataResponse() + .addAllTabletServers( + Arrays.asList( + new PbServerNode() + .setNodeId(1) + .setHost("broker-1") + .setPort(9092) + .setRack("rack-a"), + new PbServerNode() + .setNodeId(2) + .setHost("broker-2") + .setPort(9093))) + .addAllTableMetadatas(topics)); + } + + private void putTable(String topic, long tableId) { + tables.put(topic, tableId); + } + + private void removeTable(String topic) { + tables.remove(topic); + } + + private static PbTableMetadata tableMetadata( + String topic, + long tableId, + boolean leaderAvailable, + int[] isr, + Integer bucketEpoch) { + PbTableMetadata table = + new PbTableMetadata() + .setTablePath( + new PbTablePath().setDatabaseName("kafka").setTableName(topic)) + .setTableId(tableId) + .addAllBucketMetadatas( + Arrays.asList( + new PbBucketMetadata() + .setBucketId(0) + .setLeaderId(leaderAvailable ? 1 : 3) + .setLeaderEpoch(5) + .setReplicaIds( + leaderAvailable + ? new int[] {1, 2} + : new int[] {1, 2, 3}) + .setIsrs( + bucketEpoch == null ? new int[0] : isr), + new PbBucketMetadata() + .setBucketId(1) + .setLeaderId(2) + .setLeaderEpoch(6) + .setReplicaIds(new int[] {1, 2}) + .setIsrs( + bucketEpoch == null + ? new int[0] + : new int[] {1, 2}))); + if (bucketEpoch != null) { + for (PbBucketMetadata bucket : table.getBucketMetadatasList()) { + bucket.setBucketEpoch(bucketEpoch); + } + } + return table; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index e6b8e961128..c4476c69d59 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.kafka; +import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; @@ -113,6 +114,7 @@ private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.METADATA.id, ApiKeys.METADATA.oldestVersion(), (short) 11), tuple( ApiKeys.API_VERSIONS.id, ApiKeys.API_VERSIONS.oldestVersion(), @@ -208,6 +210,7 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(); + TestingTabletGatewayService service = new TestingTabletGatewayService(); + return new KafkaRequestHandler(service, service, "kafka"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java new file mode 100644 index 00000000000..a0a2036d4d5 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.kafka.common.Uuid; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link KafkaTopicMapper}. */ +public class KafkaTopicMapperTest { + + @Test + public void testTopicNameAndIdMapping() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + + assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + Uuid topicId = mapper.toTopicId(123L); + assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); + assertThat(mapper.isMappedTopicId(topicId)).isTrue(); + assertThat(mapper.toTableId(topicId)).isEqualTo(123L); + + Uuid firstTableTopicId = mapper.toTopicId(0L); + assertThat(firstTableTopicId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(mapper.isMappedTopicId(firstTableTopicId)).isTrue(); + assertThat(mapper.toTableId(firstTableTopicId)).isZero(); + } +} From 6b9626d16d84b47e632ce9ab43f13af3fce116c1 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 15:28:16 +0800 Subject: [PATCH 04/10] [kafka] Define DDL table mapping for Kafka compatibility Extract topic identity and the raw/string table mapping contract before Metadata. Validate table kinds, field projections and metadata columns independently of request handling and record decoding. Validation: Java 11, mvn -o -pl fluss-kafka clean verify (47 unit tests and 2 integration tests); Checkstyle, Spotless and RAT passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 769/769 AI-Contributed/UT: 480/480 --- fluss-kafka/pom.xml | 7 + .../fluss/kafka/format/KafkaDataFormat.java | 73 ++++ .../fluss/kafka/mapping/KafkaTopicMapper.java | 73 ++++ .../kafka/schema/KafkaFieldProjection.java | 123 +++++++ .../fluss/kafka/schema/KafkaTopicSchema.java | 150 ++++++++ .../schema/KafkaTopicSchemaException.java | 30 ++ .../schema/KafkaTopicSchemaResolver.java | 313 +++++++++++++++++ .../kafka/mapping/KafkaTopicMapperTest.java | 62 ++++ .../kafka/schema/KafkaTableMappingITCase.java | 88 +++++ .../schema/KafkaTopicSchemaResolverTest.java | 330 ++++++++++++++++++ 10 files changed, 1249 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index 124001ca7b5..cfb6e2ca31b 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,6 +64,13 @@ + + org.apache.curator + curator-test + ${curator.version} + test + + org.apache.fluss fluss-test-utils diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java new file mode 100644 index 00000000000..ad0039b7e78 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import java.util.Locale; + +/** Supported interpretations of Kafka record key and value bytes. */ +@Internal +public enum KafkaDataFormat { + RAW("raw"), + STRING("string"); + + /** Fluss table custom property controlling the record key format. */ + public static final String KEY_FORMAT_CONFIG = "kafka.key.format"; + + /** Fluss table custom property controlling the record value format. */ + public static final String VALUE_FORMAT_CONFIG = "kafka.value.format"; + + /** Fluss fields populated from the Kafka record key. */ + public static final String KEY_FIELDS_CONFIG = "kafka.key.fields"; + + /** Strategy for deriving fields populated from the Kafka record value. */ + public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; + + /** Fluss column populated from the Kafka record timestamp. */ + public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; + + /** Fluss column populated from the Kafka record headers. */ + public static final String HEADERS_COLUMN_CONFIG = "kafka.metadata.headers.column"; + + private final String value; + + KafkaDataFormat(String value) { + this.value = value; + } + + /** Parses a table custom property value. */ + public static KafkaDataFormat parse(String value) { + if (value == null) { + throw new IllegalArgumentException("Kafka data format must not be null."); + } + String normalized = value.trim().toLowerCase(Locale.ROOT); + for (KafkaDataFormat format : values()) { + if (format.value.equals(normalized)) { + return format; + } + } + throw new IllegalArgumentException( + "Unsupported Kafka data format '" + value + "'. Expected raw or string."); + } + + /** Returns the persisted table property value. */ + public String value() { + return value; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java new file mode 100644 index 00000000000..9f8ed63f543 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.Topic; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +@Internal +public final class KafkaTopicMapper { + + // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. + private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + + private final String databaseName; + + /** Creates a topic mapper for one Fluss database. */ + public KafkaTopicMapper(String databaseName) { + this.databaseName = checkNotNull(databaseName); + } + + /** Maps a Kafka topic name to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + Topic.validate(topicName); + return TablePath.of(databaseName, topicName); + } + + /** Returns whether a table belongs to this database and has a valid Kafka topic name. */ + public boolean isMappedTable(TablePath tablePath) { + return databaseName.equals(tablePath.getDatabaseName()) + && Topic.isValid(tablePath.getTableName()); + } + + /** Maps a Fluss table ID to a stable Kafka topic ID. */ + public Uuid toTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(TOPIC_ID_NAMESPACE, tableId); + } + + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ + public boolean isMappedTopicId(Uuid topicId) { + return topicId != null + && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && topicId.getLeastSignificantBits() >= 0L; + } + + /** Extracts the Fluss table ID encoded in a Kafka topic ID. */ + public long toTableId(Uuid topicId) { + checkArgument(isMappedTopicId(topicId), "Topic ID %s is not a Fluss topic ID.", topicId); + return topicId.getLeastSignificantBits(); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java new file mode 100644 index 00000000000..b1715758a9d --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Ordered physical Fluss fields populated by one Kafka record component. */ +@Internal +public final class KafkaFieldProjection { + + private final List positions; + private final List names; + private final List dataTypes; + + /** Creates a projection from physical row positions. */ + public KafkaFieldProjection(RowType rowType, List positions) { + checkNotNull(rowType); + checkNotNull(positions); + List positionCopy = new ArrayList<>(positions.size()); + List projectedNames = new ArrayList<>(positions.size()); + List projectedTypes = new ArrayList<>(positions.size()); + for (Integer position : positions) { + checkArgument( + position != null && position >= 0 && position < rowType.getFieldCount(), + "Invalid Kafka field projection position %s.", + position); + positionCopy.add(position); + projectedNames.add(rowType.getFieldNames().get(position)); + projectedTypes.add(rowType.getTypeAt(position)); + } + this.positions = Collections.unmodifiableList(positionCopy); + this.names = Collections.unmodifiableList(projectedNames); + this.dataTypes = Collections.unmodifiableList(projectedTypes); + } + + /** Returns the number of projected fields. */ + public int size() { + return positions.size(); + } + + /** Returns whether this projection owns no fields. */ + public boolean isEmpty() { + return positions.isEmpty(); + } + + /** Returns the physical row position at the projection position. */ + public int positionAt(int projectionPosition) { + return positions.get(projectionPosition); + } + + /** Returns the physical field name at the projection position. */ + public String nameAt(int projectionPosition) { + return names.get(projectionPosition); + } + + /** Returns the physical data type at the projection position. */ + public DataType dataTypeAt(int projectionPosition) { + return dataTypes.get(projectionPosition); + } + + /** Returns the projected physical positions. */ + public List positions() { + return positions; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaFieldProjection)) { + return false; + } + KafkaFieldProjection that = (KafkaFieldProjection) obj; + return Objects.equals(positions, that.positions) + && Objects.equals(names, that.names) + && Objects.equals(dataTypes, that.dataTypes); + } + + @Override + public int hashCode() { + return Objects.hash(positions, names, dataTypes); + } + + @Override + public String toString() { + return "KafkaFieldProjection{" + + "positions=" + + positions + + ", " + + "names=" + + names + + ", " + + "dataTypes=" + + dataTypes + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java new file mode 100644 index 00000000000..64263e69286 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.types.RowType; + +import javax.annotation.Nullable; + +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Resolved Kafka key, value, and metadata mapping for one Fluss table schema. */ +@Internal +public final class KafkaTopicSchema { + + private final RowType rowType; + private final @Nullable KafkaDataFormat keyFormat; + private final KafkaFieldProjection keyProjection; + private final KafkaDataFormat valueFormat; + private final KafkaFieldProjection valueProjection; + private final int timestampPosition; + private final int headersPosition; + + /** Creates a resolved Kafka topic schema. */ + public KafkaTopicSchema( + RowType rowType, + @Nullable KafkaDataFormat keyFormat, + KafkaFieldProjection keyProjection, + KafkaDataFormat valueFormat, + KafkaFieldProjection valueProjection, + int timestampPosition, + int headersPosition) { + this.rowType = checkNotNull(rowType); + this.keyFormat = keyFormat; + this.keyProjection = checkNotNull(keyProjection); + this.valueFormat = checkNotNull(valueFormat); + this.valueProjection = checkNotNull(valueProjection); + this.timestampPosition = timestampPosition; + this.headersPosition = headersPosition; + } + + /** Returns the physical Fluss row type. */ + public RowType rowType() { + return rowType; + } + + /** Returns the key format, or null when the Kafka key is not mapped. */ + public @Nullable KafkaDataFormat keyFormat() { + return keyFormat; + } + + /** Returns the key field projection. */ + public KafkaFieldProjection keyProjection() { + return keyProjection; + } + + /** Returns the value format. */ + public KafkaDataFormat valueFormat() { + return valueFormat; + } + + /** Returns the value field projection. */ + public KafkaFieldProjection valueProjection() { + return valueProjection; + } + + /** Returns the timestamp physical position, or -1 when it is not mapped. */ + public int timestampPosition() { + return timestampPosition; + } + + /** Returns the headers physical position, or -1 when they are not mapped. */ + public int headersPosition() { + return headersPosition; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaTopicSchema)) { + return false; + } + KafkaTopicSchema that = (KafkaTopicSchema) obj; + return Objects.equals(rowType, that.rowType) + && Objects.equals(keyFormat, that.keyFormat) + && Objects.equals(keyProjection, that.keyProjection) + && Objects.equals(valueFormat, that.valueFormat) + && Objects.equals(valueProjection, that.valueProjection) + && Objects.equals(timestampPosition, that.timestampPosition) + && Objects.equals(headersPosition, that.headersPosition); + } + + @Override + public int hashCode() { + return Objects.hash( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + @Override + public String toString() { + return "KafkaTopicSchema{" + + "rowType=" + + rowType + + ", " + + "keyFormat=" + + keyFormat + + ", " + + "keyProjection=" + + keyProjection + + ", " + + "valueFormat=" + + valueFormat + + ", " + + "valueProjection=" + + valueProjection + + ", " + + "timestampPosition=" + + timestampPosition + + ", " + + "headersPosition=" + + headersPosition + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java new file mode 100644 index 00000000000..0e6a294c4d5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that a Fluss table does not define a valid Kafka record mapping contract. */ +@Internal +public final class KafkaTopicSchemaException extends IllegalArgumentException { + + /** Creates a Kafka topic schema exception. */ + public KafkaTopicSchemaException(String message) { + super(message); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java new file mode 100644 index 00000000000..840ce8a4f58 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** Resolves and validates the Kafka record mapping stored in Fluss table custom properties. */ +@Internal +public final class KafkaTopicSchemaResolver { + + private static final String INCLUDE_ALL = "ALL"; + private static final String INCLUDE_EXCEPT_KEY = "EXCEPT_KEY"; + + private static final Set SUPPORTED_PROPERTIES = + new HashSet<>( + Arrays.asList( + KafkaDataFormat.KEY_FORMAT_CONFIG, + KafkaDataFormat.KEY_FIELDS_CONFIG, + KafkaDataFormat.VALUE_FORMAT_CONFIG, + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, + KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + + /** Resolves one table's Kafka record mapping contract. */ + public KafkaTopicSchema resolve(TableDescriptor table) { + validateTableKind(table); + RowType rowType = table.getSchema().getRowType(); + Map properties = table.getCustomProperties(); + + for (String property : properties.keySet()) { + if (property.startsWith("kafka.") && !SUPPORTED_PROPERTIES.contains(property)) { + throw invalid("Unsupported Kafka table property '" + property + "'."); + } + } + + int timestampPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG)); + int headersPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + if (timestampPosition >= 0 && timestampPosition == headersPosition) { + throw invalid("Kafka timestamp and headers cannot map to the same Fluss column."); + } + validateMetadataColumns(rowType, timestampPosition, headersPosition); + + String keyFormatValue = properties.get(KafkaDataFormat.KEY_FORMAT_CONFIG); + KafkaDataFormat keyFormat = keyFormatValue == null ? null : parseFormat(keyFormatValue); + List keyPositions = + resolveKeyPositions( + rowType, keyFormat, properties.get(KafkaDataFormat.KEY_FIELDS_CONFIG)); + for (Integer keyPosition : keyPositions) { + if (keyPosition == timestampPosition || keyPosition == headersPosition) { + throw invalid( + "Kafka key field '" + + rowType.getFieldNames().get(keyPosition) + + "' cannot be a Kafka metadata column."); + } + } + KafkaFieldProjection keyProjection = new KafkaFieldProjection(rowType, keyPositions); + + String valueFormatValue = properties.get(KafkaDataFormat.VALUE_FORMAT_CONFIG); + if (valueFormatValue == null) { + throw invalid( + "Missing required table property '" + + KafkaDataFormat.VALUE_FORMAT_CONFIG + + "'."); + } + KafkaDataFormat valueFormat = parseFormat(valueFormatValue); + String fieldsInclude = + normalizeFieldsInclude(properties.get(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG)); + if (!keyPositions.isEmpty() && INCLUDE_ALL.equals(fieldsInclude)) { + throw invalid( + "Mapping Kafka key fields requires " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + "=EXCEPT_KEY."); + } + + Set metadataPositions = new HashSet<>(); + if (timestampPosition >= 0) { + metadataPositions.add(timestampPosition); + } + if (headersPosition >= 0) { + metadataPositions.add(headersPosition); + } + List valuePositions = + resolveValuePositions(rowType, fieldsInclude, keyPositions, metadataPositions); + KafkaFieldProjection valueProjection = new KafkaFieldProjection(rowType, valuePositions); + if (valueProjection.isEmpty()) { + throw invalid("Kafka value projection must contain at least one Fluss column."); + } + + validateSingleFieldFormat(keyFormat, keyProjection, "key"); + validateSingleFieldFormat(valueFormat, valueProjection, "value"); + return new KafkaTopicSchema( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + private static void validateTableKind(TableDescriptor table) { + if (table.hasPrimaryKey()) { + throw invalid("Kafka topic table must be a log table."); + } + if (table.isPartitioned()) { + throw invalid("Partitioned Fluss tables are not supported."); + } + if (new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() + != LogFormat.ARROW) { + throw invalid("Kafka topic table must use the Arrow log format."); + } + } + + private static List resolveKeyPositions( + RowType rowType, KafkaDataFormat keyFormat, String keyFieldsValue) { + if (keyFormat == null) { + if (keyFieldsValue != null) { + throw invalid( + KafkaDataFormat.KEY_FIELDS_CONFIG + + " requires " + + KafkaDataFormat.KEY_FORMAT_CONFIG + + "."); + } + return Collections.emptyList(); + } + if (keyFieldsValue == null || keyFieldsValue.trim().isEmpty()) { + throw invalid( + "Missing required table property '" + KafkaDataFormat.KEY_FIELDS_CONFIG + "'."); + } + String[] fieldNames = keyFieldsValue.split(",", -1); + List positions = new ArrayList<>(fieldNames.length); + Set uniquePositions = new HashSet<>(); + for (String fieldNameValue : fieldNames) { + String fieldName = fieldNameValue.trim(); + if (fieldName.isEmpty()) { + throw invalid("Kafka key field names must not be empty."); + } + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka key field '" + fieldName + "' does not exist."); + } + if (!uniquePositions.add(position)) { + throw invalid("Duplicate Kafka key field '" + fieldName + "'."); + } + positions.add(position); + } + return positions; + } + + private static String normalizeFieldsInclude(String value) { + String normalized = value == null ? INCLUDE_ALL : value.trim().toUpperCase(Locale.ROOT); + if (!INCLUDE_ALL.equals(normalized) && !INCLUDE_EXCEPT_KEY.equals(normalized)) { + throw invalid( + "Invalid " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + " '" + + value + + "'. Expected ALL or EXCEPT_KEY."); + } + return normalized; + } + + private static List resolveValuePositions( + RowType rowType, + String fieldsInclude, + List keyPositions, + Set metadataPositions) { + Set excludedKeyPositions = + INCLUDE_EXCEPT_KEY.equals(fieldsInclude) + ? new HashSet<>(keyPositions) + : Collections.emptySet(); + List positions = new ArrayList<>(); + for (int position = 0; position < rowType.getFieldCount(); position++) { + if (!metadataPositions.contains(position) && !excludedKeyPositions.contains(position)) { + positions.add(position); + } + } + return positions; + } + + private static int resolveOptionalPosition(RowType rowType, String fieldNameValue) { + if (fieldNameValue == null) { + return -1; + } + if (fieldNameValue.trim().isEmpty()) { + throw invalid("Kafka metadata column name must not be empty."); + } + String fieldName = fieldNameValue.trim(); + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka metadata column '" + fieldName + "' does not exist."); + } + return position; + } + + private static void validateMetadataColumns( + RowType rowType, int timestampPosition, int headersPosition) { + if (timestampPosition >= 0) { + if (!(rowType.getTypeAt(timestampPosition) instanceof LocalZonedTimestampType) + || rowType.getTypeAt(timestampPosition).isNullable() + || ((LocalZonedTimestampType) rowType.getTypeAt(timestampPosition)) + .getPrecision() + != 3) { + throw invalid("Kafka timestamp column must be TIMESTAMP_LTZ(3) NOT NULL."); + } + } + if (headersPosition >= 0) { + validateHeadersType(rowType.getTypeAt(headersPosition)); + } + } + + private static void validateHeadersType(org.apache.fluss.types.DataType dataType) { + if (!(dataType instanceof ArrayType) || !dataType.isNullable()) { + throw invalid( + "Kafka headers column must be nullable " + + "ARRAY>."); + } + ArrayType arrayType = (ArrayType) dataType; + if (!(arrayType.getElementType() instanceof RowType)) { + throw invalid("Kafka headers elements must be rows."); + } + RowType headerType = (RowType) arrayType.getElementType(); + if (!headerType.getFieldNames().equals(Arrays.asList("name", "value")) + || !(headerType.getTypeAt(0) instanceof StringType) + || !(headerType.getTypeAt(1) instanceof BytesType) + || !headerType.getTypeAt(1).isNullable()) { + throw invalid("Kafka headers elements must be ROW."); + } + } + + private static void validateSingleFieldFormat( + KafkaDataFormat format, KafkaFieldProjection projection, String component) { + if (format == null) { + return; + } + if (format == KafkaDataFormat.RAW || format == KafkaDataFormat.STRING) { + if (projection.size() != 1) { + throw invalid( + "Kafka " + + component + + " format " + + format.value() + + " requires exactly one Fluss field."); + } + boolean validType = + format == KafkaDataFormat.RAW + ? projection.dataTypeAt(0) instanceof BytesType + : projection.dataTypeAt(0) instanceof StringType; + if (!validType) { + throw invalid( + "Kafka " + + component + + " field '" + + projection.nameAt(0) + + "' must be " + + (format == KafkaDataFormat.RAW ? "BYTES" : "STRING") + + " for format " + + format.value() + + "."); + } + } + } + + private static KafkaDataFormat parseFormat(String value) { + try { + return KafkaDataFormat.parse(value); + } catch (IllegalArgumentException e) { + throw invalid(e.getMessage()); + } + } + + private static KafkaTopicSchemaException invalid(String message) { + return new KafkaTopicSchemaException(message); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java new file mode 100644 index 00000000000..20c32ded941 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link KafkaTopicMapper}. */ +public class KafkaTopicMapperTest { + + @Test + public void testTopicNameAndIdMapping() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + + assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + Uuid topicId = mapper.toTopicId(123L); + assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); + assertThat(mapper.isMappedTopicId(topicId)).isTrue(); + assertThat(mapper.toTableId(topicId)).isEqualTo(123L); + + Uuid firstTableTopicId = mapper.toTopicId(0L); + assertThat(firstTableTopicId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(mapper.isMappedTopicId(firstTableTopicId)).isTrue(); + assertThat(mapper.toTableId(firstTableTopicId)).isZero(); + } + + @Test + public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue(); + assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse(); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath("invalid topic")) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); + assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java new file mode 100644 index 00000000000..0c92b8a7300 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies Kafka mapping properties survive the native Fluss table creation path. */ +public class KafkaTableMappingITCase { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder().setNumOfTabletServers(1).build(); + + @Test + public void testResolveMappingFromCreatedTableMetadata() throws Exception { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("event_key", DataTypes.STRING()) + .column("event_body", DataTypes.BYTES()) + .build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .build(); + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl"); + TablePath tablePath = mapper.toTablePath("events"); + long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor); + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + MetadataRequest request = new MetadataRequest(); + request.addTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()); + PbTableMetadata metadata = + FLUSS_CLUSTER_EXTENSION + .newCoordinatorClient() + .metadata(request) + .get() + .getTableMetadatasList() + .get(0); + TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); + + assertThat(metadata.getTableId()).isEqualTo(tableId); + assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId); + assertThat(metadata.getBucketMetadatasList()).hasSize(2); + assertThat(persisted.getCustomProperties()) + .containsAllEntriesOf(descriptor.getCustomProperties()); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java new file mode 100644 index 00000000000..4c698b5ad6b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the DDL contract independently of Kafka request handling and record decoding. */ +public class KafkaTopicSchemaResolverTest { + + private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); + + @Test + public void testRawMappingSurvivesTableMetadataSerialization() { + Schema schema = + Schema.newBuilder() + .column("message", DataTypes.BYTES()) + .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("attributes", headersType()) + .column("message_key", DataTypes.BYTES()) + .build(); + TableDescriptor table = + table(schema, "raw") + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .build(); + KafkaTopicSchema mapping = + resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes())); + + assertThat(mapping.rowType()).isEqualTo(schema.getRowType()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(3); + assertThat(mapping.keyProjection().nameAt(0)).isEqualTo("message_key"); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0)).isEqualTo(DataTypes.BYTES()); + assertThat(mapping.timestampPosition()).isEqualTo(1); + assertThat(mapping.headersPosition()).isEqualTo(2); + assertThat(mapping).isEqualTo(resolver.resolve(table)); + assertThatThrownBy(() -> mapping.keyProjection().positions().add(1)) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void testValueOnlyStringAndNullableColumns() { + for (boolean nullable : new boolean[] {true, false}) { + KafkaTopicSchema mapping = + resolver.resolve( + table( + Schema.newBuilder() + .column( + "body", + DataTypes.STRING().copy(nullable)) + .build(), + " STRING ") + .build()); + assertThat(mapping.keyFormat()).isNull(); + assertThat(mapping.keyProjection().isEmpty()).isTrue(); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0).isNullable()).isEqualTo(nullable); + assertThat(mapping.timestampPosition()).isEqualTo(-1); + assertThat(mapping.headersPosition()).isEqualTo(-1); + } + } + + @Test + public void testMixedFormatsAndMetadataAreOptional() { + KafkaTopicSchema mapping = + resolver.resolve( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, " except_key ") + .build()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + } + + @Test + public void testRejectsUnsupportedTableKinds() { + Schema primaryKeySchema = + Schema.newBuilder() + .column("id", DataTypes.STRING().copy(false)) + .primaryKey("id") + .build(); + assertInvalid(table(primaryKeySchema, "string"), "must be a log table"); + assertInvalid(keyValueTable().partitionedBy("id"), "Partitioned Fluss tables"); + assertInvalid(valueTable().logFormat(LogFormat.INDEXED), "Arrow log format"); + } + + @Test + public void testRequiresExplicitValueFormat() { + assertInvalid( + TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(1) + .customProperty("fluss.value.format", "raw"), + KafkaDataFormat.VALUE_FORMAT_CONFIG); + } + + @ParameterizedTest + @ValueSource(strings = {"json", "avro", ""}) + public void testRejectsUnavailableFormats(String format) { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + } + + @Test + public void testRejectsUnsupportedKafkaOptions() { + assertInvalid( + valueTable().customProperty("kafka.value.rescue-column", "body"), + "Unsupported Kafka table property"); + assertInvalid( + valueTable().customProperty("kafka.key.field", "body"), + "Unsupported Kafka table property"); + assertThat(resolver.resolve(valueTable().customProperty("owner", "team").build())) + .isNotNull(); + } + + @Test + public void testKeyFormatAndFieldMustBeSpecifiedTogether() { + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires kafka.key.format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"), + "kafka.key.fields"); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "missing", "id,id", "id,", ",id", "id,,body"}) + public void testRejectsInvalidKeyFields(String fields) { + assertThatThrownBy( + () -> + resolver.resolve( + keyValueTable() + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty( + KafkaDataFormat.KEY_FIELDS_CONFIG, fields) + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + "EXCEPT_KEY") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class); + } + + @Test + public void testRejectsAmbiguousAndEmptyValueProjections() { + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "bad"), + "Expected ALL or EXCEPT_KEY"); + assertInvalid( + valueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + assertInvalid(keyValueTable(), "exactly one Fluss field"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id,body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + } + + @Test + public void testRejectsWrongPhysicalTypes() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"), + "must be STRING"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "must be BYTES"); + } + + @Test + public void testRejectsWrongTimestampTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + DataTypes.TIMESTAMP_LTZ(3), + DataTypes.TIMESTAMP_LTZ(6).copy(false) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), + "TIMESTAMP_LTZ(3) NOT NULL"); + } + } + + @Test + public void testRejectsWrongHeaderTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + headersType().copy(false), + DataTypes.ARRAY(DataTypes.STRING()), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("key", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES().copy(false)))) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("attributes", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes"), + "headers"); + } + } + + @Test + public void testRejectsConflictingOrMissingMetadataColumns() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "missing"), + "does not exist"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, " "), + "must not be empty"); + TableDescriptor.Builder table = + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "ts"), + "same Fluss column"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "ts"), + "metadata column"); + } + + private void assertInvalid(TableDescriptor.Builder table, String message) { + assertThatThrownBy(() -> resolver.resolve(table.build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining(message); + } + + private static TableDescriptor.Builder keyValueTable() { + return table( + Schema.newBuilder() + .column("id", DataTypes.STRING()) + .column("body", DataTypes.BYTES()) + .build(), + "raw"); + } + + private static TableDescriptor.Builder valueTable() { + return table(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "raw"); + } + + private static TableDescriptor.Builder table(Schema schema, String valueFormat) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat); + } + + private static DataType headersType() { + return DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))); + } +} From 5352f2eab2b2eae0428ba17d16799557e608bde2 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 1 Jun 2026 09:45:40 +0800 Subject: [PATCH 05/10] complete delay fetch --- .../apache/fluss/rpc/RpcGatewayService.java | 6 ++ .../rpc/netty/server/FlussRequestHandler.java | 2 + .../fluss/server/replica/ReplicaManager.java | 35 +++++++++++ .../server/replica/delay/ActionQueue.java | 39 ++++++++++++ .../replica/delay/DelayedActionQueue.java | 62 +++++++++++++++++++ .../fluss/server/tablet/TabletService.java | 5 ++ .../replica/delay/DelayedFetchLogTest.java | 62 +++++++++++++++++-- 7 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java index 8f2365add70..62a47498f9b 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java @@ -85,4 +85,10 @@ public String currentListenerName() { /** Shutdown the gateway service, release any resources. */ public abstract void shutdown(); + + /** + * Tries to complete all pending delayed actions. Default no-op for services without delayed + * action queue. + */ + public void tryCompleteActions() {} } diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java index 2ff5e0691bb..747a0f9b796 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java @@ -96,6 +96,8 @@ public void processRequest(FlussRequest request) { } catch (Throwable t) { LOG.debug("Error while executing RPC {}", api, t); request.fail(stripException(t, InvocationTargetException.class)); + } finally { + service.tryCompleteActions(); } } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index b3357019230..59f817528c5 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -108,6 +108,8 @@ import org.apache.fluss.server.metrics.group.BucketMetricGroup; import org.apache.fluss.server.metrics.group.TableMetricGroup; import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; +import org.apache.fluss.server.replica.delay.ActionQueue; +import org.apache.fluss.server.replica.delay.DelayedActionQueue; import org.apache.fluss.server.replica.delay.DelayedFetchLog; import org.apache.fluss.server.replica.delay.DelayedFetchLog.FetchBucketStatus; import org.apache.fluss.server.replica.delay.DelayedOperationManager; @@ -212,6 +214,8 @@ public class ReplicaManager implements ServerReconfigurable { */ private final DelayedOperationManager delayedFetchLogManager; + private final ActionQueue actionQueue; + private final ReplicaFetcherManager replicaFetcherManager; // The manager used to manager the replica alter, especially the isr expand and shrink. private final AdjustIsrManager adjustIsrManager; @@ -339,6 +343,7 @@ public ReplicaManager( "delay fetch log", serverId, conf.getInt(ConfigOptions.LOG_REPLICA_FETCH_OPERATION_PURGE_NUMBER)); + this.actionQueue = new DelayedActionQueue(); this.internalListenerName = conf.get(ConfigOptions.INTERNAL_LISTENER_NAME); this.replicaFetcherManager = @@ -683,6 +688,10 @@ public void appendRecordsToLog( appendToLocalLog(entriesPerBucket, requiredAcks, userContext); LOG.debug("Append records to local log in {} ms", System.currentTimeMillis() - startTime); + // Enqueue delayed fetch completions — not executed here. + // Framework layer invokes tryCompleteActions() after this method returns. + addCompletePurgatoryAction(appendResult); + // maybe do delay write operation. maybeAddDelayedWrite( timeoutMs, requiredAcks, entriesPerBucket.size(), appendResult, responseCallback); @@ -2152,6 +2161,32 @@ private boolean isNonCriticalFetchError(Errors error) { || error == Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION; } + /** + * Adds actions to complete delayed fetch log operations for successfully written buckets. + * + *

Actions are added to the {@link ActionQueue} rather than executed immediately. The + * framework layer is responsible for invoking {@link #tryCompleteActions()} to execute the + * queued actions after the write path is fully finished. + */ + private void addCompletePurgatoryAction( + Map writeResults) { + actionQueue.add( + () -> { + for (Map.Entry entry : + writeResults.entrySet()) { + if (entry.getValue().succeeded()) { + delayedFetchLogManager.checkAndComplete( + new DelayedTableBucketKey(entry.getKey())); + } + } + }); + } + + /** Tries to complete all pending delayed actions in the action queue. */ + public void tryCompleteActions() { + actionQueue.tryCompleteActions(); + } + private void completeDelayedOperations(TableBucket tableBucket) { DelayedTableBucketKey delayedTableBucketKey = new DelayedTableBucketKey(tableBucket); delayedWriteManager.checkAndComplete(delayedTableBucketKey); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java new file mode 100644 index 00000000000..47f0a25da5f --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.delay; + +import org.apache.fluss.annotation.Internal; + +/** + * A queue for collecting actions which need to be executed later. + * + *

This is used to decouple the enqueuing of delayed operation completions from their execution. + * For example, after appending records, we enqueue actions to complete delayed fetch operations, + * then execute them after the write path is fully finished. + * + * @see DelayedActionQueue + */ +@Internal +public interface ActionQueue { + + /** Adds an action to this queue. */ + void add(Runnable action); + + /** Tries to complete all pending actions in the queue. */ + void tryCompleteActions(); +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java new file mode 100644 index 00000000000..6458b2c14a4 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.delay; + +import org.apache.fluss.annotation.Internal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Default implementation of {@link ActionQueue} that collects actions into a concurrent queue and + * executes them when {@link #tryCompleteActions()} is called. + * + *

Uses {@link ConcurrentLinkedQueue} for lock-free enqueue. Actions are executed and removed + * from the queue when {@link #tryCompleteActions()} is called. + */ +@Internal +public class DelayedActionQueue implements ActionQueue { + private static final Logger LOG = LoggerFactory.getLogger(DelayedActionQueue.class); + + private final ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + + @Override + public void add(Runnable action) { + queue.add(action); + } + + @Override + public void tryCompleteActions() { + int maxToComplete = queue.size(); + int count = 0; + while (count < maxToComplete) { + Runnable action = queue.poll(); + if (action == null) { + break; + } + try { + action.run(); + } catch (Throwable t) { + LOG.error("Failed to complete delayed action.", t); + } + count++; + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java index bd3ef49b35a..ad97b3f5ed5 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java @@ -212,6 +212,11 @@ public String name() { @Override public void shutdown() {} + @Override + public void tryCompleteActions() { + replicaManager.tryCompleteActions(); + } + @Override public CompletableFuture produceLog(ProduceLogRequest request) { authorizeTable(WRITE, request.getTableId()); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java index 73d620842dc..0311d150db9 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java @@ -83,7 +83,8 @@ void testCompleteDelayedFetchLog() throws Exception { assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(1); assertThat(delayedFetchLogManager.watched()).isEqualTo(1); - // write data. + // Produce data — appendRecordsToLog enqueues a checkAndComplete action, + // which is executed when tryCompleteActions() is called (normally by framework layer). assertThat(delayedResponse.isDone()).isFalse(); CompletableFuture> future = new CompletableFuture<>(); replicaManager.appendRecordsToLog( @@ -92,11 +93,12 @@ void testCompleteDelayedFetchLog() throws Exception { Collections.singletonMap(tb, genMemoryLogRecordsByObject(DATA1)), null, future::complete); + // Simulate framework layer trigger (FlussRequestHandler calls this after invoke). + replicaManager.tryCompleteActions(); assertThat(future.get()).containsOnly(new ProduceLogResultForBucket(tb, 0, 10L)); - // check and complete manually - numComplete = delayedFetchLogManager.checkAndComplete(delayedTableBucketKey); - assertThat(numComplete).isEqualTo(1); + // The delayed fetch log should already be completed by tryCompleteActions above. + assertThat(delayedResponse.isDone()).isTrue(); assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(0); assertThat(delayedFetchLogManager.watched()).isEqualTo(0); @@ -107,6 +109,58 @@ void testCompleteDelayedFetchLog() throws Exception { assertLogRecordsEquals(DATA1_ROW_TYPE, resultForBucket.records(), DATA1); } + @Test + void testProduceAutoCompletesDelayedFetchLog() throws Exception { + TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1); + makeLogTableAsLeader(tb.getBucket()); + + // Set up a delayed fetch with follower-like params (LOG_END isolation, minBytes=1). + FetchLogResultForBucket preFetchResultForBucket = + new FetchLogResultForBucket(tb, MemoryLogRecords.EMPTY, 0L); + CompletableFuture> delayedResponse = + new CompletableFuture<>(); + DelayedFetchLog delayedFetchLog = + createDelayedFetchLogRequest( + tb, + 1, // minFetchBytes = 1, like follower fetch + Duration.ofMinutes(3).toMillis(), + new FetchBucketStatus( + new FetchReqInfo(150001L, 0L, Integer.MAX_VALUE), + new LogOffsetMetadata(0L, 0L, 0), + preFetchResultForBucket), + delayedResponse::complete); + + DelayedOperationManager delayedFetchLogManager = + replicaManager.getDelayedFetchLogManager(); + DelayedTableBucketKey delayedTableBucketKey = new DelayedTableBucketKey(tb); + delayedFetchLogManager.tryCompleteElseWatch( + delayedFetchLog, Collections.singletonList(delayedTableBucketKey)); + assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(1); + assertThat(delayedResponse.isDone()).isFalse(); + + // Produce with acks=1 — response returns immediately, action enqueued. + CompletableFuture> produceResponse = + new CompletableFuture<>(); + replicaManager.appendRecordsToLog( + 20000, + 1, + Collections.singletonMap(tb, genMemoryLogRecordsByObject(DATA1)), + null, + produceResponse::complete); + // Simulate framework layer trigger (FlussRequestHandler calls this after invoke). + replicaManager.tryCompleteActions(); + assertThat(produceResponse.get()).containsOnly(new ProduceLogResultForBucket(tb, 0, 10L)); + + // The delayed fetch should have been auto-completed by tryCompleteActions. + assertThat(delayedResponse.isDone()).isTrue(); + assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(0); + + Map result = delayedResponse.get(); + FetchLogResultForBucket resultForBucket = result.get(tb); + assertThat(resultForBucket).isNotNull(); + assertLogRecordsEquals(DATA1_ROW_TYPE, resultForBucket.records(), DATA1); + } + @Test void testDelayFetchLogTimeout() { TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1); From e15d5d82e50832b590061aac4d8d68357c412767 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 15:47:35 +0800 Subject: [PATCH 06/10] [server] Isolate delayed fetch completions and add regression tests Queue a separate fetch completion for each successfully appended bucket so an ordinary exception cannot prevent other buckets from being checked. Keep each queue drain bounded and allow fatal errors to propagate. Cover partial append failures, multiple buckets, queue execution behavior, and RPC completion timing. Include the delayed fetch wakeup analysis. Validate RPC and server unit and integration suites, including the 44 focused regression tests. The existing disk usage equality test passed in isolation after a transient failure while reading live disk space. Checkstyle, Spotless, license checks, and packaging also pass. Co-Authored-By: Codex AI-Model: gpt-6-astra AI-Contributed/Feature: 911/983 AI-Contributed/UT: 0/339 --- docs/delayed-fetch-produce-wakeup-analysis.md | 911 ++++++++++++++++++ .../apache/fluss/rpc/RpcGatewayService.java | 5 +- .../netty/server/FlussRequestHandlerTest.java | 133 +++ .../fluss/server/replica/ReplicaManager.java | 34 +- .../server/replica/delay/ActionQueue.java | 13 +- .../replica/delay/DelayedActionQueue.java | 20 +- .../replica/delay/DelayedActionQueueTest.java | 77 ++ .../replica/delay/DelayedFetchLogTest.java | 129 ++- 8 files changed, 1233 insertions(+), 89 deletions(-) create mode 100644 docs/delayed-fetch-produce-wakeup-analysis.md create mode 100644 fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/FlussRequestHandlerTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedActionQueueTest.java diff --git a/docs/delayed-fetch-produce-wakeup-analysis.md b/docs/delayed-fetch-produce-wakeup-analysis.md new file mode 100644 index 00000000000..0596a1edb1c --- /dev/null +++ b/docs/delayed-fetch-produce-wakeup-analysis.md @@ -0,0 +1,911 @@ + + +# Produce 后 DelayedFetchLog 唤醒机制分析与实施计划 + +## 当前实现归属（2026-09-10） + +本项作为独立的 Fluss 核心修复，在 `complete-delay-fetch` 分支和已有 +[PR #3412](https://github.com/apache/fluss/pull/3412) 推进，关联 +[issue #3455](https://github.com/apache/fluss/issues/3455)。 +纳入 [umbrella #4185](https://github.com/apache/fluss/issues/4185) 的独立核心模块开发 / review 板块， +可独立 review 和合入 Apache main，不依赖 Kafka PR01–PR03，也不占用 Kafka 功能 PR04 编号。 +2026-09-10 核对 PR 为 Open、非 Draft，远端 head 仍为 `f78917765`；本地补充改动尚未推送。 +整合 `codex/wake-delayed-fetch-after-append` 的候选实现 `a01fb02a5` 到原 tip `f78917765`， +本次不改写分支历史；下文保留原始分析与方案，当前代码差异以本节为准。 + +- `enqueueDelayedFetchCompletions` 在入队前筛选成功 bucket，每个 bucket 一个 action， + 避免某个 bucket 的普通异常跳过同请求中其他 bucket 的唤醒。 +- 队列按 drain 开始时的大小限制执行次数；捕获 `Exception` 后继续执行后续 action， + `Error` 向外传播。该边界不同于下文原方案中的 `catch (Throwable)`。 +- 原生 RPC 同步调用退出后执行 drain；返回的异步响应尚未完成时也必须执行。 +- 回归测试覆盖单 bucket、多 bucket、部分 append 失败、action 只执行一次、 + 普通异常隔离、执行中新增 action 留待后续 drain，以及 RPC 成功/失败/异步返回时机。 +- 后续 Kafka Produce 只依赖此核心能力，并补自己的 drain 调用；不重复提交核心修复。 + +验证记录（2026-09-10，`f78917765` 加本次工作区修改）： + +```bash +JAVA_HOME=/Library/Java/JavaVirtualMachines/zulu-11.jdk/Contents/Home \ + mvn -o -pl fluss-rpc,fluss-server -am clean test \ + -Dtest=FlussRequestHandlerTest,DelayedActionQueueTest,DelayedFetchLogTest,ReplicaManagerTest \ + -Dsurefire.failIfNoSpecifiedTests=false +``` + +- `FlussRequestHandlerTest` 3 个、`DelayedActionQueueTest` 3 个、 + `DelayedFetchLogTest` 4 个、`ReplicaManagerTest` 34 个，共 44 个测试通过，0 failure/error/skip。 +- Checkstyle、Spotless、RAT 和 `git diff --check` 通过；未执行全仓库测试。 +- 首次增量构建遇到旧 class 的 JDK 签名不兼容；Java 11 clean 重编译后上述测试全部通过。 +- 本次仅整合并验证工作区内容，没有 rebase、提交或推送。 + + +## 背景 + +当 follower 向 leader 发送 `FetchLogRequest` 时，如果当前没有足够数据（`bytesReadable < minFetchBytes`），请求会被放入 `DelayedFetchLog` 中等待，直到有新数据或超时（`maxWaitMs`，默认 500ms）。 + +问题：**Produce 写入 leader 后，能否主动唤醒等待中的 follower DelayedFetchLog？** + +> **范围说明**：本文仅分析 Log 表（append-only）的 produce → 复制路径。PK 表的 `putRecordsToKv` 路径存在类似问题，但不在本文讨论范围内。 + +--- + +## 第一部分：Kafka 的完整实现 + +Kafka 通过 **四个协同机制** 确保 produce 写入后 DelayedFetch 被及时唤醒。 + +### 1.1 LeaderHwChange 三态跟踪 + +**文件：** `storage/src/main/java/org/apache/kafka/storage/internals/log/LeaderHwChange.java` + +```java +public enum LeaderHwChange { + INCREASED, // LEO 增长且 HW 也增长了（如单副本或 follower 已追上） + SAME, // LEO 增长但 HW 未变（follower 还没追上，HW 不能提升） + NONE // 写入失败或无变化 +} +``` + +**文件：** `storage/src/main/java/org/apache/kafka/storage/internals/log/LogAppendInfo.java` + +`LogAppendInfo` 中包含 `leaderHwChange` 字段，由 `Partition.appendRecordsToLeader()` 填充： + +```java +// LogAppendInfo 关键字段 +private final LeaderHwChange leaderHwChange; + +public LeaderHwChange leaderHwChange() { return leaderHwChange; } + +// copy 方法用于追加后设置 HW 变化状态 +public LogAppendInfo copy(LeaderHwChange newLeaderHwChange) { + return new LogAppendInfo(..., newLeaderHwChange); +} +``` + +### 1.2 Partition.appendRecordsToLeader() — 设置 LeaderHwChange + +**文件：** `core/src/main/scala/kafka/cluster/Partition.scala:1373` + +```scala +def appendRecordsToLeader(records: MemoryRecords, origin: AppendOrigin, + requiredAcks: Int, ...): LogAppendInfo = { + val (info, leaderHWIncremented) = inReadLock(leaderIsrUpdateLock) { + leaderLogIfLocal match { + case Some(leaderLog) => + val minIsr = effectiveMinIsr(leaderLog) + val inSyncSize = partitionState.isr.size + if (inSyncSize < minIsr && requiredAcks == -1) { + throw new NotEnoughReplicasException(...) + } + + val info = leaderLog.appendAsLeader(records, ...) + + // ★ 关键：追加后尝试提升 HW，并记录 HW 是否变化 + (info, maybeIncrementLeaderHW(leaderLog)) + case None => throw new NotLeaderOrFollowerException(...) + } + } + + // ★ 将 HW 变化状态写入 LogAppendInfo 返回给上层 + info.copy(if (leaderHWIncremented) LeaderHwChange.INCREASED else LeaderHwChange.SAME) +} +``` + +**设计要点**：`appendRecordsToLeader()` 永远不会返回 `NONE` — 只要方法成功返回，就说明 LEO 增长了。`INCREASED` 表示 HW 也跟着涨了（常见于单副本场景），`SAME` 表示 HW 没变（follower 还没追上）。 + +### 1.3 ActionQueue — 延迟执行队列 + +**文件：** `core/src/main/scala/kafka/server/ActionQueue.scala` + +```scala +trait ActionQueue { + def add(action: () => Unit): Unit + def tryCompleteActions(): Unit +} + +class DelayedActionQueue extends Logging with ActionQueue { + private val queue = new ConcurrentLinkedQueue[() => Unit]() + + def add(action: () => Unit): Unit = queue.add(action) + + def tryCompleteActions(): Unit = { + val maxToComplete = queue.size() + var count = 0 + var done = false + while (!done && count < maxToComplete) { + try { + val action = queue.poll() + if (action == null) done = true + else action() + } catch { + case e: Throwable => error("failed to complete delayed actions", e) + } finally count += 1 + } + } +} +``` + +`ActionQueue` 使用 `ConcurrentLinkedQueue` 实现无锁入队，出队时执行动作。核心原则：**`ReplicaManager` 只入队，调用方负责触发执行**。 + +### 1.4 ReplicaManager.appendRecords() → addCompletePurgatoryAction() — 加入 ActionQueue + +**文件：** `core/src/main/scala/kafka/server/ReplicaManager.scala:751` + +```scala +def appendRecords(timeout: Long, requiredAcks: Short, ..., + actionQueue: ActionQueue = this.defaultActionQueue, ...): Unit = { + // Step 1: 写入本地 log + val localProduceResults = appendToLocalLog(...) + + // Step 2: ★ 将唤醒动作加入 ActionQueue（不立即执行） + addCompletePurgatoryAction(actionQueue, localProduceResults) + + // Step 3: 若 acks=-1，创建 DelayedProduce 等待 follower 复制 + maybeAddDelayedProduce(requiredAcks, ...) + // ★ 不在此处调用 tryCompleteActions —— 由调用方负责 +} +``` + +**文件：** `core/src/main/scala/kafka/server/ReplicaManager.scala:923` + +```scala +private def addCompletePurgatoryAction( + actionQueue: ActionQueue, + appendResults: Map[TopicPartition, LogAppendResult] +): Unit = { + actionQueue.add { + () => appendResults.foreach { case (topicPartition, result) => + val requestKey = TopicPartitionOperationKey(topicPartition) + result.info.leaderHwChange match { + case LeaderHwChange.INCREASED => + // HW 提升：唤醒 DelayedProduce + DelayedFetch + DelayedDeleteRecords + delayedProducePurgatory.checkAndComplete(requestKey) + delayedFetchPurgatory.checkAndComplete(requestKey) + delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.SAME => + // ★ HW 没变但 LEO 涨了：仅唤醒 DelayedFetch + // 因为 follower 使用 LOG_END isolation，只需 LEO 增长即可 + delayedFetchPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.NONE => + // 无变化，不唤醒 + } + } + } +} +``` + +**设计要点**：动作被放入 `ActionQueue` 而非直接执行，目的是避免在持有 produce 流程锁时执行唤醒逻辑，防止锁竞争。 + +### 1.5 KafkaApis.handle() 和 KafkaRequestHandler — 统一触发 + +动作在两个地方被统一触发，确保每次请求处理完毕后都能执行队列中的待完成操作。 + +**文件：** `core/src/main/scala/kafka/server/KafkaApis.scala:171` + +```scala +override def handle(request: RequestChannel.Request, requestLocal: RequestLocal): Unit = { + try { + request.header.apiKey match { + case ApiKeys.PRODUCE => handleProduceRequest(request) + case ApiKeys.FETCH => handleFetchRequest(request) + // ... 其他 API + } + } catch { ... } + finally { + // ★ 每次请求处理完后统一执行 ActionQueue + replicaManager.tryCompleteActions() + } +} +``` + +**文件：** `core/src/main/scala/kafka/server/KafkaRequestHandler.scala:148` + +```scala +// 处理 callback（如 DelayedProduce 的 responseCallback）之后也要执行 +finally { + apis.tryCompleteActions() +} +``` + +触发点在 **请求处理框架层**，而非业务层。所有 API 类型（PRODUCE、FETCH 等）处理完后都会统一触发。 + +### 1.6 ReplicaFetcherThread — Follower 侧唤醒 Consumer Fetch + +**文件：** `core/src/main/scala/kafka/server/ReplicaFetcherThread.scala` + +```scala +private[server] val partitionsWithNewHighWatermark = mutable.Buffer[TopicPartition]() + +override def doWork(): Unit = { + super.doWork() // 执行 fetch 并处理返回数据 + completeDelayedFetchRequests() // ★ 唤醒 consumer 的 delayed fetch +} + +override def processPartitionData(topicPartition: TopicPartition, + fetchOffset: Long, ...): Option[LogAppendInfo] = { + val partition = replicaMgr.getPartitionOrException(topicPartition) + val log = partition.localLogOrException + + // 追加 leader 的数据到 follower log + val logAppendInfo = partition.appendRecordsToFollowerOrFutureReplica(records, ...) + + // ★ 若 follower HW 被更新，记录下来 + log.maybeUpdateHighWatermark(partitionData.highWatermark).foreach { newHighWatermark => + partitionsWithNewHighWatermark += topicPartition + } + logAppendInfo +} + +private def completeDelayedFetchRequests(): Unit = { + if (partitionsWithNewHighWatermark.nonEmpty) { + // ★ 唤醒这些 partition 上等待 HW 推进的 consumer DelayedFetch + replicaMgr.completeDelayedFetchRequests(partitionsWithNewHighWatermark.toSeq) + partitionsWithNewHighWatermark.clear() + } +} +``` + +**文件：** `core/src/main/scala/kafka/server/ReplicaManager.scala:456` + +```scala +private[server] def completeDelayedFetchRequests(topicPartitions: Seq[TopicPartition]): Unit = { + topicPartitions.foreach(tp => + delayedFetchPurgatory.checkAndComplete(TopicPartitionOperationKey(tp))) +} +``` + +### 1.7 DelayedFetch.tryComplete() — 判断是否可完成 + +**文件：** `core/src/main/scala/kafka/server/DelayedFetch.scala:74` + +```scala +override def tryComplete(): Boolean = { + var accumulatedSize = 0 + fetchPartitionStatus.foreach { case (topicIdPartition, fetchStatus) => + val fetchOffset = fetchStatus.startOffsetMetadata + try { + if (fetchOffset != LogOffsetMetadata.UNKNOWN_OFFSET_METADATA) { + val partition = replicaManager.getPartitionOrException(...) + val offsetSnapshot = partition.fetchOffsetSnapshot(...) + + // ★ 根据 isolation 级别选择 endOffset + val endOffset = params.isolation match { + case FetchIsolation.LOG_END => offsetSnapshot.logEndOffset // follower 用这个 + case FetchIsolation.HIGH_WATERMARK => offsetSnapshot.highWatermark // consumer 用这个 + case FetchIsolation.TXN_COMMITTED => offsetSnapshot.lastStableOffset // 事务 consumer 用这个 + } + + if (fetchOffset.messageOffset < endOffset.messageOffset) { + if (fetchOffset.onOlderSegment(endOffset)) { + return forceComplete() // Case F: 跨 segment + } else if (fetchOffset.onSameSegment(endOffset)) { + val bytesAvailable = math.min(endOffset.positionDiff(fetchOffset), ...) + accumulatedSize += bytesAvailable + } + } + } + } catch { ... } + } + // Case G: 累积字节数 >= minBytes + if (accumulatedSize >= params.minBytes) forceComplete() else false +} +``` + +### Kafka 完整流程图 + +``` +Producer Request + │ + ▼ +KafkaApis.handle() + │ + ├─ handleProduceRequest() + │ │ + │ ▼ + │ ReplicaManager.appendRecords() [ReplicaManager.scala:751] + │ │ + │ ├─ (1) appendToLocalLog() [ReplicaManager.scala:1372] + │ │ │ + │ │ └─ Partition.appendRecordsToLeader() [Partition.scala:1373] + │ │ ├─ leaderLog.appendAsLeader(records) // LEO 增长 + │ │ ├─ maybeIncrementLeaderHW(leaderLog) // 尝试提升 HW + │ │ └─ return info.copy(INCREASED or SAME) // ★ 返回 HW 变化状态 + │ │ + │ ├─ (2) addCompletePurgatoryAction(actionQueue, results) [ReplicaManager.scala:923] + │ │ └─ actionQueue.add { () => // ★ 只入队，不执行 + │ │ INCREASED → checkAndComplete(delayedProduce + delayedFetch + delayedDelete) + │ │ SAME → checkAndComplete(delayedFetch) // ★ 唤醒 follower fetch + │ │ NONE → (nothing) + │ │ } + │ │ + │ └─ (3) maybeAddDelayedProduce(...) // acks=-1 时等待 follower + │ + └─ finally: + replicaManager.tryCompleteActions() [KafkaApis.scala:280] + └─ actionQueue.tryCompleteActions() [ActionQueue.scala:51] + └─ 执行队列中所有 action（包括上面的 checkAndComplete） + └─ DelayedFetch.tryComplete() [DelayedFetch.scala:74] + └─ 检查 LEO/HW 是否增长 → forceComplete() +``` + +--- + +## 第二部分：Fluss 现状分析 + +### 2.1 Produce 路径 — 缺失唤醒 + +**文件：** `fluss-server/.../replica/ReplicaManager.java:623` + +```java +public void appendRecordsToLog(int timeoutMs, int requiredAcks, + Map entriesPerBucket, + @Nullable UserContext userContext, + Consumer> responseCallback) { + // ... + Map appendResult = + appendToLocalLog(entriesPerBucket, requiredAcks, userContext); + + // ★ 缺失：这里没有唤醒 delayedFetchLogManager + + // 若 acks=-1，创建 DelayedWrite + maybeAddDelayedWrite(timeoutMs, requiredAcks, entriesPerBucket.size(), + appendResult, responseCallback); +} +``` + +### 2.2 appendToLocalLog — 未传递 HW 变化状态 + +**文件：** `fluss-server/.../replica/ReplicaManager.java:1270` + +```java +private Map appendToLocalLog( + Map entriesPerBucket, + int requiredAcks, @Nullable UserContext userContext) { + Map resultForBucketMap = new HashMap<>(); + for (Map.Entry entry : entriesPerBucket.entrySet()) { + TableBucket tb = entry.getKey(); + try { + Replica replica = getReplicaOrException(tb); + LogAppendInfo appendInfo = replica.appendRecordsToLeader(records, requiredAcks); + + // ★ 只记录了 offset，没有记录 HW 是否变化 + resultForBucketMap.put(tb, + new ProduceLogResultForBucket(tb, baseOffset, appendInfo.lastOffset() + 1)); + } catch (Exception e) { + resultForBucketMap.put(tb, + new ProduceLogResultForBucket(tb, ApiError.fromThrowable(e))); + } + } + return resultForBucketMap; +} +``` + +### 2.3 Replica.appendRecordsToLeader() — HW 变化被忽略 + +**文件：** `fluss-server/.../replica/Replica.java:1039` + +```java +public LogAppendInfo appendRecordsToLeader(MemoryLogRecords memoryLogRecords, + int requiredAcks) throws Exception { + return inReadLock(leaderIsrUpdateLock, () -> { + // ... + LogAppendInfo appendInfo; + appendInfo = logTablet.appendAsLeader(memoryLogRecords); + + // ★ maybeIncrementLeaderHW 的返回值（boolean）被忽略 + // ★ 没有将 HW 变化状态传递给上层 + // ★ 没有调用 tryCompleteDelayedOperations() + maybeIncrementLeaderHW(logTablet, clock.milliseconds()); + + return appendInfo; + }); +} +``` + +对比 Kafka 的 `Partition.appendRecordsToLeader()`： +- Kafka 记录 `maybeIncrementLeaderHW` 返回值，映射为 `LeaderHwChange.INCREASED` 或 `SAME` +- Kafka 通过 `info.copy(LeaderHwChange)` 把状态传回 `ReplicaManager` +- Fluss 直接丢弃了这个返回值 + +### 2.4 Fluss LogAppendInfo — 缺少 LeaderHwChange + +**文件：** `fluss-server/.../log/LogAppendInfo.java` + +Fluss 的 `LogAppendInfo` 只包含 offset、timestamp、validBytes 等基础字段，**不包含 `LeaderHwChange` 状态**，无法将 HW 变化信息传递给上层。 + +### 2.5 ReplicaFetcherThread.doWork() — TODO 未实现 + +**文件：** `fluss-server/.../replica/fetcher/ReplicaFetcherThread.java:136` + +```java +@Override +public void doWork() { + maybeFetch(); + // TODO, if we support fetch from follower, we need to complete delayed fetch log operation + // here. +} +``` + +对比 Kafka 的 `ReplicaFetcherThread.doWork()`： +- Kafka 在 `processPartitionData()` 中记录 HW 更新的 partition +- `doWork()` 结束时调用 `completeDelayedFetchRequests()` 唤醒 consumer 的 DelayedFetch +- Fluss 只有一个 TODO 注释 + +### 2.6 TabletService — 无请求后处理 + +**文件：** `fluss-server/.../tablet/TabletService.java:188` + +```java +@Override +public CompletableFuture produceLog(ProduceLogRequest request) { + // ... + replicaManager.appendRecordsToLog(...); + return response; + // ★ 没有类似 KafkaApis.handle() finally 块中的 tryCompleteActions() +} +``` + +### 2.7 FlussRequestHandler — 无框架层后处理 + +**文件：** `fluss-rpc/.../netty/server/FlussRequestHandler.java:54` + +```java +public void processRequest(FlussRequest request) { + // ... + CompletableFuture responseFuture = + (CompletableFuture) api.getMethod().invoke(service, message); + // ★ 没有类似 KafkaApis.handle() finally 块中的 tryCompleteActions() + responseFuture.whenComplete(...); +} +``` + +Fluss 的 RPC 框架层没有 ActionQueue 机制，也没有在请求处理完毕后统一触发 delayed operations 的入口。 + +### 2.8 现有唤醒路径汇总 + +Fluss 中 `tryCompleteDelayedOperations()`（唤醒 DelayedWrite + DelayedFetchLog）的所有调用点： + +| 触发场景 | 代码位置 | 说明 | +|---------|---------|------| +| Follower fetch 更新 LEO → HW 提升 | `Replica.java:1278` `updateFollowerFetchState()` | 仅在 `leaderHWIncremented == true` 时触发 | +| makeLeader 时 HW 提升 | `Replica.java:474` | 仅在 `leaderHWIncremented == true` 时触发 | +| ISR 变更时 HW 提升 | `Replica.java:1914` `submitAdjustIsr()` | 仅在 `hwIncremented == true` 时触发 | +| becomeFollower | `ReplicaManager.java:1222` `completeDelayedOperations()` | 角色转换时强制完成 | +| stopReplica | `ReplicaManager.java:1925` `completeDelayedOperations()` | 停止副本时强制完成 | +| **Produce 写入 (Log 表)** | **❌ 缺失** | **核心差距** | + +### 2.9 实际影响分析 + +**对 follower 复制的影响（`FetchIsolation.LOG_END`）：** +- Follower fetch 携带 `minBytes = 1`，`maxWaitMs = 500ms` +- 当 follower fetch 到达 leader 时恰好无新数据 → 进入 `DelayedFetchLog` +- Producer 写入 → LEO 增长 → 但无人调用 `checkAndComplete` +- **必须等待 500ms 超时** 才能完成该 DelayedFetchLog +- 复制延迟从理想的 ~0ms 增加到最多 500ms + +**对 acks=-1 producer 的影响：** +- 复制延迟增大 → `DelayedWrite` 中等待的 HW 提升更慢 → 端到端延迟增加 +- 最差情况：produce 延迟增加 ~500ms + +**低吞吐 vs 高吞吐：** +- 高吞吐场景影响较小：fetch 到达时通常已有数据，不会进入 delayed 状态 +- 低吞吐场景影响显著：间歇性写入时 follower fetch 大概率进入 delayed + +--- + +## 第三部分：修复方案 — ActionQueue 机制 + +### 方案选型 + +基于对 Kafka 和 Fluss 代码的深入分析，评估了三种方案： + +#### 不采用：直接 checkAndComplete（方案 A） + +在 `appendRecordsToLog()` 末尾直接调用 `delayedFetchLogManager.checkAndComplete()`。虽然最简单，但： +- 不够可扩展：未来新增 delayed operation 类型或写入路径时需逐一手动添加 +- 不与 Kafka 对齐：缺少 ActionQueue 作为策略抽象的灵活性 + +#### 不采用：LeaderHwChange 三态（方案 C） + +分析 `maybeIncrementLeaderHW()`（`Replica.java:1161`）在 produce 路径上的行为： + +| LeaderHwChange | 场景 | 需唤醒 DelayedFetchLog? | 需唤醒 DelayedWrite? | 实际存在 DelayedWrite? | +|---|---|---|---|---| +| `INCREASED` | 单副本 produce | 无 follower，无 delayed fetch | 是 | 否（`delayedWriteRequired` = false） | +| `SAME` | 多副本 produce | **是** | 否（HW 没变） | 可能有，但 HW 没变无法完成 | +| `NONE` | produce 失败 | 否 | 否 | — | + +三态跟踪的精细区分最终退化为：**produce 成功就唤醒 `delayedFetchLogManager`，失败就不唤醒** — 这正是通过 `succeeded()` 判断就能做到的事情。 + +此外，`ReplicaFetcherThread.completeDelayedFetchRequests()` 是为了唤醒 **consumer** 的 DelayedFetch（使用 `HIGH_WATERMARK` isolation），与当前要解决的 **follower 复制延迟**问题无关，不应混在同一个改动中。 + +#### 采用：ActionQueue 机制（方案 B） + +参照 Kafka 的设计，引入 ActionQueue 架构： +- `ReplicaManager.appendRecordsToLog()` 中通过 `addCompletePurgatoryAction()` 将唤醒 action 入队 +- 请求处理框架层（`FlussRequestHandler.processRequest()`）在方法调用后统一触发 `tryCompleteActions()` + +**为什么选择 ActionQueue：** + +1. **可扩展** — 未来新增 delayed operation 类型（如 DelayedDeleteRecords）或新增写入路径时，只需在 `addCompletePurgatoryAction()` 中追加即可 +2. **策略灵活** — ActionQueue 是接口，调用方可选择延迟执行（`DelayedActionQueue`）或立即执行（Kafka 中 `CoordinatorPartitionWriter` 的 `directActionQueue` 模式） +3. **与 Kafka 对齐** — 维护者熟悉的设计模式，降低理解成本 + +**简化设计 — 不引入 LeaderHwChange 三态：** + +`addCompletePurgatoryAction()` 只需检查 `succeeded()` 即可，不需要引入 `LeaderHwChange` 枚举。只唤醒 `delayedFetchLogManager`，不唤醒 `delayedWriteManager`（理由见上表分析）。 + +### 触发策略：请求处理框架层触发 + +Kafka 的 `tryCompleteActions()` 触发点在 **请求处理框架层**（`KafkaApis.handle()` finally 块），而非业务层。对应到 Fluss，触发点应在 `FlussRequestHandler.processRequest()` 中，而非 `TabletService.produceLog()`。 + +``` +Kafka: KafkaApis.handle() finally { tryCompleteActions() } +Fluss: FlussRequestHandler.processRequest() invoke 之后 tryCompleteActions() +``` + +#### 为什么 Fluss 的异步模型（CompletableFuture）下这样做是等价的 + +Fluss 的 `FlussRequestHandler.processRequest()` 通过反射调用 `TabletService.produceLog()` 并得到一个 `CompletableFuture`。虽然返回的是异步 future，但 **所有写入工作在 `invoke` 返回前已同步完成**： + +``` +RequestProcessor thread 上的执行时间线： +────────────────────────────────────────────────────────────── +FlussRequestHandler.processRequest(request) + │ + ├─ CompletableFuture responseFuture = api.invoke(service, message) + │ │ + │ └─ 实际调用 TabletService.produceLog() + │ └─ replicaManager.appendRecordsToLog() + │ ├─ appendToLocalLog() // ① 写入 log（同步） + │ ├─ addCompletePurgatoryAction() // ② 入队（同步） + │ └─ maybeAddDelayedWrite() // ③ 创建 DelayedWrite（同步） + │ + │ ← invoke 返回时，① ② ③ 已全部在当前线程同步完成 + │ + ├─ ★ tryCompleteActions() // ④ 执行队列（同步） + │ + └─ responseFuture.whenComplete(...) // 注册响应回调 +────────────────────────────────────────────────────────────── +``` + +- **acks=1**：`maybeAddDelayedWrite` 内部直接调用 `responseCallback`，future 在 invoke 返回前已 complete +- **acks=-1**：future 还未 complete（等 follower 复制 → HW 提升 → DelayedWrite 完成），但 action 入队 ② 已同步发生 + +因此在 `invoke` 之后调用 `tryCompleteActions()` ④，时机与 Kafka 的 `finally { tryCompleteActions() }` 等价——都是在写入 + 入队同步完成后、同一线程上立即执行。 + +#### 框架层触发 vs 业务层触发 + +将 `tryCompleteActions()` 放在框架层（`FlussRequestHandler`）而非业务层（`TabletService`）： +- **不遗漏** — 框架层对所有 API 请求统一后处理，新增写入 API 不需要记得手动加 +- **职责清晰** — `ReplicaManager` 负责入队，框架层负责触发，`TabletService` 专注业务逻辑 +- **与 Kafka 对齐** — `KafkaApis.handle()` 也是框架层，对所有 API 类型统一 finally + +--- + +## 第四部分：Kafka vs Fluss 完整对比 + +| 维度 | Kafka | Fluss（修复前） | 差距 | +|------|-------|---------|------| +| **Produce 后唤醒 DelayedFetch** | ✅ `addCompletePurgatoryAction` 将动作加入 `ActionQueue` | ❌ 缺失 | 核心差距 | +| **LeaderHwChange 跟踪** | ✅ INCREASED/SAME/NONE 三态，在 `LogAppendInfo` 中传递 | ❌ `maybeIncrementLeaderHW` 返回值被忽略，`LogAppendInfo` 无此字段 | 信息丢失 | +| **ActionQueue 机制** | ✅ 请求处理完后统一执行，避免锁竞争 | ❌ 不存在 | 架构缺失 | +| **RPC 层后处理** | ✅ `KafkaApis.handle()` finally + `KafkaRequestHandler` callback finally | ❌ `FlussRequestHandler` 和 `TabletService` 均无 finally 逻辑 | 执行点缺失 | +| **Follower fetcher 唤醒 consumer fetch** | ✅ `doWork()` 中 `completeDelayedFetchRequests()` | ❌ 有 TODO 注释未实现 | 未实现 | +| **Follower 无新数据时的复制延迟** | ≈ 0ms（produce 后立即唤醒） | 最差 500ms（`maxWaitMs` 超时） | 性能差距 | +| **DelayedFetch tryComplete 逻辑** | ✅ 支持 LOG_END / HIGH_WATERMARK / TXN_COMMITTED 三种 isolation | ✅ 支持 LOG_END / HIGH_WATERMARK 两种 isolation | 基本一致 | +| **Follower fetch 参数** | `minBytes=1`, `maxWaitMs=500ms` | `minBytes=1`, `maxWaitMs=500ms` | 一致 | + +--- + +## 第五部分：实施计划 + +### Step 1: 创建 ActionQueue 接口 + +**新建文件：** `fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java` + +**包位置：** `org.apache.fluss.server.replica.delay` — 与 `DelayedOperationManager`、`DelayedFetchLog` 等同包 + +```java +package org.apache.fluss.server.replica.delay; + +import org.apache.fluss.annotation.Internal; + +/** + * A queue for collecting actions which need to be executed later. + * + *

This is used to decouple the enqueuing of delayed operation completions from their execution. + * For example, after appending records, we enqueue actions to complete delayed fetch operations, + * then execute them after the write path is fully finished. + */ +@Internal +public interface ActionQueue { + + /** Adds an action to this queue. */ + void add(Runnable action); + + /** Tries to complete all pending actions in the queue. */ + void tryCompleteActions(); +} +``` + +### Step 2: 创建 DelayedActionQueue 实现 + +**新建文件：** `fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java` + +```java +package org.apache.fluss.server.replica.delay; + +import org.apache.fluss.annotation.Internal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Default implementation of {@link ActionQueue} that collects actions into a concurrent queue and + * executes them when {@link #tryCompleteActions()} is called. + * + *

Uses {@link ConcurrentLinkedQueue} for lock-free enqueue. Actions are executed and removed + * from the queue when {@link #tryCompleteActions()} is called. + */ +@Internal +public class DelayedActionQueue implements ActionQueue { + private static final Logger LOG = LoggerFactory.getLogger(DelayedActionQueue.class); + + private final ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); + + @Override + public void add(Runnable action) { + queue.add(action); + } + + @Override + public void tryCompleteActions() { + int maxToComplete = queue.size(); + int count = 0; + while (count < maxToComplete) { + Runnable action = queue.poll(); + if (action == null) { + break; + } + try { + action.run(); + } catch (Throwable t) { + LOG.error("Failed to complete delayed action.", t); + } + count++; + } + } +} +``` + +### Step 3: 集成到 ReplicaManager + +**修改文件：** `fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java` + +#### 3a. 新增字段 + +```java +private final ActionQueue actionQueue; +``` + +在构造函数中初始化： + +```java +this.actionQueue = new DelayedActionQueue(); +``` + +#### 3b. 新增 addCompletePurgatoryAction 方法 + +```java +/** + * Adds actions to complete delayed fetch log operations for successfully written buckets. + * + *

Actions are added to the {@link ActionQueue} rather than executed immediately. The caller + * is responsible for invoking {@link #tryCompleteActions()} to execute the queued actions after + * the write path is fully finished. + */ +private void addCompletePurgatoryAction( + Map writeResults) { + actionQueue.add( + () -> { + for (Map.Entry entry : + writeResults.entrySet()) { + if (entry.getValue().succeeded()) { + delayedFetchLogManager.checkAndComplete( + new DelayedTableBucketKey(entry.getKey())); + } + } + }); +} +``` + +#### 3c. 修改 appendRecordsToLog() + +删除现有的直接 `checkAndComplete` 循环，替换为 `addCompletePurgatoryAction` 入队调用： + +```java +public void appendRecordsToLog( + int timeoutMs, + int requiredAcks, + Map entriesPerBucket, + @Nullable UserContext userContext, + Consumer> responseCallback) { + // ... validation ... + + Map appendResult = + appendToLocalLog(entriesPerBucket, requiredAcks, userContext); + + // Enqueue delayed fetch completions — not executed here. + // Framework layer invokes tryCompleteActions() after this method returns. + addCompletePurgatoryAction(appendResult); + + // Maybe create DelayedWrite for acks=-1, or invoke callback for acks=1. + maybeAddDelayedWrite( + timeoutMs, requiredAcks, entriesPerBucket.size(), appendResult, responseCallback); +} +``` + +#### 3d. 暴露公共方法 + +```java +/** Tries to complete all pending delayed actions in the action queue. */ +public void tryCompleteActions() { + actionQueue.tryCompleteActions(); +} +``` + +### Step 4: 修改 FlussRequestHandler（框架层触发） + +**修改文件：** `fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/server/FlussRequestHandler.java` + +在 `processRequest()` 中，`invoke` 之后、`whenComplete` 之前，调用 `tryCompleteActions()`： + +```java +@Override +public void processRequest(FlussRequest request) { + // ... session setup, leader check ... + try { + // invoke the corresponding method on RpcGateway instance. + CompletableFuture responseFuture = + (CompletableFuture) api.getMethod().invoke(service, message); + + // ★ 新增：执行写入路径中入队的延迟操作（如唤醒 DelayedFetchLog） + service.tryCompleteActions(); + + responseFuture.whenComplete( + (response, throwable) -> { + // ... response handling ... + }); + } catch (Throwable t) { + // ... error handling ... + } +} +``` + +这需要在 `RpcGatewayService` 接口中增加 `tryCompleteActions()` 默认方法： + +**修改文件：** `fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java` + +```java +/** Tries to complete all pending delayed actions. Default no-op for services without ActionQueue. */ +default void tryCompleteActions() {} +``` + +**修改文件：** `fluss-server/src/main/java/org/apache/fluss/server/tablet/TabletService.java` + +```java +@Override +public void tryCompleteActions() { + replicaManager.tryCompleteActions(); +} +``` + +### Step 5: 验证测试 + +**文件：** `fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java` + +现有测试直接调用 `replicaManager.appendRecordsToLog()`，绕过了框架层。需要在测试中 produce 后补充 `replicaManager.tryCompleteActions()` 调用。 + +需验证： +- `testCompleteDelayedFetchLog` — 需在 produce 后补充 `tryCompleteActions()` 调用 +- `testProduceAutoCompletesDelayedFetchLog` — 同上 + +--- + +## 执行顺序分析 + +### 完整调用链 + +``` +FlussRequestHandler.processRequest(request) + │ + ├─ api.invoke(service, message) → TabletService.produceLog() + │ │ + │ └─ replicaManager.appendRecordsToLog(...) + │ ├─ appendToLocalLog() → LEO 增长 + │ ├─ addCompletePurgatoryAction() → 将 checkAndComplete action 入队（不执行） + │ └─ maybeAddDelayedWrite() → acks=1: 直接 callback + │ → acks=-1: 创建 DelayedWrite + │ ← invoke 返回（同步） + │ + ├─ service.tryCompleteActions() → 执行队列中的 action，唤醒 follower fetch + │ + └─ responseFuture.whenComplete(...) → 注册异步响应回调 +``` + +### 为什么先 `addCompletePurgatoryAction` 后 `maybeAddDelayedWrite`？ + +- `addCompletePurgatoryAction` 只是入队，不执行，顺序不影响行为 +- `maybeAddDelayedWrite` 需要在 `tryCompleteActions` 之前完成，确保 acks=-1 时 DelayedWrite 已注册 watch +- `tryCompleteActions` 由框架层在 `invoke` 返回后统一执行，此时 produce 响应已发出（acks=1）或 DelayedWrite 已就绪（acks=-1） + +--- + +## 涉及文件总结 + +| 文件 | 操作 | 说明 | +|------|------|------| +| `.../delay/ActionQueue.java` | **新建** | 接口定义 | +| `.../delay/DelayedActionQueue.java` | **新建** | ConcurrentLinkedQueue 实现 | +| `.../replica/ReplicaManager.java` | **修改** | 集成 ActionQueue，修改 `appendRecordsToLog` | +| `.../rpc/RpcGatewayService.java` | **修改** | 新增 `tryCompleteActions()` 默认方法 | +| `.../rpc/netty/server/FlussRequestHandler.java` | **修改** | `invoke` 后调用 `service.tryCompleteActions()` | +| `.../tablet/TabletService.java` | **修改** | 实现 `tryCompleteActions()` 委托给 ReplicaManager | +| `.../delay/DelayedFetchLogTest.java` | **修改** | 补充 `tryCompleteActions()` 调用 | + +**预期效果**：Log 表 follower 复制延迟从最差 500ms 降低到 ~0ms（produce 后立即唤醒）。 + +**后续独立优化**（不在本次范围内）： +- PK 表 `putRecordsToKv` 路径接入 ActionQueue +- `ReplicaFetcherThread.doWork()` 中实现 `completeDelayedFetchRequests()`，用于支持 consumer fetch from follower 场景 + +--- + +## 验证步骤 + +```bash +# 1. 运行 delayed fetch 相关测试 +./mvnw test -Dtest=DelayedFetchLogTest -pl fluss-server + +# 2. 代码格式检查 +./mvnw spotless:check -pl fluss-server + +# 3. 完整模块测试 +./mvnw verify -pl fluss-server +``` diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java index 62a47498f9b..d1738a54ada 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/RpcGatewayService.java @@ -87,8 +87,9 @@ public String currentListenerName() { public abstract void shutdown(); /** - * Tries to complete all pending delayed actions. Default no-op for services without delayed - * action queue. + * Tries to complete actions that were deferred while handling the current request. + * + *

Services without deferred actions do not need to override this method. */ public void tryCompleteActions() {} } diff --git a/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/FlussRequestHandlerTest.java b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/FlussRequestHandlerTest.java new file mode 100644 index 00000000000..e1eff680fc8 --- /dev/null +++ b/fluss-rpc/src/test/java/org/apache/fluss/rpc/netty/server/FlussRequestHandlerTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.rpc.netty.server; + +import org.apache.fluss.rpc.TestingGatewayService; +import org.apache.fluss.rpc.messages.ApiMessage; +import org.apache.fluss.rpc.messages.ApiVersionsRequest; +import org.apache.fluss.rpc.messages.ApiVersionsResponse; +import org.apache.fluss.rpc.protocol.ApiKeys; +import org.apache.fluss.rpc.protocol.ApiManager; +import org.apache.fluss.security.acl.FlussPrincipal; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; + +import org.junit.jupiter.api.Test; + +import java.net.InetAddress; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link FlussRequestHandler}. */ +class FlussRequestHandlerTest { + + @Test + void testCompletesActionsAfterSuccessfulInvocation() { + CompletableFuture serviceResponse = + CompletableFuture.completedFuture(new ApiVersionsResponse()); + TestingActionGatewayService service = new TestingActionGatewayService(serviceResponse); + FlussRequest request = createApiVersionsRequest(); + + new FlussRequestHandler(service).processRequest(request); + + assertThat(service.completedActions()).isOne(); + assertThat(request.getResponseFuture()).isCompletedWithValue(serviceResponse.join()); + } + + @Test + void testCompletesActionsAfterSynchronousInvocationFailure() { + IllegalStateException expected = new IllegalStateException("expected test failure"); + TestingActionGatewayService service = new TestingActionGatewayService(expected); + FlussRequest request = createApiVersionsRequest(); + + new FlussRequestHandler(service).processRequest(request); + + assertThat(service.completedActions()).isOne(); + assertThatThrownBy(request.getResponseFuture()::join) + .isInstanceOf(CompletionException.class) + .hasCause(expected); + } + + @Test + void testCompletesActionsBeforeAsynchronousResponseFinishes() { + CompletableFuture serviceResponse = new CompletableFuture<>(); + TestingActionGatewayService service = new TestingActionGatewayService(serviceResponse); + FlussRequest request = createApiVersionsRequest(); + + new FlussRequestHandler(service).processRequest(request); + + assertThat(service.completedActions()).isOne(); + assertThat(request.getResponseFuture()).isNotDone(); + + ApiVersionsResponse response = new ApiVersionsResponse(); + serviceResponse.complete(response); + assertThat(request.getResponseFuture()).isCompletedWithValue(response); + assertThat(service.completedActions()).isOne(); + } + + private static FlussRequest createApiVersionsRequest() { + return new FlussRequest( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.highestSupportedVersion, + 1, + ApiManager.forApiKey(ApiKeys.API_VERSIONS.id), + new ApiVersionsRequest(), + Unpooled.EMPTY_BUFFER, + "FLUSS", + false, + FlussPrincipal.ANONYMOUS, + InetAddress.getLoopbackAddress(), + new CompletableFuture()); + } + + private static final class TestingActionGatewayService extends TestingGatewayService { + private final CompletableFuture response; + private final RuntimeException synchronousFailure; + private final AtomicInteger completedActions = new AtomicInteger(); + + private TestingActionGatewayService(CompletableFuture response) { + this.response = response; + this.synchronousFailure = null; + } + + private TestingActionGatewayService(RuntimeException synchronousFailure) { + this.response = null; + this.synchronousFailure = synchronousFailure; + } + + @Override + public CompletableFuture apiVersions(ApiVersionsRequest request) { + if (synchronousFailure != null) { + throw synchronousFailure; + } + return response; + } + + @Override + public void tryCompleteActions() { + completedActions.incrementAndGet(); + } + + private int completedActions() { + return completedActions.get(); + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 59f817528c5..f2a1e5db115 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -688,9 +688,9 @@ public void appendRecordsToLog( appendToLocalLog(entriesPerBucket, requiredAcks, userContext); LOG.debug("Append records to local log in {} ms", System.currentTimeMillis() - startTime); - // Enqueue delayed fetch completions — not executed here. - // Framework layer invokes tryCompleteActions() after this method returns. - addCompletePurgatoryAction(appendResult); + // Queue fetch completion before registering a delayed write. The request handler drains + // the actions only after both steps have returned. + enqueueDelayedFetchCompletions(appendResult); // maybe do delay write operation. maybeAddDelayedWrite( @@ -2161,28 +2161,20 @@ private boolean isNonCriticalFetchError(Errors error) { || error == Errors.UNKNOWN_TABLE_OR_BUCKET_EXCEPTION; } - /** - * Adds actions to complete delayed fetch log operations for successfully written buckets. - * - *

Actions are added to the {@link ActionQueue} rather than executed immediately. The - * framework layer is responsible for invoking {@link #tryCompleteActions()} to execute the - * queued actions after the write path is fully finished. - */ - private void addCompletePurgatoryAction( - Map writeResults) { - actionQueue.add( - () -> { - for (Map.Entry entry : - writeResults.entrySet()) { - if (entry.getValue().succeeded()) { - delayedFetchLogManager.checkAndComplete( - new DelayedTableBucketKey(entry.getKey())); - } + private void enqueueDelayedFetchCompletions( + Map appendResults) { + appendResults.forEach( + (tableBucket, appendResult) -> { + if (appendResult.succeeded()) { + actionQueue.add( + () -> + delayedFetchLogManager.checkAndComplete( + new DelayedTableBucketKey(tableBucket))); } }); } - /** Tries to complete all pending delayed actions in the action queue. */ + /** Tries to complete actions deferred by log appends. */ public void tryCompleteActions() { actionQueue.tryCompleteActions(); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java index 47f0a25da5f..997295f6718 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/ActionQueue.java @@ -20,20 +20,15 @@ import org.apache.fluss.annotation.Internal; /** - * A queue for collecting actions which need to be executed later. - * - *

This is used to decouple the enqueuing of delayed operation completions from their execution. - * For example, after appending records, we enqueue actions to complete delayed fetch operations, - * then execute them after the write path is fully finished. - * - * @see DelayedActionQueue + * A queue for collecting actions that must run after the current request invocation releases its + * write-path locks. */ @Internal public interface ActionQueue { - /** Adds an action to this queue. */ + /** Adds an action to the queue. */ void add(Runnable action); - /** Tries to complete all pending actions in the queue. */ + /** Tries to execute pending actions without waiting for actions added concurrently. */ void tryCompleteActions(); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java index 6458b2c14a4..a5070de3a55 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/delay/DelayedActionQueue.java @@ -25,11 +25,11 @@ import java.util.concurrent.ConcurrentLinkedQueue; /** - * Default implementation of {@link ActionQueue} that collects actions into a concurrent queue and - * executes them when {@link #tryCompleteActions()} is called. + * Thread-safe {@link ActionQueue} backed by a concurrent queue. * - *

Uses {@link ConcurrentLinkedQueue} for lock-free enqueue. Actions are executed and removed - * from the queue when {@link #tryCompleteActions()} is called. + *

Each drain bounds its work using the queue's weakly consistent size at the start. Actions + * added while draining may remain available for a later drain. A failing action is logged and does + * not prevent the remaining bounded set from running. */ @Internal public class DelayedActionQueue implements ActionQueue { @@ -44,19 +44,17 @@ public void add(Runnable action) { @Override public void tryCompleteActions() { - int maxToComplete = queue.size(); - int count = 0; - while (count < maxToComplete) { + int actionsToComplete = queue.size(); + for (int completed = 0; completed < actionsToComplete; completed++) { Runnable action = queue.poll(); if (action == null) { - break; + return; } try { action.run(); - } catch (Throwable t) { - LOG.error("Failed to complete delayed action.", t); + } catch (Exception e) { + LOG.error("Failed to complete delayed action.", e); } - count++; } } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedActionQueueTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedActionQueueTest.java new file mode 100644 index 00000000000..27bc0fa70ad --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedActionQueueTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.delay; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link DelayedActionQueue}. */ +class DelayedActionQueueTest { + + @Test + void testActionsExecuteExactlyOnce() { + DelayedActionQueue actionQueue = new DelayedActionQueue(); + AtomicInteger executions = new AtomicInteger(); + actionQueue.add(executions::incrementAndGet); + actionQueue.add(executions::incrementAndGet); + + actionQueue.tryCompleteActions(); + actionQueue.tryCompleteActions(); + + assertThat(executions).hasValue(2); + } + + @Test + void testActionFailureDoesNotPreventLaterActions() { + DelayedActionQueue actionQueue = new DelayedActionQueue(); + AtomicInteger executions = new AtomicInteger(); + actionQueue.add( + () -> { + throw new RuntimeException("expected test failure"); + }); + actionQueue.add(executions::incrementAndGet); + + actionQueue.tryCompleteActions(); + actionQueue.tryCompleteActions(); + + assertThat(executions).hasValue(1); + } + + @Test + void testDrainUsesPendingActionSnapshot() { + DelayedActionQueue actionQueue = new DelayedActionQueue(); + List executions = new ArrayList<>(); + actionQueue.add( + () -> { + executions.add(1); + actionQueue.add(() -> executions.add(2)); + }); + + actionQueue.tryCompleteActions(); + assertThat(executions).containsExactly(1); + + actionQueue.tryCompleteActions(); + assertThat(executions).isEqualTo(Arrays.asList(1, 2)); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java index 0311d150db9..87df0b844d2 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/delay/DelayedFetchLogTest.java @@ -32,6 +32,7 @@ import java.time.Duration; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -83,8 +84,7 @@ void testCompleteDelayedFetchLog() throws Exception { assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(1); assertThat(delayedFetchLogManager.watched()).isEqualTo(1); - // Produce data — appendRecordsToLog enqueues a checkAndComplete action, - // which is executed when tryCompleteActions() is called (normally by framework layer). + // Appending data enqueues completion, but does not run it under the append call. assertThat(delayedResponse.isDone()).isFalse(); CompletableFuture> future = new CompletableFuture<>(); replicaManager.appendRecordsToLog( @@ -93,11 +93,10 @@ void testCompleteDelayedFetchLog() throws Exception { Collections.singletonMap(tb, genMemoryLogRecordsByObject(DATA1)), null, future::complete); - // Simulate framework layer trigger (FlussRequestHandler calls this after invoke). - replicaManager.tryCompleteActions(); assertThat(future.get()).containsOnly(new ProduceLogResultForBucket(tb, 0, 10L)); + assertThat(delayedResponse.isDone()).isFalse(); - // The delayed fetch log should already be completed by tryCompleteActions above. + replicaManager.tryCompleteActions(); assertThat(delayedResponse.isDone()).isTrue(); assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(0); assertThat(delayedFetchLogManager.watched()).isEqualTo(0); @@ -110,55 +109,69 @@ void testCompleteDelayedFetchLog() throws Exception { } @Test - void testProduceAutoCompletesDelayedFetchLog() throws Exception { - TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1); - makeLogTableAsLeader(tb.getBucket()); - - // Set up a delayed fetch with follower-like params (LOG_END isolation, minBytes=1). - FetchLogResultForBucket preFetchResultForBucket = - new FetchLogResultForBucket(tb, MemoryLogRecords.EMPTY, 0L); + void testSuccessfulBucketCompletesWhenAnotherBucketAppendFails() throws Exception { + TableBucket successfulBucket = new TableBucket(DATA1_TABLE_ID, 1); + TableBucket failedBucket = new TableBucket(DATA1_TABLE_ID, 2); + makeLogTableAsLeader(successfulBucket.getBucket()); CompletableFuture> delayedResponse = + watchDelayedFetch(successfulBucket); + + Map entries = new HashMap<>(); + entries.put(successfulBucket, genMemoryLogRecordsByObject(DATA1)); + entries.put(failedBucket, genMemoryLogRecordsByObject(DATA1)); + CompletableFuture> produceResponse = new CompletableFuture<>(); - DelayedFetchLog delayedFetchLog = - createDelayedFetchLogRequest( - tb, - 1, // minFetchBytes = 1, like follower fetch - Duration.ofMinutes(3).toMillis(), - new FetchBucketStatus( - new FetchReqInfo(150001L, 0L, Integer.MAX_VALUE), - new LogOffsetMetadata(0L, 0L, 0), - preFetchResultForBucket), - delayedResponse::complete); - DelayedOperationManager delayedFetchLogManager = - replicaManager.getDelayedFetchLogManager(); - DelayedTableBucketKey delayedTableBucketKey = new DelayedTableBucketKey(tb); - delayedFetchLogManager.tryCompleteElseWatch( - delayedFetchLog, Collections.singletonList(delayedTableBucketKey)); - assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(1); - assertThat(delayedResponse.isDone()).isFalse(); + replicaManager.appendRecordsToLog(20000, 1, entries, null, produceResponse::complete); + + List produceResults = produceResponse.get(); + assertThat(produceResults).hasSize(2); + assertThat(produceResults) + .filteredOn(result -> result.getTableBucket().equals(successfulBucket)) + .hasSize(1) + .allSatisfy(result -> assertThat(result.succeeded()).isTrue()); + assertThat(produceResults) + .filteredOn(result -> result.getTableBucket().equals(failedBucket)) + .hasSize(1) + .allSatisfy(result -> assertThat(result.failed()).isTrue()); + assertThat(delayedResponse).isNotDone(); + + replicaManager.tryCompleteActions(); - // Produce with acks=1 — response returns immediately, action enqueued. + assertThat(delayedResponse).isDone(); + assertThat(replicaManager.getDelayedFetchLogManager().numDelayed()).isZero(); + } + + @Test + void testDrainCompletesDelayedFetchesForMultipleBuckets() throws Exception { + TableBucket firstBucket = new TableBucket(DATA1_TABLE_ID, 1); + TableBucket secondBucket = new TableBucket(DATA1_TABLE_ID, 2); + makeLogTableAsLeader(firstBucket.getBucket()); + makeLogTableAsLeader(secondBucket.getBucket()); + CompletableFuture> firstResponse = + watchDelayedFetch(firstBucket); + CompletableFuture> secondResponse = + watchDelayedFetch(secondBucket); + + Map entries = new HashMap<>(); + entries.put(firstBucket, genMemoryLogRecordsByObject(DATA1)); + entries.put(secondBucket, genMemoryLogRecordsByObject(DATA1)); CompletableFuture> produceResponse = new CompletableFuture<>(); - replicaManager.appendRecordsToLog( - 20000, - 1, - Collections.singletonMap(tb, genMemoryLogRecordsByObject(DATA1)), - null, - produceResponse::complete); - // Simulate framework layer trigger (FlussRequestHandler calls this after invoke). - replicaManager.tryCompleteActions(); - assertThat(produceResponse.get()).containsOnly(new ProduceLogResultForBucket(tb, 0, 10L)); - // The delayed fetch should have been auto-completed by tryCompleteActions. - assertThat(delayedResponse.isDone()).isTrue(); - assertThat(delayedFetchLogManager.numDelayed()).isEqualTo(0); + replicaManager.appendRecordsToLog(20000, 1, entries, null, produceResponse::complete); - Map result = delayedResponse.get(); - FetchLogResultForBucket resultForBucket = result.get(tb); - assertThat(resultForBucket).isNotNull(); - assertLogRecordsEquals(DATA1_ROW_TYPE, resultForBucket.records(), DATA1); + assertThat(produceResponse.get()) + .hasSize(2) + .allSatisfy(result -> assertThat(result.succeeded()).isTrue()); + assertThat(firstResponse).isNotDone(); + assertThat(secondResponse).isNotDone(); + + replicaManager.tryCompleteActions(); + + assertThat(firstResponse).isDone(); + assertThat(secondResponse).isDone(); + assertThat(replicaManager.getDelayedFetchLogManager().numDelayed()).isZero(); } @Test @@ -219,4 +232,28 @@ private DelayedFetchLog createDelayedFetchLogRequest( TestingMetricGroups.TABLET_SERVER_METRICS, null); } + + private CompletableFuture> watchDelayedFetch( + TableBucket tableBucket) { + FetchLogResultForBucket previousResult = + new FetchLogResultForBucket(tableBucket, MemoryLogRecords.EMPTY, 0L); + CompletableFuture> response = + new CompletableFuture<>(); + DelayedFetchLog delayedFetchLog = + createDelayedFetchLogRequest( + tableBucket, + 1, + Duration.ofMinutes(3).toMillis(), + new FetchBucketStatus( + new FetchReqInfo(150001L, 0L, Integer.MAX_VALUE), + new LogOffsetMetadata(0L, 0L, 0), + previousResult), + response::complete); + replicaManager + .getDelayedFetchLogManager() + .tryCompleteElseWatch( + delayedFetchLog, + Collections.singletonList(new DelayedTableBucketKey(tableBucket))); + return response; + } } From 936fd3f39e9e6fe595ed6bf8e30f0e2b5185ffe3 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:18:03 +0800 Subject: [PATCH 07/10] [kafka] Add independently testable Produce protocol handling Validate non-idempotent Produce v3-v11 and isolate invalid partitions before invoking a protocol-independent backend. Preserve partition order, acknowledgements and owned record data. Leave production registration to the append integration PR. Validation: Java 11, mvn -o -pl fluss-kafka spotless:apply clean verify; 69 unit tests and 2 integration tests passed, including Checkstyle, Spotless and RAT. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 621/621 AI-Contributed/UT: 366/366 --- .../kafka/api/produce/ProduceHandler.java | 284 ++++++++++++++ .../backend/produce/KafkaProduceBackend.java | 29 ++ .../backend/produce/KafkaProduceCommand.java | 197 ++++++++++ .../backend/produce/KafkaProduceResult.java | 111 ++++++ .../fluss/kafka/KafkaProduceProtocolTest.java | 366 ++++++++++++++++++ 5 files changed, 987 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java new file mode 100644 index 00000000000..0eb14b26ff2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/produce/ProduceHandler.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InvalidRequestException; +import org.apache.kafka.common.errors.InvalidRequiredAcksException; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.BaseRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements non-idempotent Kafka Produce versions 3 through 11. */ +@Internal +public final class ProduceHandler implements KafkaApiHandler { + + private static final short MIN_SUPPORTED_VERSION = 3; + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec(ApiKeys.PRODUCE, MIN_SUPPORTED_VERSION, (short) 11, true); + + private final KafkaProduceBackend backend; + + /** Creates a non-idempotent Produce handler. */ + public ProduceHandler(KafkaProduceBackend backend) { + this.backend = checkNotNull(backend); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ProduceRequest request) { + validateRequest(request); + List topics = new ArrayList<>(); + Map failures = new HashMap<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + try { + if (!Topic.isValid(topic.name())) { + throw new InvalidTopicException("Invalid Kafka topic name " + topic.name()); + } + if (partition.index() < 0) { + throw new InvalidRequestException("Negative Kafka partition ID."); + } + partitions.add( + new PartitionWrite( + partition.index(), + copyRecords(request.version(), partition.records()))); + } catch (RuntimeException failure) { + failures.put( + new TopicPartition(topic.name(), partition.index()), + failedPartition(partition.index(), failure)); + } + } + if (!partitions.isEmpty()) { + topics.add(new TopicWrite(topic.name(), partitions)); + } + } + KafkaProduceCommand command = + new KafkaProduceCommand( + request.acks(), + request.timeout(), + topics, + context.listenerName(), + clientAddress(context.remoteAddress())); + CompletableFuture result; + try { + result = + topics.isEmpty() + ? CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList())) + : checkNotNull(backend.write(command)); + } catch (RuntimeException failure) { + result = new CompletableFuture<>(); + result.completeExceptionally(failure); + } + return result.handle( + (written, failure) -> { + Map results = new HashMap<>(failures); + if (failure == null && written != null) { + for (TopicResult topic : written.topics()) { + for (PartitionResult partition : topic.partitions()) { + results.putIfAbsent( + new TopicPartition( + topic.topicName(), partition.partitionId()), + partition); + } + } + } + List ordered = new ArrayList<>(); + for (TopicProduceData topic : request.data().topicData()) { + List partitions = new ArrayList<>(); + for (PartitionProduceData partition : topic.partitionData()) { + TopicPartition key = + new TopicPartition(topic.name(), partition.index()); + PartitionResult value = results.get(key); + if (value == null) { + value = + failedPartition( + partition.index(), + failure == null + ? new IllegalStateException( + "Produce backend omitted this partition.") + : failure); + } + partitions.add(value); + } + ordered.add(new TopicResult(topic.name(), partitions)); + } + return toResponse(new KafkaProduceResult(ordered)); + }); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + while (failure instanceof CompletionException && failure.getCause() != null) { + failure = failure.getCause(); + } + return new PartitionResult( + partitionId, Errors.forException(failure), -1L, failure.getMessage()); + } + + private static void validateRequest(ProduceRequest request) { + Set names = new HashSet<>(); + for (TopicProduceData topic : request.data().topicData()) { + if (!names.add(topic.name())) { + throw new InvalidRequestException("Duplicate Kafka topic in Produce request."); + } + Set partitions = new HashSet<>(); + for (PartitionProduceData partition : topic.partitionData()) { + if (!partitions.add(partition.index())) { + throw new InvalidRequestException( + "Duplicate Kafka partition in Produce request."); + } + } + } + if (request.transactionalId() != null) { + throw new InvalidRequestException( + "Transactional Produce is not supported by the Fluss Kafka compatibility layer."); + } + if (request.acks() != -1 && request.acks() != 0 && request.acks() != 1) { + throw new InvalidRequiredAcksException("Invalid required acks " + request.acks()); + } + } + + private static List copyRecords( + short version, BaseRecords baseRecords) { + if (!(baseRecords instanceof Records)) { + throw new InvalidRequestException("Unsupported Kafka records representation."); + } + ProduceRequest.validateRecords(version, baseRecords); + Records records = (Records) baseRecords; + if (records.sizeInBytes() == 0) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + List copied = new ArrayList<>(); + int validBytes = 0; + for (RecordBatch batch : records.batches()) { + validBytes += batch.sizeInBytes(); + batch.ensureValid(); + if (batch.hasProducerId() || batch.isTransactional() || batch.isControlBatch()) { + throw new InvalidRequestException( + "Idempotent, transactional, and control record batches are not supported."); + } + for (org.apache.kafka.common.record.Record record : batch) { + record.ensureValid(); + copied.add( + new KafkaProduceCommand.Record( + record.timestamp(), + copyBuffer(record.hasKey() ? record.key() : null), + copyBuffer(record.hasValue() ? record.value() : null), + copyHeaders(record.headers()))); + } + } + if (copied.isEmpty() || validBytes != records.sizeInBytes()) { + throw new InvalidRequestException("Empty or truncated Kafka record batch."); + } + return copied; + } + + private static ProduceResponse toResponse(KafkaProduceResult result) { + ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(0); + for (TopicResult topic : result.topics()) { + ProduceResponseData.TopicProduceResponse topicResponse = + new ProduceResponseData.TopicProduceResponse().setName(topic.topicName()); + for (PartitionResult partition : topic.partitions()) { + topicResponse + .partitionResponses() + .add( + new ProduceResponseData.PartitionProduceResponse() + .setIndex(partition.partitionId()) + .setErrorCode(partition.error().code()) + .setBaseOffset(partition.baseOffset()) + .setLogAppendTimeMs(-1L) + .setLogStartOffset(-1L) + .setErrorMessage(partition.errorMessage())); + } + data.responses().add(topicResponse); + } + return new ProduceResponse(data); + } + + private static List copyHeaders(Header[] headers) { + List copied = new ArrayList<>(headers.length); + for (Header header : headers) { + copied.add(new RecordHeader(header.key(), header.value())); + } + return copied; + } + + private static byte[] copyBuffer(ByteBuffer buffer) { + if (buffer == null) { + return null; + } + ByteBuffer duplicate = buffer.duplicate(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return bytes; + } + + private static InetAddress clientAddress(SocketAddress remoteAddress) { + if (remoteAddress instanceof InetSocketAddress) { + return ((InetSocketAddress) remoteAddress).getAddress(); + } + return null; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java new file mode 100644 index 00000000000..3b8428c5670 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceBackend.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import java.util.concurrent.CompletableFuture; + +/** Narrow backend used by the Kafka Produce API. */ +@Internal +public interface KafkaProduceBackend { + /** Writes copied Kafka records through the native Fluss write path. */ + CompletableFuture write(KafkaProduceCommand command); +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java new file mode 100644 index 00000000000..722f3d3d245 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceCommand.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import javax.annotation.Nullable; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Protocol-independent write command used by the Kafka Produce backend. */ +@Internal +public final class KafkaProduceCommand { + + private final short acks; + private final int timeoutMs; + private final List topics; + private final String listenerName; + private final @Nullable InetAddress clientAddress; + + /** Creates a Kafka write command. */ + public KafkaProduceCommand( + short acks, + int timeoutMs, + List topics, + String listenerName, + @Nullable InetAddress clientAddress) { + this.acks = acks; + this.timeoutMs = timeoutMs; + this.topics = immutableCopy(topics); + this.listenerName = checkNotNull(listenerName); + this.clientAddress = clientAddress; + } + + /** Returns Kafka required acknowledgements. */ + public short acks() { + return acks; + } + + /** Returns the Produce timeout in milliseconds. */ + public int timeoutMs() { + return timeoutMs; + } + + /** Returns the topic writes in request order. */ + public List topics() { + return topics; + } + + /** Returns the listener that received the request. */ + public String listenerName() { + return listenerName; + } + + /** Returns the client network address when available. */ + public @Nullable InetAddress clientAddress() { + return clientAddress; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Records addressed to one Kafka topic. */ + @Internal + public static final class TopicWrite { + private final String topicName; + private final List partitions; + + /** Creates the writes for one topic. */ + public TopicWrite(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns partition writes in request order. */ + public List partitions() { + return partitions; + } + } + + /** Records addressed to one Kafka partition. */ + @Internal + public static final class PartitionWrite { + private final int partitionId; + private final List records; + + /** Creates the writes for one partition. */ + public PartitionWrite(int partitionId, List records) { + this.partitionId = partitionId; + this.records = immutableCopy(records); + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns copied records in append order. */ + public List records() { + return records; + } + } + + /** A copied Kafka record whose lifetime is independent of the network request buffer. */ + @Internal + public static final class Record { + private final long timestamp; + private final @Nullable byte[] key; + private final @Nullable byte[] value; + private final List headers; + + /** Creates a copied Kafka record. */ + public Record( + long timestamp, + @Nullable byte[] key, + @Nullable byte[] value, + List headers) { + this.timestamp = timestamp; + this.key = copyNullable(key); + this.value = copyNullable(value); + this.headers = immutableCopy(headers); + } + + /** Returns the Kafka record timestamp. */ + public long timestamp() { + return timestamp; + } + + /** Returns a copy of the nullable Kafka record key. */ + public @Nullable byte[] key() { + return copyNullable(key); + } + + /** Returns a copy of the nullable Kafka record value. */ + public @Nullable byte[] value() { + return copyNullable(value); + } + + /** Returns the copied Kafka headers in record order. */ + public List headers() { + return headers; + } + + private static @Nullable byte[] copyNullable(@Nullable byte[] value) { + return value == null ? null : value.clone(); + } + } + + /** A copied Kafka record header. */ + @Internal + public static final class RecordHeader { + private final String name; + private final @Nullable byte[] value; + + /** Creates a copied Kafka record header. */ + public RecordHeader(String name, @Nullable byte[] value) { + this.name = checkNotNull(name); + this.value = value == null ? null : value.clone(); + } + + /** Returns the header name. */ + public String name() { + return name; + } + + /** Returns a copy of the nullable header value. */ + public @Nullable byte[] value() { + return value == null ? null : value.clone(); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java new file mode 100644 index 00000000000..218c566bde4 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/KafkaProduceResult.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; + +import org.apache.kafka.common.protocol.Errors; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Result of a Kafka Produce backend invocation. */ +@Internal +public final class KafkaProduceResult { + private final List topics; + + /** Creates a Produce result. */ + public KafkaProduceResult(List topics) { + this.topics = immutableCopy(topics); + } + + /** Returns topic results in request order. */ + public List topics() { + return topics; + } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList<>(checkNotNull(values))); + } + + /** Results for one topic. */ + @Internal + public static final class TopicResult { + private final String topicName; + private final List partitions; + + /** Creates the result for one topic. */ + public TopicResult(String topicName, List partitions) { + this.topicName = checkNotNull(topicName); + this.partitions = immutableCopy(partitions); + } + + /** Returns the Kafka topic name. */ + public String topicName() { + return topicName; + } + + /** Returns the partition results in request order. */ + public List partitions() { + return partitions; + } + } + + /** Result for one partition. */ + @Internal + public static final class PartitionResult { + private final int partitionId; + private final Errors error; + private final long baseOffset; + private final @Nullable String errorMessage; + + /** Creates the result for one partition. */ + public PartitionResult( + int partitionId, Errors error, long baseOffset, @Nullable String errorMessage) { + this.partitionId = partitionId; + this.error = checkNotNull(error); + this.baseOffset = baseOffset; + this.errorMessage = errorMessage; + } + + /** Returns the Kafka partition ID. */ + public int partitionId() { + return partitionId; + } + + /** Returns the Kafka protocol error. */ + public Errors error() { + return error; + } + + /** Returns the first appended offset, or {@code -1} on failure. */ + public long baseOffset() { + return baseOffset; + } + + /** Returns an optional diagnostic error message. */ + public @Nullable String errorMessage() { + return errorMessage; + } + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java new file mode 100644 index 00000000000..bf043823689 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceProtocolTest.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.kafka.api.produce.ProduceHandler; +import org.apache.fluss.kafka.backend.produce.KafkaProduceBackend; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.rpc.netty.server.RequestChannel; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; + +import org.apache.kafka.common.compress.Compression; +import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.message.ProduceRequestData.PartitionProduceData; +import org.apache.kafka.common.message.ProduceRequestData.TopicProduceData; +import org.apache.kafka.common.message.ProduceResponseData.PartitionProduceResponse; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Protocol tests independent of the Fluss append and record conversion backends. */ +class KafkaProduceProtocolTest { + + @Test + void testSupportedVersionsCopyRecordsAndForwardContext() { + for (short version = 3; version <= 11; version++) { + AtomicReference copied = new AtomicReference<>(); + ProduceResponse response = + dispatch( + request(version, (short) -1, partition(0, records())), + command -> { + copied.set(command); + return successful(command); + }); + assertThat(response.errorCounts()).containsOnlyKeys(Errors.NONE); + assertThat( + response.data() + .responses() + .find("topic") + .partitionResponses() + .get(0) + .baseOffset()) + .isEqualTo(42L); + KafkaProduceCommand command = copied.get(); + assertThat(command.acks()).isEqualTo((short) -1); + assertThat(command.timeoutMs()).isEqualTo(4321); + assertThat(command.listenerName()).isEqualTo("KAFKA"); + KafkaProduceCommand.Record record = + command.topics().get(0).partitions().get(0).records().get(0); + assertThat(record.timestamp()).isEqualTo(123L); + assertThat(record.key()).containsExactly((byte) 1); + assertThat(record.value()).containsExactly((byte) 2); + assertThat(record.headers()).hasSize(1); + assertThat(record.headers().get(0).value()).isNull(); + byte[] key = record.key(); + key[0] = 9; + assertThat(record.key()).containsExactly((byte) 1); + } + } + + @Test + void testCorruptPartitionDoesNotSuppressValidPartition() { + MemoryRecords corrupt = records(); + corrupt.buffer().put(corrupt.sizeInBytes() - 1, (byte) 99); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, corrupt), + partition(1, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + assertThat(command.topics().get(0).partitions().get(0).partitionId()) + .isEqualTo(1); + return successful(command); + }); + assertErrors(response, Errors.CORRUPT_MESSAGE, Errors.NONE); + } + + @Test + void testInvalidTopicIsIsolated() { + ProduceRequest request = request((short) 11, (short) 1, partition(0, records())); + request.data() + .topicData() + .add( + new TopicProduceData() + .setName("bad/name") + .setPartitionData( + Collections.singletonList(partition(0, records())))); + ProduceResponse response = + dispatch( + request, + command -> { + assertThat(command.topics()).hasSize(1); + return successful(command); + }); + assertThat(response.errorCounts()) + .containsEntry(Errors.NONE, 1) + .containsEntry(Errors.INVALID_TOPIC_EXCEPTION, 1); + } + + @Test + void testEmptyNegativeAndIdempotentPartitionsAreRejectedLocally() { + MemoryRecords idempotent = + MemoryRecords.withIdempotentRecords( + Compression.NONE, 10L, (short) 0, 0, new SimpleRecord(new byte[] {1})); + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(-1, records()), + partition(2, idempotent), + partition(3, records())), + command -> { + assertThat(command.topics().get(0).partitions()).hasSize(1); + return successful(command); + }); + assertErrors( + response, + Errors.INVALID_RECORD, + Errors.INVALID_REQUEST, + Errors.INVALID_REQUEST, + Errors.NONE); + } + + @Test + void testRequestValidationNeverCallsBackend() { + KafkaProduceBackend unused = + command -> { + throw new AssertionError("Backend must not be called"); + }; + assertErrors( + dispatch(request((short) 11, (short) 2, partition(0, records())), unused), + Errors.INVALID_REQUIRED_ACKS); + ProduceRequest transactional = request((short) 11, (short) 1, partition(0, records())); + transactional.data().setTransactionalId("transaction"); + assertErrors( + dispatch(new ProduceRequest(transactional.data(), transactional.version()), unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch( + request( + (short) 11, + (short) 1, + partition(0, records()), + partition(0, records())), + unused), + Errors.INVALID_REQUEST); + assertErrors( + dispatch(request((short) 2, (short) 1, partition(0, records())), unused), + Errors.UNSUPPORTED_VERSION); + } + + @Test + void testBackendFailuresAndMissingResultsPreserveValidationErrors() { + for (boolean synchronous : Arrays.asList(true, false)) { + ProduceResponse response = + dispatch( + request( + (short) 11, + (short) 1, + partition(0, MemoryRecords.EMPTY), + partition(1, records())), + command -> { + if (synchronous) { + throw new TimeoutException("append timed out"); + } + CompletableFuture failed = + new CompletableFuture<>(); + failed.completeExceptionally( + new TimeoutException("append timed out")); + return failed; + }); + assertErrors(response, Errors.INVALID_RECORD, Errors.REQUEST_TIMED_OUT); + } + assertErrors( + dispatch( + request((short) 11, (short) 1, partition(0, records())), + command -> + CompletableFuture.completedFuture( + new KafkaProduceResult(Collections.emptyList()))), + Errors.UNKNOWN_SERVER_ERROR); + } + + @Test + void testAcksZeroSuccessAndFailureReleaseBufferWithoutSendingResponse() { + for (boolean failure : Arrays.asList(false, true)) { + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + ProduceRequest body = request((short) 11, (short) 0, partition(0, records())); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, (short) 11, "producer", 1); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + CompletableFuture pending = new CompletableFuture<>(); + try { + channel.writeInbound(buffer); + KafkaRequest parsed = (KafkaRequest) requests.pollRequest(1000); + dispatcher(command -> pending) + .dispatch(parsed) + .whenComplete( + (response, error) -> { + if (error == null) { + parsed.complete(response); + } else { + parsed.fail(error); + } + }); + assertThat(parsed.future()).isNotDone(); + if (failure) { + pending.completeExceptionally(new TimeoutException("failure")); + } else { + pending.complete( + new KafkaProduceResult( + Collections.singletonList( + new TopicResult( + "topic", + Collections.singletonList( + new PartitionResult( + 0, Errors.NONE, 42L, null)))))); + } + channel.runPendingTasks(); + assertThat(parsed.future()).isDone(); + assertThat((Object) channel.readOutbound()).isNull(); + assertThat(buffer.refCnt()).isZero(); + } finally { + channel.finishAndReleaseAll(); + } + } + } + + private static ProduceResponse dispatch(ProduceRequest body, KafkaProduceBackend backend) { + ByteBuf buffer = Unpooled.buffer(1); + RequestHeader header = new RequestHeader(ApiKeys.PRODUCE, body.version(), "producer", 1); + KafkaRequest request = + new KafkaRequest( + ApiKeys.PRODUCE, + body.version(), + header, + body, + "KAFKA", + buffer, + new TestingChannelHandlerContext(), + new CompletableFuture<>()); + buffer.release(); + request.complete(dispatcher(backend).dispatch(request).join()); + ByteBuf response = request.responseBuffer(); + try { + ByteBuffer bytes = response.nioBuffer(); + ResponseHeader.parse(bytes, header.toResponseHeader().headerVersion()); + return (ProduceResponse) + AbstractResponse.parseResponse(ApiKeys.PRODUCE, bytes, body.version()); + } finally { + response.release(); + } + } + + private static KafkaRequestDispatcher dispatcher(KafkaProduceBackend backend) { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ProduceHandler(backend)); + registry.freeze(); + return new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); + } + + private static CompletableFuture successful(KafkaProduceCommand command) { + List topics = new ArrayList<>(); + for (KafkaProduceCommand.TopicWrite topic : command.topics()) { + List partitions = new ArrayList<>(); + for (KafkaProduceCommand.PartitionWrite partition : topic.partitions()) { + partitions.add( + new PartitionResult(partition.partitionId(), Errors.NONE, 42L, null)); + } + topics.add(new TopicResult(topic.topicName(), partitions)); + } + return CompletableFuture.completedFuture(new KafkaProduceResult(topics)); + } + + private static ProduceRequest request( + short version, short acks, PartitionProduceData... partitions) { + TopicProduceData topic = + new TopicProduceData().setName("topic").setPartitionData(Arrays.asList(partitions)); + return new ProduceRequest( + new ProduceRequestData() + .setAcks(acks) + .setTimeoutMs(4321) + .setTopicData( + new ProduceRequestData.TopicProduceDataCollection( + Collections.singletonList(topic).iterator())), + version); + } + + private static PartitionProduceData partition(int index, MemoryRecords records) { + return new PartitionProduceData().setIndex(index).setRecords(records); + } + + private static MemoryRecords records() { + return MemoryRecords.withRecords( + RecordBatch.MAGIC_VALUE_V2, + 0L, + Compression.NONE, + new SimpleRecord( + 123L, + new byte[] {1}, + new byte[] {2}, + new Header[] {new RecordHeader("header", null)})); + } + + private static void assertErrors(ProduceResponse response, Errors... errors) { + assertThat(response.data().responses().find("topic").partitionResponses()) + .extracting(PartitionProduceResponse::errorCode) + .containsExactly(Arrays.stream(errors).map(Errors::code).toArray(Short[]::new)); + } +} From 503136295d6bbaed81b7e7845dceb2d64db2d9d4 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:21:35 +0800 Subject: [PATCH 08/10] [kafka] Transcode mapped raw and string records to Arrow Reuse the DDL contract to assemble key, value, timestamp and ordered headers into physical rows. Decode strings with strict UTF-8 validation and preserve nullable values. Return owned heap log bytes and recycle Arrow writers on success and failure. Validation: Java 11, mvn -o -pl fluss-kafka spotless:apply clean verify; 75 unit tests and 2 integration tests passed, including Checkstyle, Spotless and RAT. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 314/314 AI-Contributed/UT: 326/326 --- .../transcode/ArrowKafkaRecordTranscoder.java | 87 +++++ .../transcode/FlussArrowRecordEncoder.java | 71 ++++ .../KafkaRecordEncodingException.java | 35 ++ .../transcode/KafkaRecordTranscoder.java | 32 ++ .../kafka/transcode/KafkaRowAssembler.java | 89 +++++ .../ArrowKafkaRecordTranscoderTest.java | 326 ++++++++++++++++++ 6 files changed, 640 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java new file mode 100644 index 00000000000..717f5cd33ff --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoder.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericRow; + +import javax.annotation.Nullable; + +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** Converts raw/string Kafka records using the DDL mapping into owned Fluss Arrow log bytes. */ +@Internal +public final class ArrowKafkaRecordTranscoder implements KafkaRecordTranscoder { + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final FlussArrowRecordEncoder arrowRecordEncoder = new FlussArrowRecordEncoder(); + + @Override + public BytesView transcode(List records, TableInfo tableInfo) throws Exception { + checkArgument(!records.isEmpty(), "Cannot transcode an empty Kafka partition."); + KafkaTopicSchema schema = schemaResolver.resolve(tableInfo.toTableDescriptor()); + KafkaRowAssembler assembler = new KafkaRowAssembler(schema); + List rows = new ArrayList<>(records.size()); + for (Record record : records) { + Object[] key = + schema.keyFormat() == null + ? new Object[0] + : new Object[] {decode(schema.keyFormat(), record.key())}; + rows.add( + assembler.assemble( + key, + new Object[] {decode(schema.valueFormat(), record.value())}, + record.timestamp(), + record.headers())); + } + return arrowRecordEncoder.encode(rows, tableInfo); + } + + private static @Nullable Object decode(KafkaDataFormat format, @Nullable byte[] bytes) { + if (bytes == null || format == KafkaDataFormat.RAW) { + return bytes; + } + if (format != KafkaDataFormat.STRING) { + throw new KafkaRecordEncodingException("Unsupported Kafka data format: " + format); + } + try { + return BinaryString.fromString( + StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString()); + } catch (CharacterCodingException e) { + throw new KafkaRecordEncodingException("Kafka string field is not valid UTF-8.", e); + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java new file mode 100644 index 00000000000..f74680ccaa2 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/FlussArrowRecordEncoder.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.memory.UnmanagedPagedOutputView; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.MemoryLogRecordsArrowBuilder; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.arrow.ArrowWriter; +import org.apache.fluss.row.arrow.ArrowWriterPool; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator; +import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator; + +import java.util.List; + +/** Encodes assembled physical Fluss rows into one native Arrow log batch. */ +@Internal +public final class FlussArrowRecordEncoder { + + private static final int INITIAL_PAGE_SIZE = 4096; + + /** Encodes all rows using the table's current schema ID and Arrow compression settings. */ + public BytesView encode(List rows, TableInfo tableInfo) throws Exception { + try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); + ArrowWriterPool provider = new ArrowWriterPool(allocator)) { + ArrowWriter writer = + provider.getOrCreateWriter( + tableInfo.getTableId(), + tableInfo.getSchemaId(), + Integer.MAX_VALUE, + tableInfo.getRowType(), + tableInfo.getTableConfig().getArrowCompressionInfo()); + long epoch = writer.getEpoch(); + try { + MemoryLogRecordsArrowBuilder builder = + MemoryLogRecordsArrowBuilder.builder( + tableInfo.getSchemaId(), + writer, + new UnmanagedPagedOutputView(INITIAL_PAGE_SIZE), + true, + null); + for (GenericRow row : rows) { + builder.append(ChangeType.APPEND_ONLY, row); + } + // The output view owns heap pages. The result remains valid after the Arrow + // writer and allocator close, including across asynchronous append completion. + return builder.build(); + } finally { + writer.recycle(epoch); + } + } + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java new file mode 100644 index 00000000000..b48dd9d894f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordEncodingException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that Kafka record bytes cannot be decoded using the configured data format. */ +@Internal +public final class KafkaRecordEncodingException extends IllegalArgumentException { + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message) { + super(message); + } + + /** Creates a record encoding exception. */ + public KafkaRecordEncodingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java new file mode 100644 index 00000000000..eeff2f3d14f --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRecordTranscoder.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.record.bytesview.BytesView; + +import java.util.List; + +/** Converts copied Kafka records into the native Fluss log representation. */ +@Internal +public interface KafkaRecordTranscoder { + /** Transcodes records according to the target Fluss table schema and log format. */ + BytesView transcode(List records, TableInfo tableInfo) throws Exception; +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java new file mode 100644 index 00000000000..4ebd506dc3c --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/transcode/KafkaRowAssembler.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.schema.KafkaFieldProjection; +import org.apache.fluss.kafka.schema.KafkaTopicSchema; +import org.apache.fluss.row.BinaryString; +import org.apache.fluss.row.GenericArray; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.TimestampLtz; + +import java.util.List; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Assembles decoded Kafka key, value, and metadata into a physical Fluss row. */ +@Internal +public final class KafkaRowAssembler { + + private final KafkaTopicSchema topicSchema; + + /** Creates an assembler for the resolved topic schema. */ + public KafkaRowAssembler(KafkaTopicSchema topicSchema) { + this.topicSchema = checkNotNull(topicSchema); + } + + /** Assembles one Fluss row. */ + public GenericRow assemble( + Object[] keyValues, Object[] valueValues, long timestamp, List headers) { + GenericRow row = new GenericRow(topicSchema.rowType().getFieldCount()); + setProjectedFields(row, topicSchema.keyProjection(), keyValues); + setProjectedFields(row, topicSchema.valueProjection(), valueValues); + if (topicSchema.timestampPosition() >= 0) { + row.setField(topicSchema.timestampPosition(), TimestampLtz.fromEpochMillis(timestamp)); + } + if (topicSchema.headersPosition() >= 0) { + row.setField(topicSchema.headersPosition(), toHeaders(headers)); + } + return row; + } + + private static void setProjectedFields( + GenericRow row, KafkaFieldProjection projection, Object[] values) { + if (values.length != projection.size()) { + throw new IllegalArgumentException( + "Kafka decoder returned " + + values.length + + " fields for a projection of " + + projection.size() + + "."); + } + for (int i = 0; i < values.length; i++) { + Object value = values[i]; + if (value == null && !projection.dataTypeAt(i).isNullable()) { + throw new KafkaRecordEncodingException( + "Kafka record cannot populate NOT NULL field '" + + projection.nameAt(i) + + "' with null."); + } + row.setField(projection.positionAt(i), value); + } + } + + private static GenericArray toHeaders(List headers) { + Object[] rows = new Object[headers.size()]; + for (int i = 0; i < headers.size(); i++) { + RecordHeader header = headers.get(i); + rows[i] = GenericRow.of(BinaryString.fromString(header.name()), header.value()); + } + return new GenericArray(rows); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java new file mode 100644 index 00000000000..6936b5c8d61 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/transcode/ArrowKafkaRecordTranscoderTest.java @@ -0,0 +1,326 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.transcode; + +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.RecordHeader; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.row.GenericRow; +import org.apache.fluss.row.InternalArray; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.CloseableIterator; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; + +/** Verifies mapped Arrow records after transcoding resources have closed. */ +class ArrowKafkaRecordTranscoderTest { + private final ArrowKafkaRecordTranscoder transcoder = new ArrowKafkaRecordTranscoder(); + + @Test + void testRawAndStringMappingsUsePhysicalColumnPositions() throws Exception { + for (boolean string : new boolean[] {false, true}) { + TableInfo table = envelope(string); + Record record = + new Record( + 123L, + bytes("键"), + bytes("message"), + Arrays.asList( + new RecordHeader("duplicate", bytes("first")), + new RecordHeader("duplicate", null))); + BytesView encoded = transcoder.transcode(Arrays.asList(record, record), table); + read( + encoded, + table, + 2, + row -> { + if (string) { + assertThat(row.getString(0).toString()).isEqualTo("message"); + assertThat(row.getString(3).toString()).isEqualTo("键"); + } else { + assertThat(row.getBytes(0)).isEqualTo(bytes("message")); + assertThat(row.getBytes(3)).isEqualTo(bytes("键")); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + InternalArray headers = row.getArray(2); + assertThat(headers.size()).isEqualTo(2); + assertThat(headers.getRow(0, 2).getString(0).toString()) + .isEqualTo("duplicate"); + assertThat(headers.getRow(0, 2).getBytes(1)).isEqualTo(bytes("first")); + assertThat(headers.getRow(1, 2).isNullAt(1)).isTrue(); + }); + } + } + + @Test + void testNullsAndEmptyBytesRemainDistinct() throws Exception { + TableInfo table = envelope(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, null, null, Collections.emptyList())), + table), + table, + 1, + row -> { + assertThat(row.isNullAt(0)).isTrue(); + assertThat(row.isNullAt(3)).isTrue(); + assertThat(row.getArray(2).size()).isZero(); + }); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, new byte[0], new byte[0], Collections.emptyList())), + table), + table, + 1, + row -> { + assertThat(row.getBytes(0)).isEmpty(); + assertThat(row.getBytes(3)).isEmpty(); + }); + } + + @Test + void testUnmappedKeyIsIgnoredAndMixedFormatsWork() throws Exception { + TableInfo valueOnly = valueTable(false); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + new byte[] {(byte) 0xff}, + bytes("value"), + Collections.emptyList())), + valueOnly), + valueOnly, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("value")); + Schema schema = + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("id", DataTypes.STRING()) + .build(); + TableInfo mixed = + table( + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .build()); + read( + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + bytes("key"), + new byte[] {(byte) 0xff}, + Collections.emptyList())), + mixed), + mixed, + 1, + row -> { + assertThat(row.getString(1).toString()).isEqualTo("key"); + assertThat(row.getBytes(0)).containsExactly((byte) 0xff); + }); + } + + @Test + void testRejectsMalformedUtf8AndNotNullViolationsThenRemainsUsable() throws Exception { + TableInfo table = valueTable(false); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + new byte[] {(byte) 0xc3, 0x28}, + Collections.emptyList())), + table)) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("UTF-8"); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, null, null, Collections.emptyList())), + valueTable(true))) + .isInstanceOf(KafkaRecordEncodingException.class) + .hasMessageContaining("NOT NULL"); + read( + transcoder.transcode( + Collections.singletonList( + new Record(1L, null, bytes("ok"), Collections.emptyList())), + table), + table, + 1, + row -> assertThat(row.getString(0).toString()).isEqualTo("ok")); + } + + @Test + void testRejectsInvalidMappingAndEmptyPartition() { + TableInfo invalid = + table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING()) + .build()) + .distributedBy(1) + .build()); + assertThatThrownBy( + () -> + transcoder.transcode( + Collections.singletonList( + new Record( + 1L, + null, + bytes("value"), + Collections.emptyList())), + invalid)) + .isInstanceOf(KafkaTopicSchemaException.class); + assertThatThrownBy(() -> transcoder.transcode(Collections.emptyList(), valueTable(false))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("empty"); + } + + @Test + void testArrowWriterFailureClosesAllocatorWithoutLeaking() { + Throwable failure = + catchThrowable( + () -> + new FlussArrowRecordEncoder() + .encode( + Collections.singletonList(GenericRow.of(123)), + valueTable(false))); + assertThat(failure).isInstanceOf(ClassCastException.class); + assertThat(failure.getSuppressed()).isEmpty(); + } + + private static TableInfo valueTable(boolean notNull) { + return table( + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", DataTypes.STRING().copy(!notNull)) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string") + .build()); + } + + private static TableInfo envelope(boolean string) { + Schema schema = + Schema.newBuilder() + .column("body", string ? DataTypes.STRING() : DataTypes.BYTES()) + .column("time", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "attrs", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES())))) + .column("id", string ? DataTypes.STRING() : DataTypes.BYTES()) + .build(); + return table( + TableDescriptor.builder() + .schema(schema) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty( + KafkaDataFormat.VALUE_FORMAT_CONFIG, string ? "string" : "raw") + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, string ? "string" : "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "time") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attrs") + .build()); + } + + private static TableInfo table(TableDescriptor descriptor) { + return TableInfo.of(TablePath.of("kafka", "topic"), 42L, 3, descriptor, null, 1L, 1L); + } + + private static void read( + BytesView encoded, TableInfo table, int count, Consumer verify) + throws Exception { + ByteBuf buffer = encoded.getByteBuf(); + try { + LogRecordBatch batch = + MemoryLogRecords.pointToByteBuffer(buffer.nioBuffer()) + .batches() + .iterator() + .next(); + batch.ensureValid(); + assertThat(batch.schemaId()).isEqualTo((short) table.getSchemaId()); + assertThat(batch.getRecordCount()).isEqualTo(count); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + table.getRowType(), + table.getSchemaId(), + new TestingSchemaGetter( + new SchemaInfo( + table.getSchema(), table.getSchemaId()))); + CloseableIterator records = batch.records(context)) { + for (int i = 0; i < count; i++) { + assertThat(records.hasNext()).isTrue(); + verify.accept(records.next().getRow()); + } + assertThat(records.hasNext()).isFalse(); + } + } finally { + buffer.release(); + } + } + + private static byte[] bytes(String text) { + return text.getBytes(StandardCharsets.UTF_8); + } +} From dc5fb6599b8c5e155cb17280df71a74ee34c9ffa Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 20:31:15 +0800 Subject: [PATCH 09/10] [kafka] Connect Produce to native Fluss append Register Produce v3-v11 and adapt the TabletServer gateway using the shared DDL contract and raw/string transcoder. Isolate table, conversion and bucket failures, preserve acknowledgements and offsets, and drain deferred actions immediately after the synchronous append invocation. Validation: Java 11, mvn -o -pl fluss-kafka spotless:apply clean verify; 82 unit tests and 4 integration tests passed. Three-replica integration covers native DDL, Kafka Metadata and gzip Produce with raw/string and acks=0/1/all, offsets, and Fluss readback. Checkstyle, Spotless and RAT passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 325/325 AI-Contributed/UT: 481/481 --- fluss-kafka/pom.xml | 6 + .../fluss/kafka/KafkaRequestHandler.java | 10 + .../produce/GatewayKafkaProduceBackend.java | 309 ++++++++++++++++++ .../fluss/kafka/KafkaProduceAppendITCase.java | 190 +++++++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 1 + .../GatewayKafkaProduceBackendTest.java | 290 ++++++++++++++++ 6 files changed, 806 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index cfb6e2ca31b..f84842a7c8d 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,6 +64,12 @@ + + ${project.groupId} + fluss-client + ${project.version} + test + org.apache.curator curator-test diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index de561509b66..f7c083e9abb 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -18,11 +18,14 @@ package org.apache.fluss.kafka; import org.apache.fluss.kafka.api.metadata.MetadataHandler; +import org.apache.fluss.kafka.api.produce.ProduceHandler; import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; import org.apache.fluss.kafka.backend.metadata.GatewayKafkaMetadataBackend; +import org.apache.fluss.kafka.backend.produce.GatewayKafkaProduceBackend; import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; import org.apache.fluss.kafka.error.KafkaErrorMapper; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; import org.apache.fluss.rpc.RpcGatewayService; import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.netty.server.RequestHandler; @@ -46,6 +49,13 @@ public KafkaRequestHandler( registry.register( new MetadataHandler( new GatewayKafkaMetadataBackend(service, gateway, kafkaDatabase))); + registry.register( + new ProduceHandler( + new GatewayKafkaProduceBackend( + service, + gateway, + kafkaDatabase, + new ArrowKafkaRecordTranscoder()))); registry.freeze(); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java new file mode 100644 index 00000000000..0d9e8d8b3b8 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackend.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.TopicResult; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaException; +import org.apache.fluss.kafka.schema.KafkaTopicSchemaResolver; +import org.apache.fluss.kafka.transcode.KafkaRecordEncodingException; +import org.apache.fluss.kafka.transcode.KafkaRecordTranscoder; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.bytesview.BytesView; +import org.apache.fluss.rpc.RpcGatewayService; +import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.rpc.netty.server.Session; +import org.apache.fluss.security.acl.FlussPrincipal; + +import org.apache.kafka.common.protocol.Errors; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Adapts the local TabletServer write gateway to the Kafka Produce backend contract. */ +@Internal +public final class GatewayKafkaProduceBackend implements KafkaProduceBackend { + + private final RpcGatewayService service; + private final TabletServerGateway gateway; + private final String databaseName; + private final KafkaTopicMapper topicMapper; + private final KafkaTopicSchemaResolver schemaResolver = new KafkaTopicSchemaResolver(); + private final KafkaRecordTranscoder transcoder; + + /** Creates a Produce backend backed by the local TabletServer gateway. */ + public GatewayKafkaProduceBackend( + RpcGatewayService service, + TabletServerGateway gateway, + String databaseName, + KafkaRecordTranscoder transcoder) { + this.service = checkNotNull(service); + this.gateway = checkNotNull(gateway); + this.databaseName = checkNotNull(databaseName); + this.topicMapper = new KafkaTopicMapper(databaseName); + this.transcoder = checkNotNull(transcoder); + } + + @Override + public CompletableFuture write(KafkaProduceCommand command) { + List> futures = new ArrayList<>(); + for (TopicWrite topic : command.topics()) { + futures.add(writeTopic(command, topic)); + } + CompletableFuture all = + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + return all.thenApply( + ignored -> { + List results = new ArrayList<>(); + for (CompletableFuture future : futures) { + results.add(future.join()); + } + return new KafkaProduceResult(results); + }); + } + + private CompletableFuture writeTopic( + KafkaProduceCommand command, TopicWrite topic) { + try { + TablePath path = topicMapper.toTablePath(topic.topicName()); + setCurrentSession(command); + GetTableInfoRequest request = new GetTableInfoRequest(); + request.setTablePath() + .setDatabaseName(path.getDatabaseName()) + .setTableName(path.getTableName()); + return gateway.getTableInfo(request) + .thenCompose( + response -> produceTopic(command, topic, toTableInfo(topic, response))) + .exceptionally(failure -> failedTopic(topic, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedTopic(topic, failure)); + } + } + + private CompletableFuture produceTopic( + KafkaProduceCommand command, TopicWrite topic, TableInfo tableInfo) { + // Admission must match Metadata, including when the schema changes between requests. + schemaResolver.resolve(tableInfo.toTableDescriptor()); + ProduceLogRequest request = + new ProduceLogRequest() + .setTableId(tableInfo.getTableId()) + .setAcks(command.acks()) + .setTimeoutMs(command.timeoutMs()); + Map failures = new HashMap<>(); + for (PartitionWrite partition : topic.partitions()) { + if (partition.partitionId() < 0 + || partition.partitionId() >= tableInfo.getNumBuckets()) { + failures.put( + partition.partitionId(), + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_TOPIC_OR_PARTITION, + -1L, + "Partition is outside the table bucket range.")); + continue; + } + try { + BytesView records = transcoder.transcode(partition.records(), tableInfo); + request.addBucketsReq() + .setBucketId(partition.partitionId()) + .setRecordsBytesView(records); + } catch (Exception failure) { + failures.put( + partition.partitionId(), failedPartition(partition.partitionId(), failure)); + } + } + if (request.getBucketsReqsCount() == 0) { + return CompletableFuture.completedFuture( + toTopicResult(topic, new ProduceLogResponse(), failures)); + } + try { + setCurrentSession(command); + CompletableFuture appended; + try { + appended = gateway.produceLog(request); + } finally { + // produceLog appends synchronously, but acks=-1 can finish later. Wake delayed + // fetches now so replicas can fetch the append needed to complete that response. + service.tryCompleteActions(); + } + return appended.handle( + (response, failure) -> + failure == null + ? toTopicResult(topic, response, failures) + : failedAppend(topic, failures, failure)); + } catch (Exception failure) { + return CompletableFuture.completedFuture(failedAppend(topic, failures, failure)); + } + } + + private static TopicResult failedAppend( + TopicWrite topic, Map failures, Throwable failure) { + List results = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult local = failures.get(partition.partitionId()); + results.add(local == null ? failedPartition(partition.partitionId(), failure) : local); + } + return new TopicResult(topic.topicName(), results); + } + + private TableInfo toTableInfo(TopicWrite topic, GetTableInfoResponse response) { + return TableInfo.of( + TablePath.of(databaseName, topic.topicName()), + response.getTableId(), + response.getSchemaId(), + TableDescriptor.fromJsonBytes(response.getTableJson()), + response.hasRemoteDataDir() ? response.getRemoteDataDir() : null, + response.getCreatedTime(), + response.getModifiedTime()); + } + + private static TopicResult toTopicResult( + TopicWrite topic, ProduceLogResponse response, Map failures) { + Map responses = new HashMap<>(); + for (PbProduceLogRespForBucket bucket : response.getBucketsRespsList()) { + responses.put(bucket.getBucketId(), bucket); + } + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + PartitionResult localFailure = failures.get(partition.partitionId()); + if (localFailure != null) { + partitions.add(localFailure); + continue; + } + PbProduceLogRespForBucket bucket = responses.get(partition.partitionId()); + if (bucket == null) { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.UNKNOWN_SERVER_ERROR, + -1L, + "Fluss Produce response omitted this bucket.")); + } else if (bucket.hasErrorCode() && bucket.getErrorCode() != 0) { + partitions.add( + new PartitionResult( + partition.partitionId(), + toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forCode( + bucket.getErrorCode())), + -1L, + bucket.hasErrorMessage() ? bucket.getErrorMessage() : null)); + } else { + partitions.add( + new PartitionResult( + partition.partitionId(), + Errors.NONE, + bucket.hasBaseOffset() ? bucket.getBaseOffset() : -1L, + null)); + } + } + return new TopicResult(topic.topicName(), partitions); + } + + private static TopicResult failedTopic(TopicWrite topic, Throwable failure) { + List partitions = new ArrayList<>(); + for (PartitionWrite partition : topic.partitions()) { + partitions.add(failedPartition(partition.partitionId(), failure)); + } + return new TopicResult(topic.topicName(), partitions); + } + + private static PartitionResult failedPartition(int partitionId, Throwable failure) { + Throwable cause = unwrap(failure); + Errors kafkaError = + cause instanceof KafkaTopicSchemaException + ? Errors.INVALID_TOPIC_EXCEPTION + : cause instanceof KafkaRecordEncodingException + ? Errors.CORRUPT_MESSAGE + : cause instanceof IllegalArgumentException + ? Errors.INVALID_REQUEST + : toKafkaError( + org.apache.fluss.rpc.protocol.Errors.forException( + cause)); + return new PartitionResult(partitionId, kafkaError, -1L, cause.getMessage()); + } + + private static Errors toKafkaError(org.apache.fluss.rpc.protocol.Errors error) { + switch (error) { + case NONE: + return Errors.NONE; + case TABLE_NOT_EXIST: + case UNKNOWN_TABLE_OR_BUCKET_EXCEPTION: + return Errors.UNKNOWN_TOPIC_OR_PARTITION; + case NOT_LEADER_OR_FOLLOWER: + return Errors.NOT_LEADER_OR_FOLLOWER; + case LEADER_NOT_AVAILABLE_EXCEPTION: + return Errors.LEADER_NOT_AVAILABLE; + case RECORD_TOO_LARGE_EXCEPTION: + return Errors.MESSAGE_TOO_LARGE; + case CORRUPT_MESSAGE: + case CORRUPT_RECORD_EXCEPTION: + return Errors.CORRUPT_MESSAGE; + case INVALID_REQUIRED_ACKS: + return Errors.INVALID_REQUIRED_ACKS; + case REQUEST_TIME_OUT: + return Errors.REQUEST_TIMED_OUT; + case NOT_ENOUGH_REPLICAS_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS; + case NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION: + return Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND; + case AUTHORIZATION_EXCEPTION: + return Errors.TOPIC_AUTHORIZATION_FAILED; + case LOG_STORAGE_EXCEPTION: + case STORAGE_EXCEPTION: + case DISK_WRITE_LOCKED: + return Errors.KAFKA_STORAGE_ERROR; + default: + return Errors.UNKNOWN_SERVER_ERROR; + } + } + + private void setCurrentSession(KafkaProduceCommand command) { + service.setCurrentSession( + new Session( + (short) 0, + command.listenerName(), + false, + command.clientAddress(), + FlussPrincipal.ANONYMOUS)); + } + + private static Throwable unwrap(Throwable failure) { + Throwable current = failure; + while (current instanceof CompletionException && current.getCause() != null) { + current = current.getCause(); + } + return current; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java new file mode 100644 index 00000000000..b9c1a2bdc4f --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceAppendITCase.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.ScanRecord; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Exercises native DDL, Kafka discovery and Produce, replication, and native Fluss readback. */ +class KafkaProduceAppendITCase { + private static final String DATABASE = "kafka"; + private static final byte[] KEY = "key".getBytes(StandardCharsets.UTF_8); + private static final byte[] VALUE = "value".getBytes(StandardCharsets.UTF_8); + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @ParameterizedTest + @ValueSource(strings = {"raw", "string"}) + void testPrecreatedTableRoundTripForAllAckModes(String format) throws Exception { + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + org.apache.fluss.client.admin.Admin admin = connection.getAdmin()) { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + for (String acks : Arrays.asList("0", "1", "all")) { + String topic = "produce_" + format + "_" + acks; + TablePath path = TablePath.of(DATABASE, topic); + admin.createTable(path, descriptor(format), false).get(); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + try (KafkaProducer producer = producer(acks)) { + assertThat(producer.partitionsFor(topic)).hasSize(1); + ProducerRecord record = + new ProducerRecord<>( + topic, + 0, + 123L, + KEY, + VALUE, + Arrays.asList( + new RecordHeader("source", VALUE), + new RecordHeader("source", null))); + RecordMetadata first = producer.send(record).get(30, TimeUnit.SECONDS); + RecordMetadata second = producer.send(record).get(30, TimeUnit.SECONDS); + if (!acks.equals("0")) { + assertThat(first.offset()).isZero(); + assertThat(second.offset()).isEqualTo(1L); + } + producer.flush(); + assertReadback(connection, path, format); + } finally { + admin.dropTable(path, true).get(); + } + } + } + } + + private static void assertReadback(Connection connection, TablePath path, String format) + throws Exception { + try (Table table = connection.getTable(path); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + int count = 0; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30); + while (count < 2 && System.nanoTime() < deadline) { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + for (ScanRecord record : records) { + InternalRow row = record.getRow(); + if (format.equals("raw")) { + assertThat(row.getBytes(0)).isEqualTo(VALUE); + assertThat(row.getBytes(3)).isEqualTo(KEY); + } else { + assertThat(row.getString(0).toString()).isEqualTo("value"); + assertThat(row.getString(3).toString()).isEqualTo("key"); + } + assertThat(row.getTimestampLtz(1, 3).getEpochMillisecond()).isEqualTo(123L); + assertThat(row.getArray(2).size()).isEqualTo(2); + assertThat(row.getArray(2).getRow(0, 2).getString(0).toString()) + .isEqualTo("source"); + assertThat(row.getArray(2).getRow(1, 2).isNullAt(1)).isTrue(); + count++; + } + } + assertThat(count).isEqualTo(2); + } + } + + private static TableDescriptor descriptor(String format) { + boolean raw = format.equals("raw"); + return TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("body", raw ? DataTypes.BYTES() : DataTypes.STRING()) + .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column( + "attributes", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD( + "value", DataTypes.BYTES())))) + .column("message_key", raw ? DataTypes.BYTES() : DataTypes.STRING()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format) + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format) + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .build(); + } + + private static KafkaProducer producer(String acks) { + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + Map config = new HashMap<>(); + config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, node.host() + ":" + node.port()); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "gzip"); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 10000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index c4476c69d59..55eb4cb06c9 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -114,6 +114,7 @@ private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) .containsExactly( + tuple(ApiKeys.PRODUCE.id, (short) 3, (short) 11), tuple(ApiKeys.METADATA.id, ApiKeys.METADATA.oldestVersion(), (short) 11), tuple( ApiKeys.API_VERSIONS.id, diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java new file mode 100644 index 00000000000..c06b8f262f2 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/backend/produce/GatewayKafkaProduceBackendTest.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.backend.produce; + +import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.PartitionWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.Record; +import org.apache.fluss.kafka.backend.produce.KafkaProduceCommand.TopicWrite; +import org.apache.fluss.kafka.backend.produce.KafkaProduceResult.PartitionResult; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.transcode.ArrowKafkaRecordTranscoder; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.TestingTabletGatewayService; +import org.apache.fluss.rpc.messages.GetTableInfoRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket; +import org.apache.fluss.rpc.messages.ProduceLogRequest; +import org.apache.fluss.rpc.messages.ProduceLogResponse; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.common.protocol.Errors; +import org.junit.jupiter.api.Test; + +import java.net.InetAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests append admission, error isolation, session propagation and delayed-fetch completion. */ +class GatewayKafkaProduceBackendTest { + + @Test + void testDrainsImmediatelyWhileAcksAllResponseIsPending() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) -1, topic("topic", good(0)))); + assertThat(service.append.getAcks()).isEqualTo(-1); + assertThat(service.append.getTimeoutMs()).isEqualTo(4321); + assertThat(service.append.getTableId()).isEqualTo(42L); + assertThat(service.drains).isEqualTo(1); + assertThat(result).isNotDone(); + assertThat(service.pendingAppend).isNotDone(); + service.pendingAppend.complete(success(0)); + assertThat(result.join().topics().get(0).partitions().get(0).baseOffset()).isEqualTo(17L); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testDrainsAfterSynchronousAppendFailure() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.throwAppend = true; + assertErrors( + backend(service).write(command((short) 1, topic("topic", good(0)))).join(), + Errors.UNKNOWN_SERVER_ERROR); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testConversionAndBucketErrorsAreIsolatedFromValidWrites() throws Exception { + TestingProduceService service = new TestingProduceService(); + PartitionWrite malformed = + new PartitionWrite( + 0, + Collections.singletonList( + new Record( + 1L, + null, + new byte[] {(byte) 0xff}, + Collections.emptyList()))); + KafkaProduceResult result = + backend(service) + .write(command((short) 1, topic("topic", malformed, good(1), good(8)))) + .join(); + assertErrors( + result, Errors.CORRUPT_MESSAGE, Errors.NONE, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append.getBucketsReqsList()).hasSize(1); + assertThat(service.append.getBucketsReqsList().get(0).getBucketId()).isEqualTo(1); + assertThat(service.drains).isEqualTo(1); + } + + @Test + void testInvalidAndMissingTablesDoNotSuppressOtherTopics() throws Exception { + for (boolean async : new boolean[] {false, true}) { + TestingProduceService service = new TestingProduceService(); + service.asyncMissing = async; + KafkaProduceResult result = + backend(service) + .write( + command( + (short) 1, + topic("invalid", good(0)), + topic("missing", good(0)), + topic("topic", good(0)))) + .join(); + assertThat(result.topics()).hasSize(3); + assertThat(result.topics().get(0).partitions().get(0).error()) + .isEqualTo(Errors.INVALID_TOPIC_EXCEPTION); + assertThat(result.topics().get(1).partitions().get(0).error()) + .isEqualTo(Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(result.topics().get(2).partitions().get(0).error()).isEqualTo(Errors.NONE); + assertThat(service.drains).isEqualTo(1); + } + } + + @Test + void testNoAppendOrDrainWhenAllPartitionsFailConversion() throws Exception { + TestingProduceService service = new TestingProduceService(); + KafkaProduceResult result = + backend(service).write(command((short) 1, topic("topic", good(8)))).join(); + assertErrors(result, Errors.UNKNOWN_TOPIC_OR_PARTITION); + assertThat(service.append).isNull(); + assertThat(service.drains).isZero(); + } + + @Test + void testMapsReplicationFailuresAndMissingResponsesPerPartition() throws Exception { + org.apache.fluss.rpc.protocol.Errors[] flussErrors = { + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND_EXCEPTION, + org.apache.fluss.rpc.protocol.Errors.REQUEST_TIME_OUT, + org.apache.fluss.rpc.protocol.Errors.NOT_LEADER_OR_FOLLOWER + }; + Errors[] kafkaErrors = { + Errors.NOT_ENOUGH_REPLICAS, + Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND, + Errors.REQUEST_TIMED_OUT, + Errors.NOT_LEADER_OR_FOLLOWER + }; + for (int i = 0; i < flussErrors.length; i++) { + TestingProduceService service = new TestingProduceService(); + service.pendingAppend = + CompletableFuture.completedFuture( + new ProduceLogResponse() + .addAllBucketsResps( + Arrays.asList( + new PbProduceLogRespForBucket() + .setBucketId(0) + .setErrorCode(flussErrors[i].code()), + new PbProduceLogRespForBucket() + .setBucketId(1) + .setErrorCode(0) + .setBaseOffset(17L)))); + KafkaProduceResult result = + backend(service) + .write(command((short) -1, topic("topic", good(0), good(1), good(2)))) + .join(); + assertErrors(result, kafkaErrors[i], Errors.NONE, Errors.UNKNOWN_SERVER_ERROR); + assertThat(result.topics().get(0).partitions().get(1).baseOffset()).isEqualTo(17L); + } + } + + @Test + void testAsyncMetadataCompletionRestoresSessionAndAcksZero() throws Exception { + TestingProduceService service = new TestingProduceService(); + service.pendingMetadata = new CompletableFuture<>(); + CompletableFuture result = + backend(service).write(command((short) 0, topic("topic", good(0)))); + CompletableFuture.runAsync(() -> service.pendingMetadata.complete(metadata(false))).join(); + assertErrors(result.join(), Errors.NONE); + assertThat(service.append.getAcks()).isZero(); + assertThat(service.drains).isEqualTo(1); + } + + private static GatewayKafkaProduceBackend backend(TestingProduceService service) { + return new GatewayKafkaProduceBackend( + service, service, "kafka", new ArrowKafkaRecordTranscoder()); + } + + private static KafkaProduceCommand command(short acks, TopicWrite... topics) throws Exception { + return new KafkaProduceCommand( + acks, 4321, Arrays.asList(topics), "KAFKA", InetAddress.getLoopbackAddress()); + } + + private static TopicWrite topic(String name, PartitionWrite... partitions) { + return new TopicWrite(name, Arrays.asList(partitions)); + } + + private static PartitionWrite good(int id) { + return new PartitionWrite( + id, + Collections.singletonList( + new Record(123L, null, new byte[] {65}, Collections.emptyList()))); + } + + private static GetTableInfoResponse metadata(boolean invalid) { + TableDescriptor.Builder descriptor = + TableDescriptor.builder() + .schema(Schema.newBuilder().column("value", DataTypes.STRING()).build()) + .distributedBy(3) + .logFormat(LogFormat.ARROW); + if (!invalid) { + descriptor.customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"); + } + return new GetTableInfoResponse() + .setTableId(42L) + .setSchemaId(1) + .setTableJson(descriptor.build().toJsonBytes()) + .setCreatedTime(1L) + .setModifiedTime(1L); + } + + private static ProduceLogResponse success(int id) { + return new ProduceLogResponse() + .addAllBucketsResps( + Collections.singletonList( + new PbProduceLogRespForBucket() + .setBucketId(id) + .setBaseOffset(17L))); + } + + private static void assertErrors(KafkaProduceResult result, Errors... errors) { + List partitions = result.topics().get(0).partitions(); + assertThat(partitions).extracting(PartitionResult::error).containsExactly(errors); + for (PartitionResult partition : partitions) { + if (partition.error() != Errors.NONE) { + assertThat(partition.baseOffset()).isEqualTo(-1L); + } + } + } + + private static class TestingProduceService extends TestingTabletGatewayService { + private ProduceLogRequest append; + private CompletableFuture pendingAppend; + private CompletableFuture pendingMetadata; + private boolean throwAppend; + private boolean asyncMissing; + private int drains; + + @Override + public CompletableFuture getTableInfo(GetTableInfoRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(request.getTablePath().getDatabaseName()).isEqualTo("kafka"); + String name = request.getTablePath().getTableName(); + if (name.equals("missing")) { + if (!asyncMissing) { + throw new TableNotExistException("missing"); + } + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(new TableNotExistException("missing")); + return failed; + } + return pendingMetadata == null + ? CompletableFuture.completedFuture( + GatewayKafkaProduceBackendTest.metadata(name.equals("invalid"))) + : pendingMetadata; + } + + @Override + public CompletableFuture produceLog(ProduceLogRequest request) { + assertThat(currentListenerName()).isEqualTo("KAFKA"); + assertThat(currentSession().getInetAddress()) + .isEqualTo(InetAddress.getLoopbackAddress()); + append = request; + if (throwAppend) { + throw new IllegalStateException("append failed"); + } + return pendingAppend == null + ? CompletableFuture.completedFuture( + success(request.getBucketsReqsList().get(0).getBucketId())) + : pendingAppend; + } + + @Override + public void tryCompleteActions() { + assertThat(append).isNotNull(); + drains++; + } + } +} From 7a0aaa106e0b4c2d4e67efa975979f85c36c752f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 11 Sep 2026 14:44:26 +0800 Subject: [PATCH 10/10] [kafka] Cover Produce lifecycle and replication regressions Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 364/364 --- .../kafka/KafkaProduceRegressionITCase.java | 364 ++++++++++++++++++ 1 file changed, 364 insertions(+) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java new file mode 100644 index 00000000000..0748bd81503 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaProduceRegressionITCase.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.scanner.log.LogScanner; +import org.apache.fluss.client.table.scanner.log.ScanRecords; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.cluster.AlterConfig; +import org.apache.fluss.config.cluster.AlterConfigOpType; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.NotEnoughReplicasException; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; +import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.MetadataRequest; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.tuple; + +/** Regression coverage for native DDL lifecycle, metadata refresh, and Produce minimum ISR. */ +class KafkaProduceRegressionITCase { + private static final String DATABASE = "kafka"; + private static final short METADATA_VERSION = 11; + + @RegisterExtension + static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder() + .setNumOfTabletServers(3) + .setClusterConf(clusterConfig()) + .setTabletServerListeners("FLUSS://localhost:0,KAFKA://localhost:0") + .build(); + + @Test + void testAdvertisedListenersAndTopicIdentityLifecycle() throws Exception { + String topic = "metadata_lifecycle"; + TablePath path = TablePath.of(DATABASE, topic); + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, topic, 3); + MetadataResponse initial = waitForTopic(topic); + Uuid initialId = initial.data().topics().find(topic).topicId(); + assertThat(initialId).isNotEqualTo(Uuid.ZERO_UUID); + assertKafkaListenerEndpoints(initial); + assertThat(fetchMetadata(topic).data().topics().find(topic).topicId()) + .isEqualTo(initialId); + + admin.dropTable(path, false).get(); + retry( + Duration.ofMinutes(1), + () -> + assertThat(fetchMetadata(topic).errors()) + .containsEntry(topic, Errors.UNKNOWN_TOPIC_OR_PARTITION)); + + createTable(admin, topic, 3); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponseTopic recreated = + fetchMetadata(topic).data().topics().find(topic); + assertThat(recreated.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(recreated.topicId()) + .isNotEqualTo(Uuid.ZERO_UUID) + .isNotEqualTo(initialId); + }); + } finally { + admin.dropTable(path, true).get(); + } + } + } + + @Test + void testProducerRefreshesMetadataAfterLeaderFailover() throws Exception { + String topic = "metadata_failover"; + int stoppedLeader = -1; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, topic, 3); + MetadataResponsePartition initial = + waitForTopic(topic).data().topics().find(topic).partitions().get(0); + stoppedLeader = initial.leaderId(); + try (KafkaProducer producer = producer("1", true)) { + assertThat(send(producer, topic, "before_failover").offset()).isNotNegative(); + CLUSTER.stopTabletServer(stoppedLeader); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponseTopic metadata = + fetchMetadata(topic).data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + MetadataResponsePartition partition = metadata.partitions().get(0); + assertThat(partition.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(partition.leaderId()) + .isNotEqualTo(-1) + .isNotEqualTo(initial.leaderId()); + assertThat(partition.leaderEpoch()) + .isGreaterThan(initial.leaderEpoch()); + }); + // Reuse the producer's cached route so the failed request triggers a refresh. + // acks=1 permits data loss and offset reuse after failover; this verifies + // continued routing, not durability of the first acknowledged record. + assertThat(send(producer, topic, "after_failover").offset()).isNotNegative(); + } + } finally { + try { + if (stoppedLeader >= 0 && CLUSTER.getTabletServerById(stoppedLeader) == null) { + CLUSTER.startTabletServer(stoppedLeader); + CLUSTER.assertHasTabletServerNumber(3); + CLUSTER.waitUntilAllGatewayHasSameMetadata(); + } + } finally { + admin.dropTable(TablePath.of(DATABASE, topic), true).get(); + } + } + } + } + + @Test + void testClusterMinIsrPolicyAndDynamicReload() throws Exception { + String replicated = "acks_all_replicated"; + String underReplicated = "acks_all_under_replicated"; + try (Connection connection = ConnectionFactory.createConnection(CLUSTER.getClientConfig()); + Admin admin = connection.getAdmin()) { + try { + createTable(admin, replicated, 3); + createTable(admin, underReplicated, 1); + try (KafkaProducer producer = producer("all", false)) { + assertThat(send(producer, replicated, "replicated").offset()).isZero(); + assertThatThrownBy(() -> send(producer, underReplicated, "rejected")) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(NotEnoughReplicasException.class); + + setMinIsr(admin, 1); + assertThat(send(producer, underReplicated, "accepted").offset()).isZero(); + // Offset zero and native readback also prove the rejected write was not stored. + assertSingleValue(connection, underReplicated, "accepted"); + } + } finally { + try { + setMinIsr(admin, 2); + } finally { + admin.dropTable(TablePath.of(DATABASE, replicated), true).get(); + admin.dropTable(TablePath.of(DATABASE, underReplicated), true).get(); + } + } + } + } + + private static void createTable(Admin admin, String topic, int replicas) throws Exception { + admin.createDatabase(DATABASE, DatabaseDescriptor.EMPTY, true).get(); + TablePath path = TablePath.of(DATABASE, topic); + admin.createTable( + path, + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_REPLICATION_FACTOR, replicas) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .build(), + false) + .get(); + CLUSTER.waitUntilTableReady(admin.getTableInfo(path).get().getTableId()); + } + + private static void setMinIsr(Admin admin, int minIsr) throws Exception { + admin.alterClusterConfigs( + Collections.singletonList( + new AlterConfig( + ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER.key(), + Integer.toString(minIsr), + AlterConfigOpType.SET))) + .get(30, TimeUnit.SECONDS); + retry( + Duration.ofMinutes(1), + () -> + assertThat(CLUSTER.getTabletServers()) + .allSatisfy( + server -> + assertThat( + server.getReplicaManager() + .getMinInSyncReplicas()) + .isEqualTo(minIsr))); + } + + private static void assertSingleValue(Connection connection, String topic, String expected) + throws Exception { + try (Table table = connection.getTable(TablePath.of(DATABASE, topic)); + LogScanner scanner = table.newScan().createLogScanner()) { + scanner.subscribeFromBeginning(0); + retry( + Duration.ofSeconds(30), + () -> { + ScanRecords records = scanner.poll(Duration.ofSeconds(1)); + assertThat(records).hasSize(1); + assertThat(records.iterator().next().getRow().getBytes(0)) + .isEqualTo(expected.getBytes(StandardCharsets.UTF_8)); + }); + } + } + + private static MetadataResponse waitForTopic(String topic) { + AtomicReference result = new AtomicReference<>(); + retry( + Duration.ofMinutes(1), + () -> { + MetadataResponse response = fetchMetadata(topic); + MetadataResponseTopic metadata = response.data().topics().find(topic); + assertThat(metadata.errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(metadata.partitions()).hasSize(1); + assertThat(metadata.partitions().get(0).leaderId()).isNotEqualTo(-1); + result.set(response); + }); + return result.get(); + } + + private static void assertKafkaListenerEndpoints(MetadataResponse response) { + assertThat(response.brokers()) + .extracting( + org.apache.kafka.common.Node::id, + org.apache.kafka.common.Node::host, + org.apache.kafka.common.Node::port, + org.apache.kafka.common.Node::rack) + .containsExactlyInAnyOrderElementsOf( + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map( + node -> + tuple( + node.id(), + node.host(), + node.port(), + node.rack())) + .collect(Collectors.toList())); + } + + private static MetadataResponse fetchMetadata(String topic) throws Exception { + MetadataRequest request = + new MetadataRequest.Builder(Collections.singletonList(topic), false) + .build(METADATA_VERSION); + RequestHeader header = + new RequestHeader(ApiKeys.METADATA, METADATA_VERSION, "regression-test", 1); + ByteBuffer serialized = + RequestUtils.serialize( + header.data(), header.headerVersion(), request.data(), request.version()); + byte[] requestBytes = new byte[serialized.remaining()]; + serialized.get(requestBytes); + ServerNode node = CLUSTER.getTabletServerNodes("KAFKA").get(0); + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(node.host(), node.port()), 5000); + socket.setSoTimeout(5000); + try (DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + DataInputStream input = new DataInputStream(socket.getInputStream())) { + output.writeInt(requestBytes.length); + output.write(requestBytes); + output.flush(); + int size = input.readInt(); + assertThat(size).isBetween(1, 1024 * 1024); + byte[] response = new byte[size]; + input.readFully(response); + return (MetadataResponse) + AbstractResponse.parseResponse(ByteBuffer.wrap(response), header); + } + } + } + + private static RecordMetadata send( + KafkaProducer producer, String topic, String value) throws Exception { + return producer.send( + new ProducerRecord<>( + topic, 0, null, value.getBytes(StandardCharsets.UTF_8))) + .get(60, TimeUnit.SECONDS); + } + + private static KafkaProducer producer(String acks, boolean retryOnFailure) { + Map config = new HashMap<>(); + config.put( + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, + CLUSTER.getTabletServerNodes("KAFKA").stream() + .map(node -> node.host() + ":" + node.port()) + .collect(Collectors.joining(","))); + config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + config.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, false); + config.put(ProducerConfig.ACKS_CONFIG, acks); + config.put(ProducerConfig.RETRIES_CONFIG, retryOnFailure ? 20 : 0); + config.put(ProducerConfig.RETRY_BACKOFF_MS_CONFIG, 200); + config.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 5000); + config.put(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, 30000); + config.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 30000); + return new KafkaProducer<>(config); + } + + private static Configuration clusterConfig() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DATABASE, DATABASE); + config.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + config.set(ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER, 2); + return config; + } +}