From c0cce4d30f1bf719909f8e8451ff5acae9729750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Schmitz=20von=20H=C3=BClst?= Date: Mon, 10 Aug 2026 21:56:52 +0200 Subject: [PATCH 1/4] ARTEMIS-6179: Add regression test for depageLock/synchronized(this) deadlock --- ...DeadlockDepageLockDeleteReferenceTest.java | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/DeadlockDepageLockDeleteReferenceTest.java diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/DeadlockDepageLockDeleteReferenceTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/DeadlockDepageLockDeleteReferenceTest.java new file mode 100644 index 00000000000..1f77424f4f1 --- /dev/null +++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/DeadlockDepageLockDeleteReferenceTest.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.integration.paging; + +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.activemq.artemis.api.core.QueueConfiguration; +import org.apache.activemq.artemis.api.core.RoutingType; +import org.apache.activemq.artemis.api.core.client.ClientConsumer; +import org.apache.activemq.artemis.api.core.client.ClientMessage; +import org.apache.activemq.artemis.api.core.client.ClientProducer; +import org.apache.activemq.artemis.api.core.client.ClientSession; +import org.apache.activemq.artemis.api.core.client.ClientSessionFactory; +import org.apache.activemq.artemis.api.core.client.ServerLocator; +import org.apache.activemq.artemis.api.core.management.QueueControl; +import org.apache.activemq.artemis.api.core.management.ResourceNames; +import org.apache.activemq.artemis.core.server.ActiveMQServer; +import org.apache.activemq.artemis.tests.util.ActiveMQTestBase; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +/** + * Regression test for ARTEMIS-6179. + * + *

QueueImpl.depage() acquires {@code depageLock} then {@code synchronized(this)}. + * QueueImpl.deleteReference() is {@code synchronized(this)}, then calls {@code iterQueue()} + * which acquires {@code depageLock}. Concurrent execution causes a classic ABBA deadlock. + * + *

Bug introduced in commit e9dbc11 (ARTEMIS-5376, Feb 27 2026). First affected release: 2.53.0. + */ +public class DeadlockDepageLockDeleteReferenceTest extends ActiveMQTestBase { + + private static final String ADDRESS = "test.deadlock.address"; + private static final String QUEUE = "test.deadlock.queue"; + + // Small address limit so paging kicks in after ~50 messages. + private static final int PAGE_SIZE_BYTES = 128 * 1024; // 128 KB per page file + private static final int MAX_SIZE_BYTES = 512 * 1024; // 512 KB address limit + private static final int MESSAGE_SIZE = 10 * 1024; // 10 KB per message + // 100 × 10 KB = 1 MB >> 512 KB; keeps ~50 messages paged to disk. + private static final int INITIAL_MESSAGES = 100; + + private static final long DEADLOCK_TIMEOUT_MS = 5_000; + + protected ServerLocator locator; + + @Override + @BeforeEach + public void setUp() throws Exception { + super.setUp(); + locator = createInVMNonHALocator(); + } + + @Test + @Timeout(60) + public void testNoDeadlockDuringDepageAndDeleteReference() throws Exception { + ActiveMQServer server = createServer(true, createDefaultInVMConfig(), PAGE_SIZE_BYTES, MAX_SIZE_BYTES); + server.start(); + + server.createQueue(QueueConfiguration.of(QUEUE) + .setAddress(ADDRESS) + .setRoutingType(RoutingType.ANYCAST) + .setDurable(true)); + + QueueControl queueControl = (QueueControl) server.getManagementService() + .getResource(ResourceNames.QUEUE + QUEUE); + + ClientSessionFactory factory = locator.createSessionFactory(); + + // Fill queue past address limit to trigger paging. + try (ClientSession sendSession = factory.createSession(false, true, true)) { + ClientProducer producer = sendSession.createProducer(ADDRESS); + byte[] body = new byte[MESSAGE_SIZE]; + Arrays.fill(body, (byte) 'X'); + for (int i = 0; i < INITIAL_MESSAGES; i++) { + ClientMessage msg = sendSession.createMessage(true); + msg.getBodyBuffer().writeBytes(body); + producer.send(msg); + } + } + + // Background sender: keeps paging active by replenishing faster than the consumer drains. + ClientSession senderSession = factory.createSession(false, true, true); + ClientProducer bgProducer = senderSession.createProducer(ADDRESS); + byte[] bgBody = new byte[MESSAGE_SIZE]; + Arrays.fill(bgBody, (byte) 'Y'); + Thread senderThread = new Thread(() -> { + try { + while (!Thread.currentThread().isInterrupted()) { + ClientMessage msg = senderSession.createMessage(true); + msg.getBodyBuffer().writeBytes(bgBody); + bgProducer.send(msg); + Thread.sleep(10); // ~100 msg/s >> consumer rate, keeps paging active + } + } catch (InterruptedException ignored) { + } catch (Exception e) { + if (!Thread.currentThread().isInterrupted()) { + logger.warn("sender error: {}", e.getMessage()); + } + } + }, "repro-sender"); + senderThread.setDaemon(true); + senderThread.start(); + + // Consumer: triggers continuous depage() (acquires depageLock, then synchronized(this)). + ClientSession consumeSession = factory.createSession(false, false, false); + consumeSession.start(); + ClientConsumer consumer = consumeSession.createConsumer(QUEUE); + Thread consumerThread = new Thread(() -> { + try { + int batch = 0; + while (!Thread.currentThread().isInterrupted()) { + ClientMessage msg = consumer.receive(200); + if (msg != null) { + msg.acknowledge(); + if (++batch % 10 == 0) { + consumeSession.commit(); + } + Thread.sleep(100); // ~10 msg/s; sender at 100 msg/s keeps queue growing + } + } + } catch (InterruptedException ignored) { + } catch (Exception e) { + if (!Thread.currentThread().isInterrupted()) { + logger.warn("consumer error: {}", e.getMessage()); + } + } + }, "repro-consumer"); + consumerThread.setDaemon(true); + consumerThread.start(); + + AtomicBoolean stop = new AtomicBoolean(false); + ExecutorService pool = Executors.newCachedThreadPool(r -> { + Thread t = new Thread(r); + t.setDaemon(true); + return t; + }); + + // Management threads: removeMessage() -> deleteReference() -> iterQueue() -> depageLock. + // deleteReference() holds synchronized(QueueImpl.this), then acquires depageLock inside + // iterQueue(). This is the opposite order from depage(), causing the ABBA deadlock. + for (int i = 0; i < 5; i++) { + pool.submit(() -> { + while (!stop.get()) { + try { + @SuppressWarnings("unchecked") + Map[] msgs = queueControl.listMessages(null); + if (msgs != null && msgs.length > 0) { + Object id = msgs[0].get("messageID"); + if (id instanceof Number n) { + queueControl.removeMessage(n.longValue()); + } + } + } catch (Exception ignored) { + } + } + }); + } + + // Probe: isPaused() requires synchronized(QueueImpl.this). If the monitor is deadlocked, + // this call will block until the @Timeout triggers and fails the test. + long deadline = System.currentTimeMillis() + 30_000; + boolean deadlockDetected = false; + while (System.currentTimeMillis() < deadline) { + Thread.sleep(1_000); + + AtomicBoolean probeReturned = new AtomicBoolean(false); + Thread probe = new Thread(() -> { + try { + queueControl.isPaused(); + probeReturned.set(true); + } catch (Exception ignored) { + probeReturned.set(true); + } + }, "deadlock-probe"); + probe.setDaemon(true); + probe.start(); + probe.join(DEADLOCK_TIMEOUT_MS); + + if (!probeReturned.get()) { + deadlockDetected = true; + break; + } + } + + stop.set(true); + pool.shutdownNow(); + senderThread.interrupt(); + consumerThread.interrupt(); + senderThread.join(2_000); + consumerThread.join(2_000); + consumeSession.close(); + senderSession.close(); + factory.close(); + + if (deadlockDetected) { + fail("Deadlock detected: isPaused() blocked for >" + DEADLOCK_TIMEOUT_MS + + "ms — ARTEMIS-6179: depageLock / synchronized(QueueImpl.this) ordering inversion"); + } + } +} \ No newline at end of file From 7665d200fc28333b8556e809a517cb49ad97e264 Mon Sep 17 00:00:00 2001 From: moritzschmitz-oviva Date: Mon, 10 Aug 2026 22:34:51 +0200 Subject: [PATCH 2/4] ARTEMIS-6179: Fix test infrastructure for Maven 4 and logger visibility --- .../apache/activemq/artemis/tests/util/ActiveMQTestBase.java | 2 +- tests/integration-tests/pom.xml | 2 +- tests/smoke-tests/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java b/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java index 3a65d746f65..f29be999cc6 100644 --- a/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java +++ b/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java @@ -176,7 +176,7 @@ @ExtendWith(LibaioContextCheckExtension.class) public abstract class ActiveMQTestBase extends ArtemisTestCase { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); static { Env.setTestEnv(true); diff --git a/tests/integration-tests/pom.xml b/tests/integration-tests/pom.xml index fddff750a41..3ea6f9ca294 100644 --- a/tests/integration-tests/pom.xml +++ b/tests/integration-tests/pom.xml @@ -463,7 +463,7 @@ org.apache.maven.plugins maven-surefire-plugin - + **/integration/security/*Test.java **/integration/ssl/*Test.java **/integration/amqp/JMSSaslExternalTest.java diff --git a/tests/smoke-tests/pom.xml b/tests/smoke-tests/pom.xml index 4b7e72e2eaf..191e55ab0bc 100644 --- a/tests/smoke-tests/pom.xml +++ b/tests/smoke-tests/pom.xml @@ -441,7 +441,7 @@ 1 false - + **/smoke/upgradeTest/*Test.java From 5fd808bd2b9f012d8ad9d6ad9b99888bfaabff33 Mon Sep 17 00:00:00 2001 From: moritzschmitz-oviva Date: Mon, 10 Aug 2026 22:58:46 +0200 Subject: [PATCH 3/4] ARTEMIS-6179: Address review feedback - Revert ActiveMQTestBase.logger to private; add local logger to test class - Drop unused TimeUnit import; fix MethodHandles import ordering - Bump DEADLOCK_TIMEOUT_MS 5s -> 10s to reduce CI false-positive risk - Bump @Timeout 60s -> 120s to match extended probe window - ServerLocator locator: protected -> private - Add missing newline at end of file --- .../artemis/tests/util/ActiveMQTestBase.java | 2 +- .../DeadlockDepageLockDeleteReferenceTest.java | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java b/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java index f29be999cc6..3a65d746f65 100644 --- a/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java +++ b/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java @@ -176,7 +176,7 @@ @ExtendWith(LibaioContextCheckExtension.class) public abstract class ActiveMQTestBase extends ArtemisTestCase { - protected static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); static { Env.setTestEnv(true); diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/DeadlockDepageLockDeleteReferenceTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/DeadlockDepageLockDeleteReferenceTest.java index 1f77424f4f1..e9cc532cecb 100644 --- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/DeadlockDepageLockDeleteReferenceTest.java +++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/DeadlockDepageLockDeleteReferenceTest.java @@ -18,11 +18,11 @@ import static org.junit.jupiter.api.Assertions.fail; +import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.activemq.artemis.api.core.QueueConfiguration; @@ -40,6 +40,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Regression test for ARTEMIS-6179. @@ -52,6 +54,8 @@ */ public class DeadlockDepageLockDeleteReferenceTest extends ActiveMQTestBase { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String ADDRESS = "test.deadlock.address"; private static final String QUEUE = "test.deadlock.queue"; @@ -62,9 +66,9 @@ public class DeadlockDepageLockDeleteReferenceTest extends ActiveMQTestBase { // 100 × 10 KB = 1 MB >> 512 KB; keeps ~50 messages paged to disk. private static final int INITIAL_MESSAGES = 100; - private static final long DEADLOCK_TIMEOUT_MS = 5_000; + private static final long DEADLOCK_TIMEOUT_MS = 10_000; - protected ServerLocator locator; + private ServerLocator locator; @Override @BeforeEach @@ -74,7 +78,7 @@ public void setUp() throws Exception { } @Test - @Timeout(60) + @Timeout(120) public void testNoDeadlockDuringDepageAndDeleteReference() throws Exception { ActiveMQServer server = createServer(true, createDefaultInVMConfig(), PAGE_SIZE_BYTES, MAX_SIZE_BYTES); server.start(); @@ -220,4 +224,4 @@ public void testNoDeadlockDuringDepageAndDeleteReference() throws Exception { "ms — ARTEMIS-6179: depageLock / synchronized(QueueImpl.this) ordering inversion"); } } -} \ No newline at end of file +} From b6ff4ec063c2672aa974facf9b0c8a29d081302c Mon Sep 17 00:00:00 2001 From: moritzschmitz-oviva Date: Mon, 10 Aug 2026 23:09:13 +0200 Subject: [PATCH 4/4] ARTEMIS-6179: Revert pom.xml combine.children change (Maven 3 not affected) --- tests/integration-tests/pom.xml | 2 +- tests/smoke-tests/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration-tests/pom.xml b/tests/integration-tests/pom.xml index 3ea6f9ca294..fddff750a41 100644 --- a/tests/integration-tests/pom.xml +++ b/tests/integration-tests/pom.xml @@ -463,7 +463,7 @@ org.apache.maven.plugins maven-surefire-plugin - + **/integration/security/*Test.java **/integration/ssl/*Test.java **/integration/amqp/JMSSaslExternalTest.java diff --git a/tests/smoke-tests/pom.xml b/tests/smoke-tests/pom.xml index 191e55ab0bc..4b7e72e2eaf 100644 --- a/tests/smoke-tests/pom.xml +++ b/tests/smoke-tests/pom.xml @@ -441,7 +441,7 @@ 1 false - + **/smoke/upgradeTest/*Test.java