From 89a3dfb9ae6ddd84fa346c23259fe3661ca0c578 Mon Sep 17 00:00:00 2001 From: Aman Poonia Date: Sat, 5 Sep 2026 12:01:44 +0530 Subject: [PATCH 1/4] HBASE-30357 OpenRegionProcedure#restoreSucceedState ignores persisted transitionCode, forcing OPEN even after a real FAILED_OPEN On master-failover restore, RegionRemoteProcedureBase.stateLoaded() calls restoreSucceedState() for any region whose remote-open report was already persisted with state REGION_REMOTE_PROCEDURE_REPORT_SUCCEED, but before this change OpenRegionProcedure ignored the real persisted TransitionCode and always forced the region to OPEN. If the RS had actually reported FAILED_OPEN and the master crashed before persisting that to hbase:meta, the region would come back as a "phantom" OPEN region on restart: no procedure watching it, no automatic retry, and invisible to the RegionInTransition tracker since OPEN is the only non-RIT terminal state. Widen RegionRemoteProcedureBase#restoreSucceedState to also receive the persisted TransitionCode, and make OpenRegionProcedure branch on it: on FAILED_OPEN, call AssignmentManager#regionFailedOpen(regionNode, false), mirroring what the live (non-restart) reportTransition path already does. This leaves the region non-OPEN so TransitRegionStateProcedure#confirmOpened sees it and drives the normal retry loop instead of finishing silently. CloseRegionProcedure's override is updated to match the new signature but ignores the parameter, since CLOSE has no failure variant. Add TestOpenRegionProcedureRestoreFailedOpen, which reproduces the restore timing directly (invoking the same package-private stateLoaded() hook that a real master restart triggers) and asserts the region is not left in OPEN state after a FAILED_OPEN report. --- .../master/assignment/AssignmentManager.java | 9 +- .../assignment/CloseRegionProcedure.java | 5 +- .../assignment/OpenRegionProcedure.java | 7 +- .../assignment/RegionRemoteProcedureBase.java | 4 +- ...tOpenRegionProcedureRestoreFailedOpen.java | 95 +++++++++++++++++++ 5 files changed, 113 insertions(+), 7 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 5baf30846e08..a9ae6f72a29b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -2232,8 +2232,13 @@ CompletableFuture regionOpening(RegionStateNode regionNode) { } // should be called under the RegionStateNode lock - // The parameter 'giveUp' means whether we will try to open the region again, if it is true, then - // we will persist the FAILED_OPEN state into hbase:meta. + // Called when a region open attempt has failed. The 'giveUp' parameter says whether we will try + // to open the region again. If true, this is a terminal failure with no more retries: we set + // RegionState.State to FAILED_OPEN and persist it to hbase:meta, and only remove the region from + // its old target server's bookkeeping once that persist succeeds. If false, we are about to + // retry the open with a new plan, so RegionState.State is intentionally left untouched (e.g. + // still OPENING), meta is not touched, and we remove the region from its old target server's + // bookkeeping immediately. In both cases the removal only happens if there was an old location. CompletableFuture regionFailedOpen(RegionStateNode regionNode, boolean giveUp) { RegionState.State state = regionNode.getState(); ServerName regionLocation = regionNode.getRegionLocation(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java index 344d7e9ad7e6..23068b3fa088 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/CloseRegionProcedure.java @@ -113,8 +113,9 @@ protected void updateTransitionWithoutPersistingToMeta(MasterProcedureEnv env, } @Override - protected void restoreSucceedState(AssignmentManager am, RegionStateNode regionNode, long seqId) - throws IOException { + protected void restoreSucceedState(AssignmentManager am, RegionStateNode regionNode, + TransitionCode transitionCode, long seqId) throws IOException { + // CLOSE has no failure variant, so transitionCode is always CLOSED here if (regionNode.getState() == State.CLOSED) { // should have already been persisted, ignore return; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java index 21218a1177a6..8e5151f2c808 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/OpenRegionProcedure.java @@ -127,7 +127,12 @@ protected void updateTransitionWithoutPersistingToMeta(MasterProcedureEnv env, @Override protected void restoreSucceedState(AssignmentManager am, RegionStateNode regionNode, - long openSeqNum) throws IOException { + TransitionCode transitionCode, long openSeqNum) throws IOException { + if (transitionCode == TransitionCode.FAILED_OPEN) { + // will not persist to meta if giveUp is false, matches the live reportTransition path + am.regionFailedOpen(regionNode, false); + return; + } if (regionNode.getState() == State.OPEN) { // should have already been persisted, ignore return; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java index fe8c9bfdd7c1..dc0fb5defe2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionRemoteProcedureBase.java @@ -268,12 +268,12 @@ void serverCrashed(MasterProcedureEnv env, RegionStateNode regionNode, ServerNam } protected abstract void restoreSucceedState(AssignmentManager am, RegionStateNode regionNode, - long seqId) throws IOException; + TransitionCode transitionCode, long seqId) throws IOException; void stateLoaded(AssignmentManager am, RegionStateNode regionNode) { if (state == RegionRemoteProcedureBaseState.REGION_REMOTE_PROCEDURE_REPORT_SUCCEED) { try { - restoreSucceedState(am, regionNode, seqId); + restoreSucceedState(am, regionNode, transitionCode, seqId); } catch (IOException e) { // should not happen as we are just restoring the state throw new AssertionError(e); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java new file mode 100644 index 000000000000..aa4c34ede9fd --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.assignment; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; + +/** + * HBASE-30357: OpenRegionProcedure#restoreSucceedState() must not force the region state to OPEN on + * master-failover restore when the persisted transition code is actually FAILED_OPEN. + */ +@Tag(MasterTests.TAG) +@Tag(LargeTests.TAG) +public class TestOpenRegionProcedureRestoreFailedOpen extends TestAssignmentManagerBase { + + /** + * On the first open attempt, reports FAILED_OPEN and then immediately simulates a master restart + * happening before the child OpenRegionProcedure gets to run its own execute() (i.e. before it + * can persist anything to meta), by directly invoking the package-private stateLoaded() hook that + * a real restart would trigger. Records the region state right after that simulated restore. + * Subsequent attempts behave like a normal healthy RS so the assign procedure can converge and + * the test does not hang. + */ + private class FailedOpenThenRestoreRsExecutor extends GoodRsExecutor { + + private final AtomicBoolean firstAttempt = new AtomicBoolean(true); + + final AtomicReference stateAfterRestore = new AtomicReference<>(); + + @Override + protected RegionOpeningState execOpenRegion(ServerName server, RegionOpenInfo openReq) + throws IOException { + if (!firstAttempt.compareAndSet(true, false)) { + return super.execOpenRegion(server, openReq); + } + RegionInfo hri = ProtobufUtil.toRegionInfo(openReq.getRegion()); + RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(hri); + TransitRegionStateProcedure trsp = (TransitRegionStateProcedure) regionNode.getProcedure(); + regionNode.lock(); + try { + sendTransitionReport(server, openReq.getRegion(), TransitionCode.FAILED_OPEN, + HConstants.NO_SEQNUM); + trsp.stateLoaded(am, regionNode); + stateAfterRestore.set(regionNode.getState()); + } finally { + regionNode.unlock(); + } + return RegionOpeningState.FAILED_OPENING; + } + } + + @Test + public void testRestoreDoesNotForceOpenAfterFailedOpen() throws Exception { + TableName tableName = TableName.valueOf(testMethodName); + RegionInfo hri = createRegionInfo(tableName, 1); + FailedOpenThenRestoreRsExecutor executor = new FailedOpenThenRestoreRsExecutor(); + rsDispatcher.setMockRsExecutor(executor); + TransitRegionStateProcedure proc = createAssignProcedure(hri); + waitOnFuture(submitProcedure(proc)); + + assertNotEquals(RegionState.State.OPEN, executor.stateAfterRestore.get()); + } +} From 2389c26ad8afa8623ed0df899ecc0c0edfd006dd Mon Sep 17 00:00:00 2001 From: Aman Poonia Date: Tue, 15 Sep 2026 20:07:04 +0530 Subject: [PATCH 2/4] HBASE-30357 Rewrite TestOpenRegionProcedureRestoreFailedOpen to restart a real ProcedureExecutor Replace the manual TransitRegionStateProcedure#stateLoaded() invocation under the mocked TestAssignmentManagerBase with a real single-master mini-cluster test. An AssignmentManagerForTest/HMasterForTest pair (pattern from TestRollbackSCP/TestReportRegionStateTransitionRetry) intercepts the RS's real OPENED report and rewrites it to FAILED_OPEN before it is persisted, then holds the RegionStateNode lock across a genuine MasterProcedureTestingUtility.restartMasterProcedureExecutor() call, exercising the actual AssignmentManager#start() -> stateLoaded() restore path instead of calling it directly. --- ...tOpenRegionProcedureRestoreFailedOpen.java | 174 ++++++++++++++---- 1 file changed, 136 insertions(+), 38 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java index aa4c34ede9fd..8cbb0d5e3aa7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java @@ -17,79 +17,177 @@ */ package org.apache.hadoop.hbase.master.assignment; +import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionStateTransitionState.REGION_STATE_TRANSITION_CONFIRM_OPENED_VALUE; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.PleaseHoldException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.RegionState; -import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; +import org.apache.hadoop.hbase.master.region.MasterRegion; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionResponse; /** * HBASE-30357: OpenRegionProcedure#restoreSucceedState() must not force the region state to OPEN on * master-failover restore when the persisted transition code is actually FAILED_OPEN. + *

+ * To reproduce the exact crash window without racing a genuinely reporting RS, we intercept the + * RS's real OPENED report on the master side and rewrite it to FAILED_OPEN before it is persisted. + * While still on the RPC handler thread (i.e. before the woken child OpenRegionProcedure gets a + * chance to run its own execute() and persist anything to meta), we lock the RegionStateNode and + * perform a genuine restart of the master's ProcedureExecutor/AssignmentManager, forcing a real + * reload from the WALProcedureStore and hbase:meta - exactly the mechanism restoreSucceedState() is + * meant to handle. */ @Tag(MasterTests.TAG) -@Tag(LargeTests.TAG) -public class TestOpenRegionProcedureRestoreFailedOpen extends TestAssignmentManagerBase { +@Tag(MediumTests.TAG) +public class TestOpenRegionProcedureRestoreFailedOpen { - /** - * On the first open attempt, reports FAILED_OPEN and then immediately simulates a master restart - * happening before the child OpenRegionProcedure gets to run its own execute() (i.e. before it - * can persist anything to meta), by directly invoking the package-private stateLoaded() hook that - * a real restart would trigger. Records the region state right after that simulated restore. - * Subsequent attempts behave like a normal healthy RS so the assign procedure can converge and - * the test does not hang. - */ - private class FailedOpenThenRestoreRsExecutor extends GoodRsExecutor { + private static final long AWAIT_TIMEOUT_SECONDS = 30; - private final AtomicBoolean firstAttempt = new AtomicBoolean(true); + private static final AtomicReference ARRIVE = new AtomicReference<>(); - final AtomicReference stateAfterRestore = new AtomicReference<>(); + private static final AtomicReference PROCEED = new AtomicReference<>(); + + private static final class AssignmentManagerForTest extends AssignmentManager { + + public AssignmentManagerForTest(MasterServices master, MasterRegion masterRegion) { + super(master, masterRegion); + } @Override - protected RegionOpeningState execOpenRegion(ServerName server, RegionOpenInfo openReq) - throws IOException { - if (!firstAttempt.compareAndSet(true, false)) { - return super.execOpenRegion(server, openReq); + public ReportRegionStateTransitionResponse reportRegionStateTransition( + ReportRegionStateTransitionRequest req) throws PleaseHoldException { + RegionStateTransition transition = req.getTransition(0); + RegionInfo hri = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); + if ( + transition.getTransitionCode() != TransitionCode.OPENED + || !hri.getTable().equals(NAME) + ) { + return super.reportRegionStateTransition(req); } - RegionInfo hri = ProtobufUtil.toRegionInfo(openReq.getRegion()); - RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(hri); - TransitRegionStateProcedure trsp = (TransitRegionStateProcedure) regionNode.getProcedure(); + CountDownLatch arrive = ARRIVE.getAndSet(null); + if (arrive == null) { + return super.reportRegionStateTransition(req); + } + ReportRegionStateTransitionRequest failedOpenReq = req.toBuilder() + .setTransition(0, transition.toBuilder().setTransitionCode(TransitionCode.FAILED_OPEN) + .setOpenSeqNum(HConstants.NO_SEQNUM).build()) + .build(); + RegionStateNode regionNode = getRegionStates().getRegionStateNode(hri); + // RegionRemoteProcedureBase#reportTransition() (called from super.reportRegionStateTransition + // below) also locks this same RegionStateNode; that only works here because the lock is + // reentrant for the same thread (see RegionStateNodeLock#lock0). regionNode.lock(); try { - sendTransitionReport(server, openReq.getRegion(), TransitionCode.FAILED_OPEN, - HConstants.NO_SEQNUM); - trsp.stateLoaded(am, regionNode); - stateAfterRestore.set(regionNode.getState()); + // persists REPORT_SUCCEED/FAILED_OPEN to the real WALProcedureStore and wakes the child + // OpenRegionProcedure, but since we still hold the RegionStateNode lock here (reentrant, + // same thread), the woken child can not resume and complete its own meta update - this is + // exactly the window a real master crash would leave us in. + ReportRegionStateTransitionResponse resp = super.reportRegionStateTransition(failedOpenReq); + arrive.countDown(); + CountDownLatch proceed = PROCEED.get(); + if (!proceed.await(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + throw new RuntimeException("Timed out waiting for PROCEED"); + } + return resp; + } catch (InterruptedException e) { + throw new RuntimeException(e); } finally { regionNode.unlock(); } - return RegionOpeningState.FAILED_OPENING; } } + public static final class HMasterForTest extends HMaster { + + public HMasterForTest(Configuration conf) throws IOException { + super(conf); + } + + @Override + protected AssignmentManager createAssignmentManager(MasterServices master, + MasterRegion masterRegion) { + return new AssignmentManagerForTest(master, masterRegion); + } + } + + private static final HBaseTestingUtil UTIL = new HBaseTestingUtil(); + + private static final TableName NAME = + TableName.valueOf("TestOpenRegionProcedureRestoreFailedOpen"); + + private static final byte[] CF = Bytes.toBytes("cf"); + + @BeforeAll + public static void setUpBeforeClass() throws Exception { + UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, HMasterForTest.class, HMaster.class); + UTIL.startMiniCluster(1); + UTIL.createTable(NAME, CF); + UTIL.waitTableAvailable(NAME); + } + + @AfterAll + public static void tearDownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + @Test public void testRestoreDoesNotForceOpenAfterFailedOpen() throws Exception { - TableName tableName = TableName.valueOf(testMethodName); - RegionInfo hri = createRegionInfo(tableName, 1); - FailedOpenThenRestoreRsExecutor executor = new FailedOpenThenRestoreRsExecutor(); - rsDispatcher.setMockRsExecutor(executor); - TransitRegionStateProcedure proc = createAssignProcedure(hri); - waitOnFuture(submitProcedure(proc)); - - assertNotEquals(RegionState.State.OPEN, executor.stateAfterRestore.get()); + HMaster master = UTIL.getMiniHBaseCluster().getMaster(); + ProcedureExecutor procExec = master.getMasterProcedureExecutor(); + AssignmentManager am = master.getAssignmentManager(); + RegionInfo region = UTIL.getAdmin().getRegions(NAME).get(0); + RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(region); + + CountDownLatch arrive = new CountDownLatch(1); + CountDownLatch proceed = new CountDownLatch(1); + ARRIVE.set(arrive); + PROCEED.set(proceed); + Future future = am.moveAsync( + new RegionPlan(region, regionNode.getRegionLocation(), regionNode.getRegionLocation())); + TransitRegionStateProcedure proc = + procExec.getProcedures().stream().filter(p -> p instanceof TransitRegionStateProcedure) + .filter(p -> !p.isFinished()).map(p -> (TransitRegionStateProcedure) p).findAny().get(); + // wait until we are suspended waiting on the RS to report the open, then let it report; the + // AssignmentManagerForTest above rewrites that report to FAILED_OPEN and blocks the RPC handler + // thread, still holding the RegionStateNode lock, until we tell it to proceed + UTIL.waitFor(30000, + () -> proc.getCurrentStateId() == REGION_STATE_TRANSITION_CONFIRM_OPENED_VALUE); + assertTrue(arrive.await(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + + MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); + RegionStateNode reloaded = am.getRegionStates().getRegionStateNode(region); + assertNotEquals(RegionState.State.OPEN, reloaded.getState()); + + proceed.countDown(); + future.get(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); } } From a7d59a71f5ee407359fa348be4ea46838c3e0cbf Mon Sep 17 00:00:00 2001 From: Aman Poonia Date: Tue, 15 Sep 2026 20:10:25 +0530 Subject: [PATCH 3/4] HBASE-30357 Fix misattributed lock comment in TestOpenRegionProcedureRestoreFailedOpen The RegionStateNode lock is actually taken by AssignmentManager's private updateRegionTransition(), not by RegionRemoteProcedureBase#reportTransition() as the comment claimed. --- .../TestOpenRegionProcedureRestoreFailedOpen.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java index 8cbb0d5e3aa7..fa7c251a1615 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java @@ -87,10 +87,7 @@ public ReportRegionStateTransitionResponse reportRegionStateTransition( ReportRegionStateTransitionRequest req) throws PleaseHoldException { RegionStateTransition transition = req.getTransition(0); RegionInfo hri = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); - if ( - transition.getTransitionCode() != TransitionCode.OPENED - || !hri.getTable().equals(NAME) - ) { + if (transition.getTransitionCode() != TransitionCode.OPENED || !hri.getTable().equals(NAME)) { return super.reportRegionStateTransition(req); } CountDownLatch arrive = ARRIVE.getAndSet(null); @@ -102,7 +99,7 @@ public ReportRegionStateTransitionResponse reportRegionStateTransition( .setOpenSeqNum(HConstants.NO_SEQNUM).build()) .build(); RegionStateNode regionNode = getRegionStates().getRegionStateNode(hri); - // RegionRemoteProcedureBase#reportTransition() (called from super.reportRegionStateTransition + // AssignmentManager#updateRegionTransition() (called from super.reportRegionStateTransition // below) also locks this same RegionStateNode; that only works here because the lock is // reentrant for the same thread (see RegionStateNodeLock#lock0). regionNode.lock(); From 11fd7ad46c39233a9b2a56434fda170e16a26fbf Mon Sep 17 00:00:00 2001 From: Aman Poonia Date: Wed, 16 Sep 2026 22:30:32 +0530 Subject: [PATCH 4/4] HBASE-30357 Remove redundant waitFor/proc lookup in TestOpenRegionProcedureRestoreFailedOpen arrive.await already synchronizes on the intercepted+persisted FAILED_OPEN report, making the state-ID poll superfluous dead code. --- .../TestOpenRegionProcedureRestoreFailedOpen.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java index fa7c251a1615..45f81b055f70 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestOpenRegionProcedureRestoreFailedOpen.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hbase.master.assignment; -import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionStateTransitionState.REGION_STATE_TRANSITION_CONFIRM_OPENED_VALUE; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -170,14 +169,8 @@ public void testRestoreDoesNotForceOpenAfterFailedOpen() throws Exception { PROCEED.set(proceed); Future future = am.moveAsync( new RegionPlan(region, regionNode.getRegionLocation(), regionNode.getRegionLocation())); - TransitRegionStateProcedure proc = - procExec.getProcedures().stream().filter(p -> p instanceof TransitRegionStateProcedure) - .filter(p -> !p.isFinished()).map(p -> (TransitRegionStateProcedure) p).findAny().get(); - // wait until we are suspended waiting on the RS to report the open, then let it report; the - // AssignmentManagerForTest above rewrites that report to FAILED_OPEN and blocks the RPC handler - // thread, still holding the RegionStateNode lock, until we tell it to proceed - UTIL.waitFor(30000, - () -> proc.getCurrentStateId() == REGION_STATE_TRANSITION_CONFIRM_OPENED_VALUE); + // arrive counts down only after the RS's OPENED report is intercepted, rewritten to + // FAILED_OPEN, and persisted, so waiting on it alone is sufficient synchronization assertTrue(arrive.await(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec);