From 5c16f953f23ce57e8bc61c44e53dc65d0eac2a40 Mon Sep 17 00:00:00 2001 From: Ma Zhengxuan <1319614897@qq.com> Date: Wed, 9 Sep 2026 10:55:46 +0800 Subject: [PATCH] HBASE-29798 Remove redundant region info from incremental backups --- .../impl/IncrementalTableBackupClient.java | 4 +- .../mapreduce/MapReduceBackupMergeJob.java | 2 - .../hadoop/hbase/backup/util/BackupUtils.java | 42 ++----------------- .../hadoop/hbase/backup/TestBackupBase.java | 4 +- .../hadoop/hbase/backup/TestBackupMerge.java | 17 ++++++++ .../backup/TestIncrementalBackupRestore.java | 24 +++++++++++ 6 files changed, 48 insertions(+), 45 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java index 345ab4c7a5a3..5e161a1a797a 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java @@ -362,8 +362,8 @@ public void execute() throws IOException, ColumnFamilyMismatchException { // case INCREMENTAL_COPY: try { - // copy out the table and region info files for each table - BackupUtils.copyTableRegionInfo(conn, backupInfo, conf); + // Copy table descriptors for restore. + BackupUtils.copyTableDescriptors(conn, backupInfo, conf); setupRegionLocator(); // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT convertWALsToHFiles(tablesToWALFileList, tablesToPrevBackupTs); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java index 0549a3371cf5..a0cf829cf4d2 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.util.Tool; @@ -227,7 +226,6 @@ protected void copyMetaData(FileSystem fs, Path tmpBackupDir, Path backupDirPath String fileName = p.toString(); if ( fileName.indexOf(FSTableDescriptors.TABLEINFO_DIR) > 0 - || fileName.indexOf(HRegionFileSystem.REGION_INFO_FILE) > 0 || fileName.indexOf(BackupManifest.MANIFEST_FILE_NAME) > 0 ) { toKeep.add(p); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java index 4154e5f06daa..4fa42392c98b 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java @@ -44,17 +44,14 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.BackupInfo; @@ -67,7 +64,6 @@ import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.master.region.MasterRegionFactory; import org.apache.hadoop.hbase.replication.ReplicationException; @@ -79,7 +75,6 @@ import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSTableDescriptors; -import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -118,20 +113,17 @@ private BackupUtils() { } /** - * copy out Table RegionInfo into incremental backup image need to consider move this logic into - * HBackupFileSystem + * Copy table descriptors into the incremental backup image for use during restore. * @param conn connection * @param backupInfo backup info * @param conf configuration * @throws IOException exception */ - public static void copyTableRegionInfo(Connection conn, BackupInfo backupInfo, Configuration conf) - throws IOException { + public static void copyTableDescriptors(Connection conn, BackupInfo backupInfo, + Configuration conf) throws IOException { Path rootDir = CommonFSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); - // for each table in the table set, copy out the table info and region - // info files in the correct directory structure try (Admin admin = conn.getAdmin()) { for (TableName table : backupInfo.getTables()) { if (!admin.tableExists(table)) { @@ -150,38 +142,10 @@ public static void copyTableRegionInfo(Connection conn, BackupInfo backupInfo, C LOG.debug("Attempting to copy table info for:" + table + " target: " + target + " descriptor: " + orig); LOG.debug("Finished copying tableinfo."); - List regions = MetaTableAccessor.getTableRegions(conn, table); - // For each region, write the region info to disk - LOG.debug("Starting to write region info for table " + table); - for (RegionInfo regionInfo : regions) { - Path regionDir = FSUtils - .getRegionDirFromTableDir(new Path(backupInfo.getTableBackupDir(table)), regionInfo); - regionDir = new Path(backupInfo.getTableBackupDir(table), regionDir.getName()); - writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo); - } - LOG.debug("Finished writing region info for table " + table); } } } - /** - * Write the .regioninfo file on-disk. - */ - public static void writeRegioninfoOnFilesystem(final Configuration conf, final FileSystem fs, - final Path regionInfoDir, RegionInfo regionInfo) throws IOException { - final byte[] content = RegionInfo.toDelimitedByteArray(regionInfo); - Path regionInfoFile = new Path(regionInfoDir, "." + HConstants.REGIONINFO_QUALIFIER_STR); - // First check to get the permissions - FsPermission perms = CommonFSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY); - // Write the RegionInfo file content - FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null); - try { - out.write(content); - } finally { - out.close(); - } - } - /** * Parses hostname:port from WAL file path * @param p path to WAL file diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 4f5997495c26..de3952979f54 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -168,8 +168,8 @@ public void execute() throws IOException { LOG.debug("For incremental backup, current table set is " + backupManager.getIncrementalBackupTableSet()); newTimestamps = ((IncrementalBackupManager) backupManager).getIncrBackupLogFileMap(); - // copy out the table and region info files for each table - BackupUtils.copyTableRegionInfo(conn, backupInfo, conf); + // Copy table descriptors for restore. + BackupUtils.copyTableDescriptors(conn, backupInfo, conf); // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT convertWALsToHFiles(new HashMap<>(), new HashMap<>()); incrementalCopyHFiles(new String[] { getBulkOutputDir().toString() }, diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java index e14e9b1f08dc..e1bd2cbf1d38 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java @@ -23,6 +23,9 @@ import java.io.File; import java.util.List; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl; @@ -30,7 +33,9 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -95,10 +100,22 @@ public void TestIncBackupMergeRestore() throws Exception { String backupIdIncMultiple2 = client.backupTables(request); assertTrue(checkSucceeded(backupIdIncMultiple2)); + // Older incremental backups contain region info files that merge does not need to retain. + RegionInfo region = admin.getRegions(table1).get(0); + Path tableBackupPath = + HBackupFileSystem.getTableBackupPath(table1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple2); + Path regionInfoFile = new Path(new Path(tableBackupPath, region.getEncodedName()), + HRegionFileSystem.REGION_INFO_FILE); + FileSystem fs = regionInfoFile.getFileSystem(conf1); + try (FSDataOutputStream out = fs.create(regionInfoFile, true)) { + out.write(RegionInfo.toDelimitedByteArray(region)); + } + try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) { String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 }; bAdmin.mergeBackups(backups); } + assertFalse(fs.exists(regionInfoFile)); // #6 - restore incremental backup for multiple tables, with overwrite TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 }; diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestore.java index e3c0a550829b..1992396cb2ea 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestore.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestore.java @@ -18,13 +18,17 @@ package org.apache.hadoop.hbase.backup; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.HashSet; import java.util.List; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.SingleProcessHBaseCluster; import org.apache.hadoop.hbase.TableName; @@ -40,9 +44,11 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -130,6 +136,7 @@ public void testIncBackupRestore() throws Exception { request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); String backupIdIncMultiple = client.backupTables(request); assertTrue(checkSucceeded(backupIdIncMultiple)); + assertIncrementalBackupMetadata(backupIdIncMultiple, tables); BackupManifest manifest = HBackupFileSystem.getManifest(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple); assertEquals(Sets.newHashSet(table1, table2), new HashSet<>(manifest.getTableList())); @@ -161,6 +168,7 @@ public void testIncBackupRestore() throws Exception { request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR); String backupIdIncMultiple2 = client.backupTables(request); assertTrue(checkSucceeded(backupIdIncMultiple2)); + assertIncrementalBackupMetadata(backupIdIncMultiple2, tables); validateRootPathCanBeOverridden(BACKUP_ROOT_DIR, backupIdIncMultiple2); // #5 - restore full backup for all tables @@ -210,4 +218,20 @@ public void testIncBackupRestore() throws Exception { } } } + + private void assertIncrementalBackupMetadata(String backupId, List tables) + throws IOException { + Path backupRoot = new Path(BACKUP_ROOT_DIR); + FileSystem fs = backupRoot.getFileSystem(conf1); + for (TableName table : tables) { + Path tablePath = HBackupFileSystem.getTableBackupPath(table, backupRoot, backupId); + assertEquals(TEST_UTIL.getAdmin().getDescriptor(table), + FSTableDescriptors.getTableDescriptorFromFs(fs, tablePath)); + RemoteIterator files = fs.listFiles(tablePath, true); + while (files.hasNext()) { + Path path = files.next().getPath(); + assertNotEquals(HRegionFileSystem.REGION_INFO_FILE, path.getName(), path.toString()); + } + } + } }