HBASE-29798 Remove redundant region info from incremental backups

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java

@@ -362,8 +362,8 @@ public void execute() throws IOException, ColumnFamilyMismatchException {
 
     // case INCREMENTAL_COPY:
     try {
-      // copy out the table and region info files for each table
-      BackupUtils.copyTableRegionInfo(conn, backupInfo, conf);
+      // Copy table descriptors for restore.
+      BackupUtils.copyTableDescriptors(conn, backupInfo, conf);
       setupRegionLocator();
       // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT
       convertWALsToHFiles(tablesToWALFileList, tablesToPrevBackupTs);

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupMergeJob.java

@@ -45,7 +45,6 @@
 import org.apache.hadoop.hbase.backup.util.BackupUtils;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.util.Tool;
@@ -227,7 +226,6 @@ protected void copyMetaData(FileSystem fs, Path tmpBackupDir, Path backupDirPath
       String fileName = p.toString();
       if (
         fileName.indexOf(FSTableDescriptors.TABLEINFO_DIR) > 0
-          || fileName.indexOf(HRegionFileSystem.REGION_INFO_FILE) > 0
           || fileName.indexOf(BackupManifest.MANIFEST_FILE_NAME) > 0
       ) {
         toKeep.add(p);

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java

@@ -44,17 +44,14 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.backup.BackupInfo;
@@ -67,7 +64,6 @@
 import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
 import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -79,7 +75,6 @@
 import org.apache.hadoop.hbase.util.CommonFSUtils;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
@@ -118,20 +113,17 @@ private BackupUtils() {
   }
 
   /**
-   * copy out Table RegionInfo into incremental backup image need to consider move this logic into
-   * HBackupFileSystem
+   * Copy table descriptors into the incremental backup image for use during restore.
    * @param conn       connection
    * @param backupInfo backup info
    * @param conf       configuration
    * @throws IOException exception
    */
-  public static void copyTableRegionInfo(Connection conn, BackupInfo backupInfo, Configuration conf)
-    throws IOException {
+  public static void copyTableDescriptors(Connection conn, BackupInfo backupInfo,
+    Configuration conf) throws IOException {
     Path rootDir = CommonFSUtils.getRootDir(conf);
     FileSystem fs = rootDir.getFileSystem(conf);
 
-    // for each table in the table set, copy out the table info and region
-    // info files in the correct directory structure
     try (Admin admin = conn.getAdmin()) {
       for (TableName table : backupInfo.getTables()) {
         if (!admin.tableExists(table)) {
@@ -150,38 +142,10 @@ public static void copyTableRegionInfo(Connection conn, BackupInfo backupInfo, C
         LOG.debug("Attempting to copy table info for:" + table + " target: " + target
           + " descriptor: " + orig);
         LOG.debug("Finished copying tableinfo.");
-        List<RegionInfo> regions = MetaTableAccessor.getTableRegions(conn, table);
-        // For each region, write the region info to disk
-        LOG.debug("Starting to write region info for table " + table);
-        for (RegionInfo regionInfo : regions) {
-          Path regionDir = FSUtils
-            .getRegionDirFromTableDir(new Path(backupInfo.getTableBackupDir(table)), regionInfo);
-          regionDir = new Path(backupInfo.getTableBackupDir(table), regionDir.getName());
-          writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo);
-        }
-        LOG.debug("Finished writing region info for table " + table);
       }
     }
   }
 
-  /**
-   * Write the .regioninfo file on-disk.
-   */
-  public static void writeRegioninfoOnFilesystem(final Configuration conf, final FileSystem fs,
-    final Path regionInfoDir, RegionInfo regionInfo) throws IOException {
-    final byte[] content = RegionInfo.toDelimitedByteArray(regionInfo);
-    Path regionInfoFile = new Path(regionInfoDir, "." + HConstants.REGIONINFO_QUALIFIER_STR);
-    // First check to get the permissions
-    FsPermission perms = CommonFSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
-    // Write the RegionInfo file content
-    FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null);
-    try {
-      out.write(content);
-    } finally {
-      out.close();
-    }
-  }
-
   /**
    * Parses hostname:port from WAL file path
    * @param p path to WAL file

hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java

@@ -168,8 +168,8 @@ public void execute() throws IOException {
         LOG.debug("For incremental backup, current table set is "
           + backupManager.getIncrementalBackupTableSet());
         newTimestamps = ((IncrementalBackupManager) backupManager).getIncrBackupLogFileMap();
-        // copy out the table and region info files for each table
-        BackupUtils.copyTableRegionInfo(conn, backupInfo, conf);
+        // Copy table descriptors for restore.
+        BackupUtils.copyTableDescriptors(conn, backupInfo, conf);
         // convert WAL to HFiles and copy them to .tmp under BACKUP_ROOT
         convertWALsToHFiles(new HashMap<>(), new HashMap<>());
         incrementalCopyHFiles(new String[] { getBulkOutputDir().toString() },

hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupMerge.java

@@ -23,14 +23,19 @@
 
 import java.io.File;
 import java.util.List;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtil;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
 import org.apache.hadoop.hbase.backup.util.BackupUtils;
 import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
 import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
@@ -95,10 +100,22 @@ public void TestIncBackupMergeRestore() throws Exception {
     String backupIdIncMultiple2 = client.backupTables(request);
     assertTrue(checkSucceeded(backupIdIncMultiple2));
 
+    // Older incremental backups contain region info files that merge does not need to retain.
+    RegionInfo region = admin.getRegions(table1).get(0);
+    Path tableBackupPath =
+      HBackupFileSystem.getTableBackupPath(table1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple2);
+    Path regionInfoFile = new Path(new Path(tableBackupPath, region.getEncodedName()),
+      HRegionFileSystem.REGION_INFO_FILE);
+    FileSystem fs = regionInfoFile.getFileSystem(conf1);
+    try (FSDataOutputStream out = fs.create(regionInfoFile, true)) {
+      out.write(RegionInfo.toDelimitedByteArray(region));
+    }
+
     try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) {
       String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 };
       bAdmin.mergeBackups(backups);
     }
+    assertFalse(fs.exists(regionInfoFile));
 
     // #6 - restore incremental backup for multiple tables, with overwrite
     TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 };

hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupRestore.java

@@ -18,13 +18,17 @@
 package org.apache.hadoop.hbase.backup;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hbase.HBaseTestingUtil;
 import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
 import org.apache.hadoop.hbase.TableName;
@@ -40,9 +44,11 @@
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.FSTableDescriptors;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -130,6 +136,7 @@ public void testIncBackupRestore() throws Exception {
       request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
       String backupIdIncMultiple = client.backupTables(request);
       assertTrue(checkSucceeded(backupIdIncMultiple));
+      assertIncrementalBackupMetadata(backupIdIncMultiple, tables);
       BackupManifest manifest =
         HBackupFileSystem.getManifest(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple);
       assertEquals(Sets.newHashSet(table1, table2), new HashSet<>(manifest.getTableList()));
@@ -161,6 +168,7 @@ public void testIncBackupRestore() throws Exception {
       request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
       String backupIdIncMultiple2 = client.backupTables(request);
       assertTrue(checkSucceeded(backupIdIncMultiple2));
+      assertIncrementalBackupMetadata(backupIdIncMultiple2, tables);
       validateRootPathCanBeOverridden(BACKUP_ROOT_DIR, backupIdIncMultiple2);
 
       // #5 - restore full backup for all tables
@@ -210,4 +218,20 @@ public void testIncBackupRestore() throws Exception {
       }
     }
   }
+
+  private void assertIncrementalBackupMetadata(String backupId, List<TableName> tables)
+    throws IOException {
+    Path backupRoot = new Path(BACKUP_ROOT_DIR);
+    FileSystem fs = backupRoot.getFileSystem(conf1);
+    for (TableName table : tables) {
+      Path tablePath = HBackupFileSystem.getTableBackupPath(table, backupRoot, backupId);
+      assertEquals(TEST_UTIL.getAdmin().getDescriptor(table),
+        FSTableDescriptors.getTableDescriptorFromFs(fs, tablePath));
+      RemoteIterator<LocatedFileStatus> files = fs.listFiles(tablePath, true);
+      while (files.hasNext()) {
+        Path path = files.next().getPath();
+        assertNotEquals(HRegionFileSystem.REGION_INFO_FILE, path.getName(), path.toString());
+      }
+    }
+  }
 }