From 28ae4bce1a2edca9ca9352241406bb1037a9e3a4 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 09:21:05 +0700 Subject: [PATCH 01/15] General framework for new repository formats. The repository format has been fixed at 5 since 1.00. It was a compile-time constant written into every info file and manifest, and any mismatch at load was a hard error. A version could only read the format it wrote, so introducing a format meant a flag day where every repository had to be recreated. 1.00 was that flag day and this branch is so the next format does not need one. A file is readable when its format is between REPOSITORY_FORMAT_MIN and REPOSITORY_FORMAT_MAX. A version can read a format it does not write and a repository can hold both while older backups and archives expire. The format is requested with repo-format, which is repo indexed and valid only for stanza-create and stanza-upgrade. It is command line only so a format left in the configuration cannot migrate a stanza as a side effect of an upgrade run for another reason, such as a PostgreSQL version upgrade. The default stays 5. stanza-upgrade --repo1-format=6 rewrites the two info files and nothing else, so a repository of any size migrates in the time it takes to write them. A downgrade is refused, since the info files are what stop a version from reading a format it does not support. The two files are saved separately, so an upgrade interrupted between them leaves a mismatch that running the upgrade again repairs and that every command reading both files now reports. A backup set adopts a new format only at a full backup. A prior backup is a candidate for diff or incr only if it is at the format new backups are written with, and a resumable backup at another format is discarded, so a backup set is never mixed and its format can be read once rather than per file. Migrating therefore costs one full backup per stanza. Once the info files are at format 6 a version that supports only format 5 cannot read the stanza at all, including the backups still at format 5. Such a version fails at info load with "expected format 5 but found 6" before touching any data. --- build/config.yaml | 17 ++ doc/xml/reference.xml | 37 ++++ doc/xml/release/2020s/2026/2.60.0.xml | 12 ++ src/command/backup/backup.c | 11 +- src/command/backup/incr.c.inc | 6 + src/command/backup/resume.c.inc | 8 + src/command/check/check.c | 4 +- src/command/check/common.c | 48 ++++-- src/command/check/common.h | 5 +- src/command/stanza/create.c | 11 +- src/command/stanza/upgrade.c | 55 +++++- src/command/verify/verify.c | 4 +- src/config/config.auto.h | 3 +- src/config/parse.auto.c.inc | 38 +++++ src/info/info.c | 58 ++++++- src/info/info.h | 12 +- src/info/infoArchive.c | 22 ++- src/info/infoArchive.h | 12 +- src/info/infoBackup.c | 24 ++- src/info/infoBackup.h | 12 +- src/info/infoPg.c | 5 +- src/info/infoPg.h | 9 +- src/info/manifest/manifest.c | 9 +- src/info/manifest/manifest.h | 13 +- src/version.h | 14 +- test/src/harness/info.c | 17 +- test/src/harness/info.h | 7 + test/src/module/command/backupTest.c | 123 ++++++++++++-- test/src/module/command/checkTest.c | 70 +++++--- test/src/module/command/infoTest.c | 71 ++++---- test/src/module/command/repoTest.c | 3 +- test/src/module/command/restoreTest.c | 11 +- test/src/module/command/stanzaTest.c | 225 ++++++++++++++++++++++++- test/src/module/command/verifyTest.c | 6 +- test/src/module/config/parseTest.c | 2 + test/src/module/info/infoArchiveTest.c | 13 +- test/src/module/info/infoBackupTest.c | 19 ++- test/src/module/info/infoPgTest.c | 12 +- test/src/module/info/infoTest.c | 65 ++++++- test/src/module/info/manifestTest.c | 149 ++++++++-------- test/src/module/integration/allTest.c | 4 + test/src/module/performance/typeTest.c | 4 +- 42 files changed, 1023 insertions(+), 227 deletions(-) diff --git a/build/config.yaml b/build/config.yaml index 1e8d0f8de0..6b500af5a5 100644 --- a/build/config.yaml +++ b/build/config.yaml @@ -1688,6 +1688,23 @@ option: deprecate: repo-path: {} + # Command-line only so that a format left in a configuration file cannot upgrade a repository as a side effect of an unrelated + # stanza-upgrade. This does not cover the environment, which can set any option that is valid for the command, so a format left + # there does migrate on the next stanza-upgrade and the option reference says so. The allow list must be kept in sync with + # REPOSITORY_FORMAT_MIN/MAX in version.h + repo-format: + group: repo + type: integer + default: 5 + allow-list: + - 5 + - 6 + command: + stanza-create: {} + stanza-upgrade: {} + command-role: + main: {} + repo-cipher-type: section: global type: string-id diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index 663ac3a260..e6b8228f40 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -2920,6 +2920,23 @@ n + + @@ -2942,6 +2959,26 @@ n + + diff --git a/doc/xml/release/2020s/2026/2.60.0.xml b/doc/xml/release/2020s/2026/2.60.0.xml index d77f97f7e3..67b3d30b9e 100644 --- a/doc/xml/release/2020s/2026/2.60.0.xml +++ b/doc/xml/release/2020s/2026/2.60.0.xml @@ -64,6 +64,18 @@ + + + + + + + + + +

General framework for new repository formats.

+ + diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c index e3170c54c5..a4e564be37 100644 --- a/src/command/backup/backup.c +++ b/src/command/backup/backup.c @@ -167,6 +167,10 @@ backupInit(const InfoBackup *const infoBackup) { result->archiveInfo = infoArchiveLoadFile(storageRepo(), INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMain()); result->archiveId = infoArchiveId(result->archiveInfo); + + // Error if the info files do not agree with each other. A stanza left half migrated by an interrupted upgrade should be + // repaired before more is written into it, and backup is the command that will run against it next. + checkStanzaInfo(cfgOptionGroupIdxDefault(cfgOptGrpRepo), infoArchivePg(result->archiveInfo), infoBackupPg(infoBackup)); } FUNCTION_LOG_RETURN(BACKUP_DATA, result); @@ -225,9 +229,10 @@ cmdBackup(void) const ManifestBlockIncrMap blockIncrMap = backupBlockIncrMap(); Manifest *const manifest = manifestNewBuild( - backupData->storagePrimary, infoPg.version, infoPg.catalogVersion, timestampStart, cfgOptionBool(cfgOptOnline), - cfgOptionBool(cfgOptChecksumPage), cfgOptionBool(cfgOptRepoBundle), cfgOptionBool(cfgOptRepoBlock), &blockIncrMap, - strLstNewVarLst(cfgOptionLst(cfgOptExclude)), backupStartResult.tablespaceList); + backupData->storagePrimary, infoPg.version, infoPg.catalogVersion, infoBackupFormat(infoBackup), timestampStart, + cfgOptionBool(cfgOptOnline), cfgOptionBool(cfgOptChecksumPage), cfgOptionBool(cfgOptRepoBundle), + cfgOptionBool(cfgOptRepoBlock), &blockIncrMap, strLstNewVarLst(cfgOptionLst(cfgOptExclude)), + backupStartResult.tablespaceList); // Validate the manifest using the copy start time manifestBuildValidate( diff --git a/src/command/backup/incr.c.inc b/src/command/backup/incr.c.inc index 96c71de703..c59698b077 100644 --- a/src/command/backup/incr.c.inc +++ b/src/command/backup/incr.c.inc @@ -36,6 +36,12 @@ backupBuildIncrPrior(const InfoBackup *const infoBackup) if (infoPg.id != backupPrior.backupPgId) continue; + // The prior backup must be at the format new backups are written with so that a backup set is never mixed-format. + // After the repository format has been upgraded no prior will match and the backup will be changed to full below, + // which is the only point where a backup set adopts a new format. + if (backupPrior.backrestFormat != infoBackupFormat(infoBackup)) + continue; + // This backup is a candidate for prior backupLabelPrior = strDup(backupPrior.backupLabel); break; diff --git a/src/command/backup/resume.c.inc b/src/command/backup/resume.c.inc index 520d870b8d..4e911b8426 100644 --- a/src/command/backup/resume.c.inc +++ b/src/command/backup/resume.c.inc @@ -238,6 +238,14 @@ backupResumeFind(const Manifest *const manifest, const CipherSpec *const cipherS "new " PROJECT_NAME " version '%s' does not match resumable " PROJECT_NAME " version '%s'", strZ(manifestData(manifest)->backrestVersion), strZ(manifestResumeData->backrestVersion)); } + // Check repository format. The format applies to an entire backup, so files written at another + // format cannot be reused no matter how little else has changed. + else if (manifestFormat(manifestResume) != manifestFormat(manifest)) + { + reason = zNewFmt( + "new repository format %u does not match resumable repository format %u", + manifestFormat(manifest), manifestFormat(manifestResume)); + } // Check backup type because new backup label must be the same type as resume backup label else if (manifestResumeData->backupType != cfgOptionStrId(cfgOptType)) { diff --git a/src/command/check/check.c b/src/command/check/check.c index 28259223b8..6c1290296c 100644 --- a/src/command/check/check.c +++ b/src/command/check/check.c @@ -91,7 +91,7 @@ checkStandby(const DbGetResult dbGroup, const unsigned int pgPathDefinedTotal) // Check that the backup and archive info files exist and are valid for the current database of the stanza checkStanzaInfoPg( - storageRepo, dbPgControl(dbGroup.standby).version, dbPgControl(dbGroup.standby).systemId, + repoIdx, storageRepo, dbPgControl(dbGroup.standby).version, dbPgControl(dbGroup.standby).systemId, cfgCipherSpecMainIdx(repoIdx)); } @@ -136,7 +136,7 @@ checkPrimary(const DbGetResult dbGroup) // Check that the backup and archive info files exist and are valid for the current database of the stanza checkStanzaInfoPg( - storageRepo, dbPgControl(dbGroup.primary).version, dbPgControl(dbGroup.primary).systemId, + repoIdx, storageRepo, dbPgControl(dbGroup.primary).version, dbPgControl(dbGroup.primary).systemId, cfgCipherSpecMainIdx(repoIdx)); // Attempt to load the archive info file and retrieve the archiveId diff --git a/src/command/check/common.c b/src/command/check/common.c index d6f5e8e4ba..af6f86857d 100644 --- a/src/command/check/common.c +++ b/src/command/check/common.c @@ -95,27 +95,48 @@ checkDbConfig(const unsigned int pgVersion, const unsigned int pgIdx, const Db * /**********************************************************************************************************************************/ FN_EXTERN void -checkStanzaInfo(const InfoPgData *const archiveInfo, const InfoPgData *const backupInfo) +checkStanzaInfo(const unsigned int repoIdx, const InfoPg *const archiveInfoPg, const InfoPg *const backupInfoPg) { FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM_P(INFO_PG_DATA, archiveInfo); - FUNCTION_TEST_PARAM_P(INFO_PG_DATA, backupInfo); + FUNCTION_TEST_PARAM(UINT, repoIdx); + FUNCTION_TEST_PARAM(INFO_PG, archiveInfoPg); + FUNCTION_TEST_PARAM(INFO_PG, backupInfoPg); FUNCTION_TEST_END(); - ASSERT(archiveInfo != NULL); - ASSERT(backupInfo != NULL); + ASSERT(archiveInfoPg != NULL); + ASSERT(backupInfoPg != NULL); + + const InfoPgData archiveInfo = infoPgData(archiveInfoPg, infoPgDataCurrentId(archiveInfoPg)); + const InfoPgData backupInfo = infoPgData(backupInfoPg, infoPgDataCurrentId(backupInfoPg)); // Error if there is a mismatch between the archive and backup info files - if (archiveInfo->id != backupInfo->id || archiveInfo->systemId != backupInfo->systemId || - archiveInfo->version != backupInfo->version) + if (archiveInfo.id != backupInfo.id || archiveInfo.systemId != backupInfo.systemId || + archiveInfo.version != backupInfo.version) { THROW_FMT( FileInvalidError, "backup info file and archive info file do not match\n" "archive: id = %u, version = %s, system-id = %" PRIu64 "\n" "backup : id = %u, version = %s, system-id = %" PRIu64 "\n" "HINT: this may be a symptom of repository corruption!", - archiveInfo->id, strZ(pgVersionToStr(archiveInfo->version)), archiveInfo->systemId, backupInfo->id, - strZ(pgVersionToStr(backupInfo->version)), backupInfo->systemId); + archiveInfo.id, strZ(pgVersionToStr(archiveInfo.version)), archiveInfo.systemId, backupInfo.id, + strZ(pgVersionToStr(backupInfo.version)), backupInfo.systemId); + } + + // Error if the info files are at different repository formats. The formats are written together but stored apart, so an + // upgrade interrupted between the two saves leaves them here. + if (infoPgFormat(archiveInfoPg) != infoPgFormat(backupInfoPg)) + { + const unsigned int formatArchive = infoPgFormat(archiveInfoPg); + const unsigned int formatBackup = infoPgFormat(backupInfoPg); + + THROW_FMT( + FileInvalidError, + "backup info file and archive info file are at different repository formats\n" + "archive: format = %u\n" + "backup : format = %u\n" + "HINT: run " CFGCMD_STANZA_UPGRADE " with --%s=%u to complete an interrupted upgrade.", + formatArchive, formatBackup, cfgOptionIdxName(cfgOptRepoFormat, repoIdx), + formatArchive > formatBackup ? formatArchive : formatBackup); } FUNCTION_TEST_RETURN_VOID(); @@ -124,9 +145,11 @@ checkStanzaInfo(const InfoPgData *const archiveInfo, const InfoPgData *const bac /**********************************************************************************************************************************/ FN_EXTERN void checkStanzaInfoPg( - const Storage *const storage, const unsigned int pgVersion, const uint64_t pgSystemId, const CipherSpec *const cipherSpecMain) + const unsigned int repoIdx, const Storage *const storage, const unsigned int pgVersion, const uint64_t pgSystemId, + const CipherSpec *const cipherSpecMain) { FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, repoIdx); FUNCTION_TEST_PARAM(STORAGE, storage); FUNCTION_TEST_PARAM(UINT, pgVersion); FUNCTION_TEST_PARAM(UINT64, pgSystemId); @@ -141,10 +164,9 @@ checkStanzaInfoPg( const InfoArchive *const infoArchive = infoArchiveLoadFile(storage, INFO_ARCHIVE_PATH_FILE_STR, cipherSpecMain); const InfoPgData archiveInfoPg = infoPgData(infoArchivePg(infoArchive), infoPgDataCurrentId(infoArchivePg(infoArchive))); const InfoBackup *const infoBackup = infoBackupLoadFile(storage, INFO_BACKUP_PATH_FILE_STR, cipherSpecMain); - const InfoPgData backupInfoPg = infoPgData(infoBackupPg(infoBackup), infoPgDataCurrentId(infoBackupPg(infoBackup))); - // Check that the info files pg data match each other - checkStanzaInfo(&archiveInfoPg, &backupInfoPg); + // Check that the info files pg data and repository format match each other + checkStanzaInfo(repoIdx, infoArchivePg(infoArchive), infoBackupPg(infoBackup)); // Check that the version and system id match the current database if (pgVersion != archiveInfoPg.version || pgSystemId != archiveInfoPg.systemId) diff --git a/src/command/check/common.h b/src/command/check/common.h index 6879b1fb3a..f323723ddc 100644 --- a/src/command/check/common.h +++ b/src/command/check/common.h @@ -16,10 +16,11 @@ Functions FN_EXTERN void checkDbConfig(const unsigned int pgVersion, const unsigned int pgIdx, const Db *dbObject, bool isStandby); // Validate the archive and backup info files -FN_EXTERN void checkStanzaInfo(const InfoPgData *archiveInfo, const InfoPgData *backupInfo); +FN_EXTERN void checkStanzaInfo(unsigned int repoIdx, const InfoPg *archiveInfoPg, const InfoPg *backupInfoPg); // Load and validate the database data of the info files against each other and the current database FN_EXTERN void checkStanzaInfoPg( - const Storage *storage, const unsigned int pgVersion, const uint64_t pgSystemId, const CipherSpec *cipherSpecMain); + unsigned int repoIdx, const Storage *storage, const unsigned int pgVersion, const uint64_t pgSystemId, + const CipherSpec *cipherSpecMain); #endif diff --git a/src/command/stanza/create.c b/src/command/stanza/create.c index f40f437781..856ffe5f6e 100644 --- a/src/command/stanza/create.c +++ b/src/command/stanza/create.c @@ -74,11 +74,14 @@ cmdStanzaCreate(void) (backupNotEmpty && archiveNotEmpty ? "and/or " : ""), (archiveNotEmpty ? "archive directory " : "")); } + // Format for the new stanza + const unsigned int format = cfgOptionIdxUInt(cfgOptRepoFormat, repoIdx); + // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent archive files const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx)); // Create and save archive info - infoArchive = infoArchiveNew(pgControl.version, pgControl.systemId, cipherSpecArchive); + infoArchive = infoArchiveNew(pgControl.version, pgControl.systemId, format, cipherSpecArchive); infoArchiveSaveFile(infoArchive, storageRepoWriteStanza, INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx)); @@ -86,7 +89,8 @@ cmdStanzaCreate(void) const CipherSpec *const cipherSpecManifest = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx)); // Create and save backup info - infoBackup = infoBackupNew(pgControl.version, pgControl.systemId, pgControl.catalogVersion, cipherSpecManifest); + infoBackup = infoBackupNew( + pgControl.version, pgControl.systemId, pgControl.catalogVersion, format, cipherSpecManifest); infoBackupSaveFile(infoBackup, storageRepoWriteStanza, INFO_BACKUP_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx)); } @@ -95,7 +99,8 @@ cmdStanzaCreate(void) { // Error if there is a mismatch between the archive and backup info files or the database version/system Id matches // current database - checkStanzaInfoPg(storageRepoReadStanza, pgControl.version, pgControl.systemId, cfgCipherSpecMainIdx(repoIdx)); + checkStanzaInfoPg( + repoIdx, storageRepoReadStanza, pgControl.version, pgControl.systemId, cfgCipherSpecMainIdx(repoIdx)); // The files are valid - upgrade const String *sourceFile = NULL; diff --git a/src/command/stanza/upgrade.c b/src/command/stanza/upgrade.c index 45a3cb81f4..0db843e7e7 100644 --- a/src/command/stanza/upgrade.c +++ b/src/command/stanza/upgrade.c @@ -58,6 +58,33 @@ cmdStanzaUpgrade(void) storageRepoReadStanza, INFO_BACKUP_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx)); InfoPgData backupInfo = infoPgData(infoBackupPg(infoBackup), infoPgDataCurrentId(infoBackupPg(infoBackup))); + // Determine the format to write. An upgrade interrupted between the two info file saves leaves them at different + // formats and the higher of the two is the only target that does not downgrade a file, so it is written to both even + // when no format was requested. A format that is not already in the repository must still be requested, since the + // option default would otherwise downgrade a repository that has already been upgraded. + const unsigned int formatArchive = infoArchiveFormat(infoArchive); + const unsigned int formatBackup = infoBackupFormat(infoBackup); + unsigned int format = formatArchive > formatBackup ? formatArchive : formatBackup; + + if (cfgOptionIdxSource(cfgOptRepoFormat, repoIdx) != cfgSourceDefault) + { + const unsigned int formatRequest = cfgOptionIdxUInt(cfgOptRepoFormat, repoIdx); + + // Error when the format would be downgraded. Backups and archives written at a newer format would no longer be + // gated by the info files, so an older version could read the info files and then fail on newer files. + if (formatRequest < format) + { + THROW_FMT( + FormatError, + "unable to downgrade repository format from %u to %u\n" + "HINT: backups and archives already written at format %u would not be readable by a version that only" + " supports format %u.", + format, formatRequest, format, formatRequest); + } + + format = formatRequest; + } + // Since the file save of archive.info and backup.info are not atomic, then check and update each separately. // Update archive if (pgControl.version != archiveInfo.version || pgControl.systemId != archiveInfo.systemId) @@ -73,11 +100,29 @@ cmdStanzaUpgrade(void) infoBackupUpgrade = true; } - // Get the backup and archive info pg data and throw an error if the ids do not match before saving (even if only one - // needed to be updated) - backupInfo = infoPgData(infoBackupPg(infoBackup), infoPgDataCurrentId(infoBackupPg(infoBackup))); - archiveInfo = infoPgData(infoArchivePg(infoArchive), infoPgDataCurrentId(infoArchivePg(infoArchive))); - checkStanzaInfo(&archiveInfo, &backupInfo); + // Update the format on both info files together so they never disagree + if (format != formatArchive || format != formatBackup) + { + // Report a repository that was found at two formats. It is repaired here but a prior upgrade did not finish, which + // the user has not been told about since the run it happened on did not get far enough to say so. + if (formatArchive != formatBackup) + LOG_WARN("repository format mismatch from an interrupted " CFGCMD_STANZA_UPGRADE " will be repaired"); + + // Log the format the repository is migrating from, which is the lower of the two when an interrupted upgrade left + // them at different formats. This cannot be undone and a version that does not support the new format will no + // longer be able to read the stanza, so say so rather than migrating silently. + LOG_INFO_FMT( + "upgrade repository format from %u to %u", formatArchive < formatBackup ? formatArchive : formatBackup, format); + + infoArchiveFormatSet(infoArchive, format); + infoBackupFormatSet(infoBackup, format); + + infoArchiveUpgrade = true; + infoBackupUpgrade = true; + } + + // Throw an error if the info files do not match before saving (even if only one needed to be updated) + checkStanzaInfo(repoIdx, infoArchivePg(infoArchive), infoBackupPg(infoBackup)); // Save archive info if (infoArchiveUpgrade) diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c index c4faf44a2f..f30d40fe3e 100644 --- a/src/command/verify/verify.c +++ b/src/command/verify/verify.c @@ -502,9 +502,7 @@ verifyPgHistory(const InfoPg *const archiveInfoPg, const InfoPg *const backupInf { // Check archive.info and backup.info current PG data matches. If there is a mismatch, verify cannot continue since // the database is not considered accessible during the verify command so no way to tell which would be valid. - const InfoPgData archiveInfoPgData = infoPgData(archiveInfoPg, infoPgDataCurrentId(archiveInfoPg)); - const InfoPgData backupInfoPgData = infoPgData(backupInfoPg, infoPgDataCurrentId(backupInfoPg)); - checkStanzaInfo(&archiveInfoPgData, &backupInfoPgData); + checkStanzaInfo(cfgOptionGroupIdxDefault(cfgOptGrpRepo), archiveInfoPg, backupInfoPg); const unsigned int archiveInfoHistoryTotal = infoPgDataTotal(archiveInfoPg); const unsigned int backupInfoHistoryTotal = infoPgDataTotal(backupInfoPg); diff --git a/src/config/config.auto.h b/src/config/config.auto.h index 3c34ee5bd0..01a584076c 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -145,7 +145,7 @@ Option constants #define CFGOPT_VERBOSE "verbose" #define CFGOPT_VERSION "version" -#define CFG_OPTION_TOTAL 199 +#define CFG_OPTION_TOTAL 200 /*********************************************************************************************************************************** Option value constants @@ -586,6 +586,7 @@ typedef enum cfgOptRepoBundleSize, cfgOptRepoCipherPass, cfgOptRepoCipherType, + cfgOptRepoFormat, cfgOptRepoGcsBucket, cfgOptRepoGcsEndpoint, cfgOptRepoGcsKey, diff --git a/src/config/parse.auto.c.inc b/src/config/parse.auto.c.inc index 49d8432dca..e6c871842c 100644 --- a/src/config/parse.auto.c.inc +++ b/src/config/parse.auto.c.inc @@ -59,6 +59,7 @@ static const StringPubConst parseRuleValueStr[] = PARSE_RULE_STRPUB("443"), // val/str PARSE_RULE_STRPUB("4MiB"), // val/str PARSE_RULE_STRPUB("4PiB"), // val/str + PARSE_RULE_STRPUB("5"), // val/str PARSE_RULE_STRPUB("512KiB"), // val/str PARSE_RULE_STRPUB("5432"), // val/str PARSE_RULE_STRPUB("5MiB"), // val/str @@ -197,6 +198,7 @@ typedef enum parseRuleValStrQT_443_QT, // val/str/enum parseRuleValStrQT_4MiB_QT, // val/str/enum parseRuleValStrQT_4PiB_QT, // val/str/enum + parseRuleValStrQT_5_QT, // val/str/enum parseRuleValStrQT_512KiB_QT, // val/str/enum parseRuleValStrQT_5432_QT, // val/str/enum parseRuleValStrQT_5MiB_QT, // val/str/enum @@ -301,6 +303,7 @@ static const int parseRuleValueInt[] = 1, // val/int 2, // val/int 3, // val/int + 5, // val/int 6, // val/int 9, // val/int 12, // val/int @@ -330,6 +333,7 @@ static const uint8_t parseRuleValueIntStrMap[] = parseRuleValStrQT_1_QT, // val/int/strmap parseRuleValStrQT_2_QT, // val/int/strmap parseRuleValStrQT_3_QT, // val/int/strmap + parseRuleValStrQT_5_QT, // val/int/strmap parseRuleValStrQT_6_QT, // val/int/strmap parseRuleValStrQT_9_QT, // val/int/strmap parseRuleValStrQT_12_QT, // val/int/strmap @@ -359,6 +363,7 @@ typedef enum parseRuleValInt1, // val/int/enum parseRuleValInt2, // val/int/enum parseRuleValInt3, // val/int/enum + parseRuleValInt5, // val/int/enum parseRuleValInt6, // val/int/enum parseRuleValInt9, // val/int/enum parseRuleValInt12, // val/int/enum @@ -6014,6 +6019,38 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] = ), // opt/repo-cipher-type ), // opt/repo-cipher-type // ----------------------------------------------------------------------------------------------------------------------------- + PARSE_RULE_OPTION // opt/repo-format + ( // opt/repo-format + PARSE_RULE_OPTION_NAME("repo-format"), // opt/repo-format + PARSE_RULE_OPTION_TYPE(Integer), // opt/repo-format + PARSE_RULE_OPTION_REQUIRED(true), // opt/repo-format + PARSE_RULE_OPTION_SECTION(CommandLine), // opt/repo-format + PARSE_RULE_OPTION_GROUP_ID(Repo), // opt/repo-format + // opt/repo-format + PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-format + ( // opt/repo-format + PARSE_RULE_OPTION_COMMAND(StanzaCreate) // opt/repo-format + PARSE_RULE_OPTION_COMMAND(StanzaUpgrade) // opt/repo-format + ), // opt/repo-format + // opt/repo-format + PARSE_RULE_OPTIONAL // opt/repo-format + ( // opt/repo-format + PARSE_RULE_OPTIONAL_GROUP // opt/repo-format + ( // opt/repo-format + PARSE_RULE_OPTIONAL_ALLOW_LIST // opt/repo-format + ( // opt/repo-format + PARSE_RULE_VAL_INT(5), // opt/repo-format + PARSE_RULE_VAL_INT(6), // opt/repo-format + ), // opt/repo-format + // opt/repo-format + PARSE_RULE_OPTIONAL_DEFAULT // opt/repo-format + ( // opt/repo-format + PARSE_RULE_VAL_INT(5), // opt/repo-format + ), // opt/repo-format + ), // opt/repo-format + ), // opt/repo-format + ), // opt/repo-format + // ----------------------------------------------------------------------------------------------------------------------------- PARSE_RULE_OPTION // opt/repo-gcs-bucket ( // opt/repo-gcs-bucket PARSE_RULE_OPTION_NAME("repo-gcs-bucket"), // opt/repo-gcs-bucket @@ -12070,6 +12107,7 @@ static const uint8_t optionResolveOrder[] = cfgOptRepoBundleLimit, // opt-resolve-order cfgOptRepoBundleSize, // opt-resolve-order cfgOptRepoCipherType, // opt-resolve-order + cfgOptRepoFormat, // opt-resolve-order cfgOptRepoGcsUserProject, // opt-resolve-order cfgOptRepoHardlink, // opt-resolve-order cfgOptRepoLocal, // opt-resolve-order diff --git a/src/info/info.c b/src/info/info.c index ad135054ee..d31941f1b1 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -94,18 +94,22 @@ BUFFER_STRDEF_STATIC(INFO_CHECKSUM_END_BUF, "}}"); /**********************************************************************************************************************************/ FN_EXTERN Info * -infoNew(const CipherSpec *const cipherSpecSub) +infoNew(const unsigned int format, const CipherSpec *const cipherSpecSub) { FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(UINT, format); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpecSub); FUNCTION_LOG_END(); + ASSERT(format >= REPOSITORY_FORMAT_MIN && format <= REPOSITORY_FORMAT_MAX); + OBJ_NEW_BEGIN(Info, .childQty = MEM_CONTEXT_QTY_MAX) { *this = (Info){}; // Cipher used to encrypt/decrypt subsequent dependent files. Value may be NULL. infoCipherSpecSet(this, cipherSpecSub); + this->pub.format = format; this->pub.backrestVersion = STRDEF(PROJECT_VERSION); } OBJ_NEW_END(); @@ -179,15 +183,33 @@ infoNewLoad( // Process backrest section if (strEqZ(value->section, INFO_SECTION_BACKREST)) { - // Validate format + // Validate and store format if (strEqZ(value->key, INFO_KEY_FORMAT)) { - if (varUInt64(jsonToVar(value->value)) != REPOSITORY_FORMAT) + const uint64_t format = varUInt64(jsonToVar(value->value)); + + // A format newer than this version can read requires an upgrade. Do not suggest a version since + // this version cannot know which version added the format. + if (format > REPOSITORY_FORMAT_MAX) + { + THROW_FMT( + FormatError, + "repository format %" PRIu64 " requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + // A format older than this version can read requires an older version to migrate the repository + if (format < REPOSITORY_FORMAT_MIN) { THROW_FMT( - FormatError, "expected format %d but found %" PRIu64, REPOSITORY_FORMAT, - varUInt64(jsonToVar(value->value))); + FormatError, + "repository format %" PRIu64 " is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); } + + this->pub.format = (unsigned int)format; } // Store pgBackRest version else if (strEqZ(value->key, INFO_KEY_VERSION)) @@ -241,7 +263,8 @@ infoNewLoad( INFO_CHECKSUM_END(checksumActualFilter); - // Verify the checksum + // Verify the checksum first so a file that is empty or not an info file at all is reported as a checksum failure + // rather than as a missing format const String *const checksumActual = strNewEncode( encodingHex, pckReadBinP(pckReadNew(ioFilterResult(checksumActualFilter)))); @@ -253,6 +276,11 @@ infoNewLoad( ChecksumError, "invalid checksum, actual '%s' but expected '%s'", strZ(checksumActual), strZ(checksumExpected)); } + + // The format defines how everything else is read, so a file without one is not an info file this version can use. The + // format is zero until the key is found and the value stored, so if we got here then the key was not found. + if (infoFormat(this) == 0) + THROW(FormatError, "repository format not found\nHINT: is this a valid " PROJECT_NAME " info file?"); } MEM_CONTEXT_TEMP_END(); @@ -371,7 +399,7 @@ infoSave(Info *const this, IoWrite *const write, InfoSaveCallback *const callbac // Add version and format callbackFunction(callbackData, STRDEF(INFO_SECTION_BACKREST), &data); - infoSaveValue(&data, INFO_SECTION_BACKREST, INFO_KEY_FORMAT, jsonFromVar(VARUINT(REPOSITORY_FORMAT))); + infoSaveValue(&data, INFO_SECTION_BACKREST, INFO_KEY_FORMAT, jsonFromVar(VARUINT(infoFormat(this)))); infoSaveValue(&data, INFO_SECTION_BACKREST, INFO_KEY_VERSION, jsonFromVar(VARSTRDEF(PROJECT_VERSION))); // Add cipher passphrase if defined @@ -405,6 +433,22 @@ infoSave(Info *const this, IoWrite *const write, InfoSaveCallback *const callbac /*********************************************************************************************************************************** Getters/Setters ***********************************************************************************************************************************/ +FN_EXTERN void +infoFormatSet(Info *const this, const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(INFO, this); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(format >= REPOSITORY_FORMAT_MIN && format <= REPOSITORY_FORMAT_MAX); + + this->pub.format = format; + + FUNCTION_TEST_RETURN_VOID(); +} + FN_EXTERN void infoCipherSpecSet(Info *const this, const CipherSpec *const cipherSpec) { diff --git a/src/info/info.h b/src/info/info.h index 2901ad65c1..3af5818afc 100644 --- a/src/info/info.h +++ b/src/info/info.h @@ -39,7 +39,7 @@ typedef void InfoSaveCallback(void *data, const String *sectionNext, InfoSave *i /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -FN_EXTERN Info *infoNew(const CipherSpec *cipherSpecSub); +FN_EXTERN Info *infoNew(unsigned int format, const CipherSpec *cipherSpecSub); // Create new object and load contents from a file. The cipher spec the file is read with supplies the type for the cipher spec // built from the pass stored in it. @@ -51,10 +51,20 @@ Getters/Setters ***********************************************************************************************************************************/ typedef struct InfoPub { + unsigned int format; // Repository format the file was written with const String *backrestVersion; // pgBackRest version const CipherSpec *cipherSpec; // Cipher spec for dependent files } InfoPub; +// Repository format +FN_INLINE_ALWAYS unsigned int +infoFormat(const Info *const this) +{ + return THIS_PUB(Info)->format; +} + +FN_EXTERN void infoFormatSet(Info *this, unsigned int format); + // Cipher spec for the files that depend on this one, e.g. the manifest for backup.info. Never NULL, so it can be handed on // without a check, and none when there is no pass. FN_INLINE_ALWAYS const CipherSpec * diff --git a/src/info/infoArchive.c b/src/info/infoArchive.c index 8752e530f2..4e38a60038 100644 --- a/src/info/infoArchive.c +++ b/src/info/infoArchive.c @@ -57,11 +57,13 @@ infoArchiveNewInternal(void) /**********************************************************************************************************************************/ FN_EXTERN InfoArchive * -infoArchiveNew(const unsigned int pgVersion, const uint64_t pgSystemId, const CipherSpec *const cipherSpecSub) +infoArchiveNew( + const unsigned int pgVersion, const uint64_t pgSystemId, const unsigned int format, const CipherSpec *const cipherSpecSub) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(UINT, pgVersion); FUNCTION_LOG_PARAM(UINT64, pgSystemId); + FUNCTION_LOG_PARAM(UINT, format); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpecSub); FUNCTION_LOG_END(); @@ -74,7 +76,7 @@ infoArchiveNew(const unsigned int pgVersion, const uint64_t pgSystemId, const Ci this = infoArchiveNewInternal(); // Initialize the pg data - this->pub.infoPg = infoPgNew(infoPgArchive, cipherSpecSub); + this->pub.infoPg = infoPgNew(infoPgArchive, format, cipherSpecSub); infoArchivePgSet(this, pgVersion, pgSystemId); } OBJ_NEW_END(); @@ -202,6 +204,22 @@ infoArchivePgSet(InfoArchive *const this, const unsigned int pgVersion, const ui FUNCTION_LOG_RETURN(INFO_ARCHIVE, this); } +/**********************************************************************************************************************************/ +FN_EXTERN void +infoArchiveFormatSet(InfoArchive *const this, const unsigned int format) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(INFO_ARCHIVE, this); + FUNCTION_LOG_PARAM(UINT, format); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + + infoFormatSet(infoPgInfo(infoArchivePg(this)), format); + + FUNCTION_LOG_RETURN_VOID(); +} + /**********************************************************************************************************************************/ typedef struct InfoArchiveLoadFileData { diff --git a/src/info/infoArchive.h b/src/info/infoArchive.h index 15647d0ffe..d8c861cd76 100644 --- a/src/info/infoArchive.h +++ b/src/info/infoArchive.h @@ -29,7 +29,8 @@ STRING_DECLARE(INFO_ARCHIVE_PATH_FILE_COPY_STR); /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -FN_EXTERN InfoArchive *infoArchiveNew(const unsigned int pgVersion, const uint64_t pgSystemId, const CipherSpec *cipherSpecSub); +FN_EXTERN InfoArchive *infoArchiveNew( + const unsigned int pgVersion, const uint64_t pgSystemId, unsigned int format, const CipherSpec *cipherSpecSub); // Create new object and load contents from IoRead FN_EXTERN InfoArchive *infoArchiveNewLoad(IoRead *read, const CipherSpec *cipherSpec); @@ -66,6 +67,15 @@ infoArchiveCipherSpec(const InfoArchive *const this) return infoPgCipherSpec(infoArchivePg(this)); } +// Repository format +FN_INLINE_ALWAYS unsigned int +infoArchiveFormat(const InfoArchive *const this) +{ + return infoPgFormat(infoArchivePg(this)); +} + +FN_EXTERN void infoArchiveFormatSet(InfoArchive *this, unsigned int format); + /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index 90097c3480..5385097a2f 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -66,13 +66,14 @@ infoBackupNewInternal(void) /**********************************************************************************************************************************/ FN_EXTERN InfoBackup * infoBackupNew( - const unsigned int pgVersion, const uint64_t pgSystemId, const unsigned int pgCatalogVersion, + const unsigned int pgVersion, const uint64_t pgSystemId, const unsigned int pgCatalogVersion, const unsigned int format, const CipherSpec *const cipherSpecSub) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(UINT, pgVersion); FUNCTION_LOG_PARAM(UINT64, pgSystemId); FUNCTION_LOG_PARAM(UINT, pgCatalogVersion); + FUNCTION_LOG_PARAM(UINT, format); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpecSub); FUNCTION_LOG_END(); @@ -85,7 +86,7 @@ infoBackupNew( this = infoBackupNewInternal(); // Initialize the pg data - this->pub.infoPg = infoPgNew(infoPgBackup, cipherSpecSub); + this->pub.infoPg = infoPgNew(infoPgBackup, format, cipherSpecSub); infoBackupPgSet(this, pgVersion, pgSystemId, pgCatalogVersion); } OBJ_NEW_END(); @@ -390,6 +391,23 @@ infoBackupPgSet( FUNCTION_LOG_RETURN(INFO_BACKUP, this); } +/**********************************************************************************************************************************/ +FN_EXTERN void +infoBackupFormatSet(InfoBackup *const this, const unsigned int format) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(INFO_BACKUP, this); + FUNCTION_LOG_PARAM(UINT, format); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + + infoFormatSet(infoPgInfo(infoBackupPg(this)), format); + this->pub.updated = true; + + FUNCTION_LOG_RETURN_VOID(); +} + /**********************************************************************************************************************************/ FN_EXTERN InfoBackupData infoBackupData(const InfoBackup *const this, const unsigned int backupDataIdx) @@ -455,7 +473,7 @@ infoBackupDataAdd(InfoBackup *const this, const Manifest *const manifest) InfoBackupData infoBackupData = { .backupLabel = strDup(manData->backupLabel), - .backrestFormat = REPOSITORY_FORMAT, + .backrestFormat = manifestFormat(manifest), .backrestVersion = strDup(manData->backrestVersion), .backupInfoRepoSize = backupRepoSize, .backupInfoRepoSizeDelta = backupRepoSizeDelta, diff --git a/src/info/infoBackup.h b/src/info/infoBackup.h index e1ca7e7acc..78fd2e26ca 100644 --- a/src/info/infoBackup.h +++ b/src/info/infoBackup.h @@ -78,7 +78,8 @@ typedef struct InfoBackupData Constructors ***********************************************************************************************************************************/ FN_EXTERN InfoBackup *infoBackupNew( - unsigned int pgVersion, uint64_t pgSystemId, unsigned int pgCatalogVersion, const CipherSpec *cipherSpecSub); + unsigned int pgVersion, uint64_t pgSystemId, unsigned int pgCatalogVersion, unsigned int format, + const CipherSpec *cipherSpecSub); // Create new object and load contents from IoRead FN_EXTERN InfoBackup *infoBackupNewLoad(IoRead *read, const CipherSpec *cipherSpec); @@ -110,6 +111,15 @@ infoBackupCipherSpec(const InfoBackup *const this) return infoPgCipherSpec(infoBackupPg(this)); } +// Repository format +FN_INLINE_ALWAYS unsigned int +infoBackupFormat(const InfoBackup *const this) +{ + return infoPgFormat(infoBackupPg(this)); +} + +FN_EXTERN void infoBackupFormatSet(InfoBackup *this, unsigned int format); + // Return a structure of the backup data from a specific index FN_EXTERN InfoBackupData infoBackupData(const InfoBackup *this, unsigned int backupDataIdx); diff --git a/src/info/infoPg.c b/src/info/infoPg.c index 1ff542b0a7..ebced1fc02 100644 --- a/src/info/infoPg.c +++ b/src/info/infoPg.c @@ -62,10 +62,11 @@ infoPgNewInternal(const InfoPgType type) /**********************************************************************************************************************************/ FN_EXTERN InfoPg * -infoPgNew(const InfoPgType type, const CipherSpec *const cipherSpecSub) +infoPgNew(const InfoPgType type, const unsigned int format, const CipherSpec *const cipherSpecSub) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING_ID, type); + FUNCTION_LOG_PARAM(UINT, format); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpecSub); FUNCTION_LOG_END(); @@ -74,7 +75,7 @@ infoPgNew(const InfoPgType type, const CipherSpec *const cipherSpecSub) OBJ_NEW_BASE_BEGIN(InfoPg, .childQty = MEM_CONTEXT_QTY_MAX) { this = infoPgNewInternal(type); - this->pub.info = infoNew(cipherSpecSub); + this->pub.info = infoNew(format, cipherSpecSub); } OBJ_NEW_END(); diff --git a/src/info/infoPg.h b/src/info/infoPg.h index 43788667bf..b0344b44f7 100644 --- a/src/info/infoPg.h +++ b/src/info/infoPg.h @@ -47,7 +47,7 @@ typedef enum /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -FN_EXTERN InfoPg *infoPgNew(InfoPgType type, const CipherSpec *cipherSpecSub); +FN_EXTERN InfoPg *infoPgNew(InfoPgType type, unsigned int format, const CipherSpec *cipherSpecSub); // Create new object and load contents from a file FN_EXTERN InfoPg *infoPgNewLoad( @@ -91,6 +91,13 @@ FN_EXTERN InfoPgData infoPgDataCurrent(const InfoPg *this); // Current history index FN_EXTERN unsigned int infoPgDataCurrentId(const InfoPg *this); +// Repository format +FN_INLINE_ALWAYS unsigned int +infoPgFormat(const InfoPg *const this) +{ + return infoFormat(infoPgInfo(this)); +} + // Total PostgreSQL data in the history FN_INLINE_ALWAYS unsigned int infoPgDataTotal(const InfoPg *const this) diff --git a/src/info/manifest/manifest.c b/src/info/manifest/manifest.c index d3759f5b8b..ebd3ea1678 100644 --- a/src/info/manifest/manifest.c +++ b/src/info/manifest/manifest.c @@ -76,14 +76,15 @@ manifestNewInternal(void) FN_EXTERN Manifest * manifestNewBuild( - const Storage *const storagePg, const unsigned int pgVersion, const unsigned int pgCatalogVersion, const time_t timestampStart, - const bool online, const bool checksumPage, const bool bundle, const bool blockIncr, const ManifestBlockIncrMap *blockIncrMap, - const StringList *const excludeList, const Pack *const tablespaceList) + const Storage *const storagePg, const unsigned int pgVersion, const unsigned int pgCatalogVersion, const unsigned int format, + const time_t timestampStart, const bool online, const bool checksumPage, const bool bundle, const bool blockIncr, + const ManifestBlockIncrMap *blockIncrMap, const StringList *const excludeList, const Pack *const tablespaceList) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STORAGE, storagePg); FUNCTION_LOG_PARAM(UINT, pgVersion); FUNCTION_LOG_PARAM(UINT, pgCatalogVersion); + FUNCTION_LOG_PARAM(UINT, format); FUNCTION_LOG_PARAM(TIME, timestampStart); FUNCTION_LOG_PARAM(BOOL, online); FUNCTION_LOG_PARAM(BOOL, checksumPage); @@ -102,7 +103,7 @@ manifestNewBuild( OBJ_NEW_BASE_BEGIN(Manifest, .childQty = MEM_CONTEXT_QTY_MAX) { this = manifestNewInternal(); - this->pub.info = infoNew(NULL); + this->pub.info = infoNew(format, NULL); this->pub.data.backrestVersion = strNewZ(PROJECT_VERSION); this->pub.data.pgVersion = pgVersion; this->pub.data.pgCatalogVersion = pgCatalogVersion; diff --git a/src/info/manifest/manifest.h b/src/info/manifest/manifest.h index 78507cdbe2..474436ee33 100644 --- a/src/info/manifest/manifest.h +++ b/src/info/manifest/manifest.h @@ -213,9 +213,9 @@ Constructors ***********************************************************************************************************************************/ // Build a new manifest for a PostgreSQL data directory FN_EXTERN Manifest *manifestNewBuild( - const Storage *storagePg, unsigned int pgVersion, unsigned int pgCatalogVersion, time_t timestampStart, bool online, - bool checksumPage, bool bundle, bool blockIncr, const ManifestBlockIncrMap *blockIncrMap, const StringList *excludeList, - const Pack *tablespaceList); + const Storage *storagePg, unsigned int pgVersion, unsigned int pgCatalogVersion, unsigned int format, time_t timestampStart, + bool online, bool checksumPage, bool bundle, bool blockIncr, const ManifestBlockIncrMap *blockIncrMap, + const StringList *excludeList, const Pack *tablespaceList); // Load a manifest from IO FN_EXTERN Manifest *manifestNewLoad(IoRead *read, const CipherSpec *cipherSpec); @@ -236,6 +236,13 @@ typedef struct ManifestPub StringList *referenceList; // List of file references } ManifestPub; +// Repository format +FN_INLINE_ALWAYS unsigned int +manifestFormat(const Manifest *const this) +{ + return infoFormat(THIS_PUB(Manifest)->info); +} + // Get cipher spec for the files this manifest describes, set the pass they are encrypted with FN_INLINE_ALWAYS const CipherSpec * manifestCipherSpec(const Manifest *const this) diff --git a/src/version.h b/src/version.h index 3c6bcc63a0..d1e078206e 100644 --- a/src/version.h +++ b/src/version.h @@ -25,10 +25,18 @@ Config include path name. The parent path will vary based on configuration. #define PROJECT_CONFIG_INCLUDE_PATH "conf.d" /*********************************************************************************************************************************** -Format Number -- defines format for info and manifest files as well as on-disk structure. If this number changes then the repository -will be invalid unless migration functions are written. +Format Number -- defines format for info and manifest files as well as on-disk structure. Each info file and manifest stores the +format it was written with, so a repository may contain more than one format while older backups and archives expire. + +A constant is defined for each format so that code which varies by format can be explicit about the format it applies to. +REPOSITORY_FORMAT_MIN/MAX are the range that can be read. The allow list for repo-format in build/config.yaml must be kept in sync +with MIN/MAX and its default is the format used for new repositories. ***********************************************************************************************************************************/ -#define REPOSITORY_FORMAT 5 +#define REPOSITORY_FORMAT_5 5 +#define REPOSITORY_FORMAT_6 6 + +#define REPOSITORY_FORMAT_MIN REPOSITORY_FORMAT_5 +#define REPOSITORY_FORMAT_MAX REPOSITORY_FORMAT_6 /*********************************************************************************************************************************** Project version components. PROJECT_VERSION and PROJECT_VERSION_NUM are automatically generated from the component parts. diff --git a/test/src/harness/info.c b/test/src/harness/info.c index e1d76594d8..d99f178e4c 100644 --- a/test/src/harness/info.c +++ b/test/src/harness/info.c @@ -23,9 +23,10 @@ This prevents churn in headers and checksums in the unit tests. We purposefully here as a cross-check of that code. ***********************************************************************************************************************************/ Buffer * -harnessInfoChecksum(const String *info) +harnessInfoChecksumFormat(const unsigned int format, const String *info) { FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(UINT, format); FUNCTION_HARNESS_PARAM(STRING, info); FUNCTION_HARNESS_END(); @@ -42,7 +43,7 @@ harnessInfoChecksum(const String *info) result = bufNew(strSize(info) + 256); bufCat(result, BUFSTRDEF("[backrest]\nbackrest-format=")); - bufCat(result, BUFSTR(jsonFromVar(VARUINT(REPOSITORY_FORMAT)))); + bufCat(result, BUFSTR(jsonFromVar(VARUINT(format)))); bufCat(result, BUFSTRDEF("\nbackrest-version=")); bufCat(result, BUFSTR(jsonFromVar(VARSTRDEF(PROJECT_VERSION)))); bufCat(result, BUFSTRDEF("\n\n")); @@ -103,6 +104,18 @@ harnessInfoChecksumZ(const char *info) FUNCTION_HARNESS_RETURN(BUFFER, harnessInfoChecksum(STR(info))); } +Buffer * +harnessInfoChecksum(const String *const info) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STRING, info); + FUNCTION_HARNESS_END(); + + ASSERT(info != NULL); + + FUNCTION_HARNESS_RETURN(BUFFER, harnessInfoChecksumFormat(REPOSITORY_FORMAT_DEFAULT, info)); +} + /*********************************************************************************************************************************** Test callback that logs the results to a string ***********************************************************************************************************************************/ diff --git a/test/src/harness/info.h b/test/src/harness/info.h index 6fdd75eebb..e1457ee7df 100644 --- a/test/src/harness/info.h +++ b/test/src/harness/info.h @@ -6,6 +6,12 @@ Harness for Generating Test Info Files #include "harness/storage.h" +/*********************************************************************************************************************************** +Format that new repositories are created with, i.e. the default of the repo-format option in build/config.yaml. Only tests need to +know this since the option supplies it everywhere else. +***********************************************************************************************************************************/ +#define REPOSITORY_FORMAT_DEFAULT REPOSITORY_FORMAT_5 + /*********************************************************************************************************************************** Write info to a file and add the checksum ***********************************************************************************************************************************/ @@ -22,6 +28,7 @@ Write info to a file and add the checksum Functions ***********************************************************************************************************************************/ Buffer *harnessInfoChecksum(const String *info); +Buffer *harnessInfoChecksumFormat(unsigned int format, const String *info); Buffer *harnessInfoChecksumZ(const char *info); void harnessInfoLoadNewCallback(void *callbackData, const String *section, const String *key, JsonRead *json); diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index 8e4008f07b..853726305b 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -13,6 +13,7 @@ Test Backup Command #include "harness/backup.h" #include "harness/blockIncr.h" #include "harness/config.h" +#include "harness/info.h" #include "harness/manifest.h" #include "harness/pack.h" #include "harness/postgres.h" @@ -1518,7 +1519,10 @@ testRun(void) HRN_CFG_LOAD(cfgCmdBackup, argList); TEST_RESULT_VOID( - backupInit(infoBackupNew(PG_VERSION_96, HRN_PG_SYSTEMID_96, hrnPgCatalogVersion(PG_VERSION_96), NULL)), "backup init"); + backupInit( + infoBackupNew( + PG_VERSION_96, HRN_PG_SYSTEMID_96, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, NULL)), + "backup init"); TEST_RESULT_LOG( "P00 WARN: option backup-standby is enabled but backup is offline - backups will be performed from the primary"); @@ -1538,13 +1542,17 @@ testRun(void) HRN_CFG_LOAD(cfgCmdBackup, argList); TEST_ERROR( - backupInit(infoBackupNew(PG_VERSION_11, HRN_PG_SYSTEMID_11, hrnPgCatalogVersion(PG_VERSION_11), NULL)), + backupInit( + infoBackupNew( + PG_VERSION_11, HRN_PG_SYSTEMID_11, hrnPgCatalogVersion(PG_VERSION_11), REPOSITORY_FORMAT_DEFAULT, NULL)), BackupMismatchError, "PostgreSQL version 10, system-id " HRN_PG_SYSTEMID_10_Z " do not match stanza version 11, system-id" " " HRN_PG_SYSTEMID_11_Z "\n" "HINT: is this the correct stanza?"); TEST_ERROR( - backupInit(infoBackupNew(PG_VERSION_10, HRN_PG_SYSTEMID_11, hrnPgCatalogVersion(PG_VERSION_10), NULL)), + backupInit( + infoBackupNew( + PG_VERSION_10, HRN_PG_SYSTEMID_11, hrnPgCatalogVersion(PG_VERSION_10), REPOSITORY_FORMAT_DEFAULT, NULL)), BackupMismatchError, "PostgreSQL version 10, system-id " HRN_PG_SYSTEMID_10_Z " do not match stanza version 10, system-id" " " HRN_PG_SYSTEMID_11_Z "\n" @@ -1581,7 +1589,10 @@ testRun(void) TEST_RESULT_VOID( dbFree( - backupInit(infoBackupNew(PG_VERSION_18, HRN_PG_SYSTEMID_18, hrnPgCatalogVersion(PG_VERSION_18), NULL))->dbPrimary), + backupInit( + infoBackupNew( + PG_VERSION_18, HRN_PG_SYSTEMID_18, hrnPgCatalogVersion(PG_VERSION_18), REPOSITORY_FORMAT_DEFAULT, + NULL))->dbPrimary), "backup init"); TEST_RESULT_BOOL(cfgOptionBool(cfgOptChecksumPage), false, "check checksum-page"); @@ -1608,7 +1619,10 @@ testRun(void) TEST_RESULT_VOID( dbFree( - backupInit(infoBackupNew(PG_VERSION_18, HRN_PG_SYSTEMID_18, hrnPgCatalogVersion(PG_VERSION_18), NULL))->dbPrimary), + backupInit( + infoBackupNew( + PG_VERSION_18, HRN_PG_SYSTEMID_18, hrnPgCatalogVersion(PG_VERSION_18), REPOSITORY_FORMAT_DEFAULT, + NULL))->dbPrimary), "backup init"); TEST_RESULT_BOOL(cfgOptionBool(cfgOptChecksumPage), false, "check checksum-page"); @@ -1621,7 +1635,10 @@ testRun(void) TEST_RESULT_VOID( dbFree( - backupInit(infoBackupNew(PG_VERSION_18, HRN_PG_SYSTEMID_18, hrnPgCatalogVersion(PG_VERSION_18), NULL))->dbPrimary), + backupInit( + infoBackupNew( + PG_VERSION_18, HRN_PG_SYSTEMID_18, hrnPgCatalogVersion(PG_VERSION_18), REPOSITORY_FORMAT_DEFAULT, + NULL))->dbPrimary), "backup init"); TEST_RESULT_BOOL(cfgOptionBool(cfgOptChecksumPage), false, "check checksum-page"); } @@ -1669,12 +1686,34 @@ testRun(void) HRN_PQ_SCRIPT_TIME_QUERY(1, 1575392589999)); BackupData *backupData = backupInit( - infoBackupNew(PG_VERSION_18, HRN_PG_SYSTEMID_18, hrnPgCatalogVersion(PG_VERSION_18), NULL)); + infoBackupNew(PG_VERSION_18, HRN_PG_SYSTEMID_18, hrnPgCatalogVersion(PG_VERSION_18), REPOSITORY_FORMAT_DEFAULT, NULL)); TEST_RESULT_INT(backupTime(backupData, true), 1575392588, "multiple tries for sleep"); TEST_ERROR(backupTime(backupData, true), KernelError, "PostgreSQL clock has not advanced to the next second after 3 tries"); dbFree(backupData->dbPrimary); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("info files that do not agree on repository format"); + + // Migrate archive.info only, which is the state an upgrade interrupted between the two saves leaves behind + InfoArchive *const infoArchive = infoArchiveLoadFile(storageRepo(), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone()); + infoArchiveFormatSet(infoArchive, REPOSITORY_FORMAT_6); + infoArchiveSaveFile(infoArchive, storageRepoWrite(), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone()); + + HRN_PQ_SCRIPT_SET( + // Connect to primary + HRN_PQ_SCRIPT_OPEN(1, "dbname='postgres' port=5432", PG_VERSION_18, TEST_PATH "/pg1", false, NULL, NULL)); + + TEST_ERROR( + backupInit( + infoBackupNew( + PG_VERSION_18, HRN_PG_SYSTEMID_18, hrnPgCatalogVersion(PG_VERSION_18), REPOSITORY_FORMAT_DEFAULT, NULL)), + FileInvalidError, + "backup info file and archive info file are at different repository formats\n" + "archive: format = 6\n" + "backup : format = 5\n" + "HINT: run stanza-upgrade with --repo1-format=6 to complete an interrupted upgrade."); } // ***************************************************************************************************************************** @@ -1724,6 +1763,7 @@ testRun(void) OBJ_NEW_BASE_BEGIN(Manifest, .childQty = MEM_CONTEXT_QTY_MAX) { manifest = manifestNewInternal(); + manifest->pub.info = infoNew(REPOSITORY_FORMAT_DEFAULT, NULL); manifest->pub.data.backupType = backupTypeFull; manifest->pub.data.backrestVersion = strNewZ("BOGUS"); } @@ -1745,7 +1785,7 @@ testRun(void) OBJ_NEW_BASE_BEGIN(Manifest, .childQty = MEM_CONTEXT_QTY_MAX) { manifestResume = manifestNewInternal(); - manifestResume->pub.info = infoNew(NULL); + manifestResume->pub.info = infoNew(REPOSITORY_FORMAT_DEFAULT, NULL); manifestResume->pub.data.backupType = backupTypeFull; manifestResume->pub.data.backupLabel = strNewZ("20191003-105320F"); manifestResume->pub.data.backupTimestampStart = 1482182860; @@ -1773,6 +1813,27 @@ testRun(void) manifest->pub.data.backrestVersion = STRDEF(PROJECT_VERSION); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("cannot resume when repository format has changed"); + + infoFormatSet(manifestResume->pub.info, REPOSITORY_FORMAT_6); + + manifestSave( + manifestResume, + storageWriteIo( + storageNewWriteP( + storageRepoWrite(), STRDEF(STORAGE_REPO_BACKUP "/20191003-105320F/" BACKUP_MANIFEST_FILE INFO_COPY_EXT)))); + + TEST_RESULT_PTR(backupResumeFind(manifest, cipherSpecNewNone()), NULL, "find resumable backup"); + + TEST_RESULT_LOG( + "P00 WARN: backup '20191003-105320F' cannot be resumed:" + " new repository format 5 does not match resumable repository format 6"); + + TEST_STORAGE_LIST_EMPTY(storageRepo(), STORAGE_REPO_BACKUP, .comment = "check backup path removed"); + + infoFormatSet(manifestResume->pub.info, REPOSITORY_FORMAT_DEFAULT); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("cannot resume when backup labels do not match (resumable is null)"); @@ -2155,6 +2216,30 @@ testRun(void) strLstSize(storageListP(storageRepoIdx(1), strNewFmt(STORAGE_PATH_BACKUP "/test1"))), backupCount + 1, "new backup repo2"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("diff changed to full when no prior backup is at the repository format"); + + // Set log level to warn so only the conversion is logged + harnessLogLevelSet(logLevelWarn); + + // Upgrade the format of the repo, which leaves every backup in it at the prior format + InfoBackup *infoBackup = infoBackupLoadFile(storageRepoIdx(1), INFO_BACKUP_PATH_FILE_STR, cfgCipherSpecMainIdx(1)); + infoBackupFormatSet(infoBackup, REPOSITORY_FORMAT_6); + infoBackupSaveFile(infoBackup, storageRepoIdxWrite(1), INFO_BACKUP_PATH_FILE_STR, cfgCipherSpecMainIdx(1)); + + HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, "VR3"); + + TEST_RESULT_VOID(hrnCmdBackup(), "backup"); + + TEST_RESULT_LOG("P00 WARN: no prior backup exists, diff backup has been changed to full"); + + // The new full backup adopts the upgraded format + infoBackup = infoBackupLoadFile(storageRepoIdx(1), INFO_BACKUP_PATH_FILE_STR, cfgCipherSpecMainIdx(1)); + + TEST_RESULT_UINT( + infoBackupData(infoBackup, infoBackupDataTotal(infoBackup) - 1).backrestFormat, REPOSITORY_FORMAT_6, + "new backup at upgraded format"); + // Cleanup hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2); harnessLogLevelReset(); @@ -2215,8 +2300,8 @@ testRun(void) // Create a backup manifest that looks like a halted backup manifest Manifest *manifestResume = manifestNewBuild( - storagePg(), PG_VERSION_14, hrnPgCatalogVersion(PG_VERSION_14), backupTimeStart, true, false, false, false, NULL, - NULL, NULL); + storagePg(), PG_VERSION_14, hrnPgCatalogVersion(PG_VERSION_14), REPOSITORY_FORMAT_DEFAULT, backupTimeStart, true, + false, false, false, NULL, NULL, NULL); manifestResume->pub.data.backupType = backupTypeFull; const String *resumeLabel = backupLabelCreate(backupTypeFull, NULL, backupTimeStart); @@ -2295,8 +2380,8 @@ testRun(void) // Create a backup manifest that looks like a halted backup manifest Manifest *manifestResume = manifestNewBuild( - storagePg(), PG_VERSION_14, hrnPgCatalogVersion(PG_VERSION_14), backupTimeStart, true, false, false, false, NULL, - NULL, NULL); + storagePg(), PG_VERSION_14, hrnPgCatalogVersion(PG_VERSION_14), REPOSITORY_FORMAT_DEFAULT, backupTimeStart, true, + false, false, false, NULL, NULL, NULL); manifestResume->pub.data.backupType = backupTypeFull; manifestResume->pub.data.backupOptionCompressType = compressTypeGz; @@ -2457,8 +2542,8 @@ testRun(void) // Create a backup manifest that looks like a halted backup manifest Manifest *manifestResume = manifestNewBuild( - storagePg(), PG_VERSION_14, hrnPgCatalogVersion(PG_VERSION_14), backupTimeStart, true, false, false, false, NULL, - NULL, NULL); + storagePg(), PG_VERSION_14, hrnPgCatalogVersion(PG_VERSION_14), REPOSITORY_FORMAT_DEFAULT, backupTimeStart, true, + false, false, false, NULL, NULL, NULL); manifestResume->pub.data.backupOptionCompressType = compressTypeGz; const String *resumeLabel = backupLabelCreate(backupTypeFull, NULL, backupTimeStart - 100000); @@ -2959,16 +3044,19 @@ testRun(void) HRN_STORAGE_PATH_REMOVE(storagePgWrite(), strZ(pgWalPath(PG_VERSION_14))); HRN_STORAGE_PATH_CREATE(storagePgWrite(), strZ(pgWalPath(PG_VERSION_11)), .noParentCreate = true); - // Upgrade stanza + // Upgrade stanza to format 6 StringList *argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath); + hrnCfgArgRawZ(argList, cfgOptRepoFormat, "6"); hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path); hrnCfgArgRawBool(argList, cfgOptOnline, false); HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); cmdStanzaUpgrade(); - TEST_RESULT_LOG("P00 INFO: stanza-upgrade for stanza 'test1' on repo1"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'test1' on repo1\n" + "P00 INFO: upgrade repository format from 5 to 6"); // Load options argList = strLstNew(); @@ -2977,7 +3065,7 @@ testRun(void) hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path); hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1"); hrnCfgArgRawBool(argList, cfgOptRepoSymlink, false); - hrnCfgArgRawStrId(argList, cfgOptType, backupTypeFull); + // Do not specify type=full since the update to repo format 6 should force a full upgrade hrnCfgArgRawBool(argList, cfgOptRepoHardlink, true); hrnCfgArgRawZ(argList, cfgOptManifestSaveThreshold, "1"); hrnCfgArgRawBool(argList, cfgOptArchiveCopy, true); @@ -3059,6 +3147,7 @@ testRun(void) #pragma GCC diagnostic pop TEST_RESULT_LOG( + "P00 WARN: no prior backup exists, incr backup has been changed to full\n" "P00 INFO: execute backup start: backup begins after the next regular checkpoint completes\n" "P00 INFO: backup start archive = 0000000105DB5DE000000000, lsn = 5db5de0/0\n" "P00 INFO: check archive for segment 0000000105DB5DE000000000\n" diff --git a/test/src/module/command/checkTest.c b/test/src/module/command/checkTest.c index 629877ec51..09888a0f39 100644 --- a/test/src/module/command/checkTest.c +++ b/test/src/module/command/checkTest.c @@ -672,22 +672,20 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("checkStanzaInfo() - files match"); - InfoArchive *archiveInfo = infoArchiveNew(PG_VERSION_96, 6569239123849665679, NULL); - InfoPgData archivePg = infoPgData(infoArchivePg(archiveInfo), infoPgDataCurrentId(infoArchivePg(archiveInfo))); + InfoArchive *archiveInfo = infoArchiveNew(PG_VERSION_96, 6569239123849665679, REPOSITORY_FORMAT_DEFAULT, NULL); + InfoBackup *backupInfo = infoBackupNew( + PG_VERSION_96, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, NULL); - InfoBackup *backupInfo = infoBackupNew(PG_VERSION_96, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_96), NULL); - InfoPgData backupPg = infoPgData(infoBackupPg(backupInfo), infoPgDataCurrentId(infoBackupPg(backupInfo))); - - TEST_RESULT_VOID(checkStanzaInfo(&archivePg, &backupPg), "stanza info files match"); + TEST_RESULT_VOID(checkStanzaInfo(0, infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), "stanza info files match"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("checkStanzaInfo() - corrupted backup file: system id"); - backupInfo = infoBackupNew(PG_VERSION_96, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_96), NULL); - backupPg = infoPgData(infoBackupPg(backupInfo), infoPgDataCurrentId(infoBackupPg(backupInfo))); + backupInfo = infoBackupNew( + PG_VERSION_96, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, NULL); TEST_ERROR( - checkStanzaInfo(&archivePg, &backupPg), FileInvalidError, + checkStanzaInfo(0, infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FileInvalidError, "backup info file and archive info file do not match\n" "archive: id = 1, version = 9.6, system-id = 6569239123849665679\n" "backup : id = 1, version = 9.6, system-id = 6569239123849665999\n" @@ -696,11 +694,11 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("checkStanzaInfo() - corrupted backup file: system id and version"); - backupInfo = infoBackupNew(PG_VERSION_18, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_18), NULL); - backupPg = infoPgData(infoBackupPg(backupInfo), infoPgDataCurrentId(infoBackupPg(backupInfo))); + backupInfo = infoBackupNew( + PG_VERSION_18, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_18), REPOSITORY_FORMAT_DEFAULT, NULL); TEST_ERROR( - checkStanzaInfo(&archivePg, &backupPg), FileInvalidError, + checkStanzaInfo(0, infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FileInvalidError, "backup info file and archive info file do not match\n" "archive: id = 1, version = 9.6, system-id = 6569239123849665679\n" "backup : id = 1, version = 18, system-id = 6569239123849665999\n" @@ -709,11 +707,11 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("checkStanzaInfo() - corrupted backup file: version"); - backupInfo = infoBackupNew(PG_VERSION_18, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_18), NULL); - backupPg = infoPgData(infoBackupPg(backupInfo), infoPgDataCurrentId(infoBackupPg(backupInfo))); + backupInfo = infoBackupNew( + PG_VERSION_18, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_18), REPOSITORY_FORMAT_DEFAULT, NULL); TEST_ERROR( - checkStanzaInfo(&archivePg, &backupPg), FileInvalidError, + checkStanzaInfo(0, infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FileInvalidError, "backup info file and archive info file do not match\n" "archive: id = 1, version = 9.6, system-id = 6569239123849665679\n" "backup : id = 1, version = 18, system-id = 6569239123849665679\n" @@ -723,15 +721,49 @@ testRun(void) TEST_TITLE("checkStanzaInfo() - corrupted backup file: db id"); infoBackupPgSet(backupInfo, PG_VERSION_96, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_96)); - backupPg = infoPgData(infoBackupPg(backupInfo), infoPgDataCurrentId(infoBackupPg(backupInfo))); TEST_ERROR( - checkStanzaInfo(&archivePg, &backupPg), FileInvalidError, + checkStanzaInfo(0, infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FileInvalidError, "backup info file and archive info file do not match\n" "archive: id = 1, version = 9.6, system-id = 6569239123849665679\n" "backup : id = 2, version = 9.6, system-id = 6569239123849665679\n" "HINT: this may be a symptom of repository corruption!"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("checkStanzaInfo() - info files at different repository formats"); + + backupInfo = infoBackupNew( + PG_VERSION_96, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, NULL); + infoArchiveFormatSet(archiveInfo, REPOSITORY_FORMAT_6); + + TEST_ERROR( + checkStanzaInfo(0, infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FileInvalidError, + "backup info file and archive info file are at different repository formats\n" + "archive: format = 6\n" + "backup : format = 5\n" + "HINT: run stanza-upgrade with --repo1-format=6 to complete an interrupted upgrade."); + + // The upgrade is always to the newer format, whichever file is behind. Add a second repository to check that the hint + // names the repository the mismatch was found in rather than sending the user to migrate repo1. + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); + hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, TEST_PATH "/repo"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 2, TEST_PATH "/repo2"); + HRN_CFG_LOAD(cfgCmdCheck, argList); + + infoArchiveFormatSet(archiveInfo, REPOSITORY_FORMAT_5); + infoBackupFormatSet(backupInfo, REPOSITORY_FORMAT_6); + + TEST_ERROR( + checkStanzaInfo(1, infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FileInvalidError, + "backup info file and archive info file are at different repository formats\n" + "archive: format = 5\n" + "backup : format = 6\n" + "HINT: run stanza-upgrade with --repo2-format=6 to complete an interrupted upgrade."); + + infoBackupFormatSet(backupInfo, REPOSITORY_FORMAT_DEFAULT); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("checkStanzaInfoPg() - version mismatch"); @@ -753,7 +785,7 @@ testRun(void) // Version mismatch TEST_ERROR( - checkStanzaInfoPg(storageRepoIdx(0), PG_VERSION_18, HRN_PG_SYSTEMID_18, cfgCipherSpecMainIdx(0)), FileInvalidError, + checkStanzaInfoPg(0, storageRepoIdx(0), PG_VERSION_18, HRN_PG_SYSTEMID_18, cfgCipherSpecMainIdx(0)), FileInvalidError, "backup and archive info files exist but do not match the database\n" "HINT: is this the correct stanza?\n" "HINT: did an error occur during stanza-upgrade?"); @@ -763,7 +795,7 @@ testRun(void) // SystemId mismatch TEST_ERROR( - checkStanzaInfoPg(storageRepoIdx(0), PG_VERSION_96, 6569239123849665699, cfgCipherSpecMainIdx(0)), FileInvalidError, + checkStanzaInfoPg(0, storageRepoIdx(0), PG_VERSION_96, 6569239123849665699, cfgCipherSpecMainIdx(0)), FileInvalidError, "backup and archive info files exist but do not match the database\n" "HINT: is this the correct stanza?\n" "HINT: did an error occur during stanza-upgrade?"); diff --git a/test/src/module/command/infoTest.c b/test/src/module/command/infoTest.c index dd156e1b72..da25dcef42 100644 --- a/test/src/module/command/infoTest.c +++ b/test/src/module/command/infoTest.c @@ -502,38 +502,43 @@ testRun(void) STORAGE_REPO_ARCHIVE "/9.5-2/0000000100000000/000000010000000000000001-ac61b8f1ec7b1e6c3eaee9345214595eb7daa9a1.gz", .comment = "write WAL db2 timeline 1 repo1"); - HRN_INFO_PUT( - storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, - "[db]\n" - "db-catalog-version=201608131\n" - "db-control-version=960\n" - "db-id=3\n" - "db-system-id=6569239123849665679\n" - "db-version=\"9.6\"\n" - "\n" - "[backup:current]\n" - "20181116-154756F={\"backrest-format\":5,\"backrest-version\":\"2.04\"," - "\"backup-archive-start\":null,\"backup-archive-stop\":null," - "\"backup-info-repo-size\":3159776,\"backup-info-repo-size-delta\":3159,\"backup-info-size\":26897030," - "\"backup-info-size-delta\":26897030,\"backup-timestamp-start\":1542383276,\"backup-timestamp-stop\":1542383289," - "\"backup-type\":\"full\",\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false," - "\"option-backup-standby\":false,\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false," - "\"option-online\":true}\n" - "20201116-154900F={\"backrest-format\":5,\"backrest-version\":\"2.30\"," - "\"backup-archive-start\":\"000000030000000000000001\",\"backup-archive-stop\":\"000000030000000000000001\"," - "\"backup-info-repo-size\":3159776,\"backup-info-repo-size-delta\":3159,\"backup-info-size\":26897033," - "\"backup-info-size-delta\":26897033,\"backup-timestamp-start\":1605541676,\"backup-timestamp-stop\":1605541680," - "\"backup-type\":\"full\",\"db-id\":3,\"option-archive-check\":true,\"option-archive-copy\":false," - "\"option-backup-standby\":false,\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false," - "\"option-online\":true}\n" - "\n" - "[db:history]\n" - "1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679" - ",\"db-version\":\"9.6\"}\n" - "2={\"db-catalog-version\":201510051,\"db-control-version\":960,\"db-system-id\":6569239123849665666" - ",\"db-version\":\"9.5\"}\n" - "3={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679" - ",\"db-version\":\"9.6\"}\n"); + // The repository was migrated to format 6 after the first backup, so each backup carries the format it was written + // with rather than the format of the info file + const Buffer *const backupInfoContent = harnessInfoChecksumFormat( + REPOSITORY_FORMAT_6, + STRDEF( + "[db]\n" + "db-catalog-version=201608131\n" + "db-control-version=960\n" + "db-id=3\n" + "db-system-id=6569239123849665679\n" + "db-version=\"9.6\"\n" + "\n" + "[backup:current]\n" + "20181116-154756F={\"backrest-format\":5,\"backrest-version\":\"2.04\"," + "\"backup-archive-start\":null,\"backup-archive-stop\":null," + "\"backup-info-repo-size\":3159776,\"backup-info-repo-size-delta\":3159,\"backup-info-size\":26897030," + "\"backup-info-size-delta\":26897030,\"backup-timestamp-start\":1542383276,\"backup-timestamp-stop\":1542383289," + "\"backup-type\":\"full\",\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false," + "\"option-backup-standby\":false,\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false," + "\"option-online\":true}\n" + "20201116-154900F={\"backrest-format\":6,\"backrest-version\":\"2.30\"," + "\"backup-archive-start\":\"000000030000000000000001\",\"backup-archive-stop\":\"000000030000000000000001\"," + "\"backup-info-repo-size\":3159776,\"backup-info-repo-size-delta\":3159,\"backup-info-size\":26897033," + "\"backup-info-size-delta\":26897033,\"backup-timestamp-start\":1605541676,\"backup-timestamp-stop\":1605541680," + "\"backup-type\":\"full\",\"db-id\":3,\"option-archive-check\":true,\"option-archive-copy\":false," + "\"option-backup-standby\":false,\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false," + "\"option-online\":true}\n" + "\n" + "[db:history]\n" + "1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679" + ",\"db-version\":\"9.6\"}\n" + "2={\"db-catalog-version\":201510051,\"db-control-version\":960,\"db-system-id\":6569239123849665666" + ",\"db-version\":\"9.5\"}\n" + "3={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679" + ",\"db-version\":\"9.6\"}\n")); + + HRN_STORAGE_PUT(storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, backupInfoContent); // Execute while backup and restore locks are held HRN_FORK_BEGIN() @@ -636,7 +641,7 @@ testRun(void) "\"stop\":\"000000030000000000000001\"" "}," "\"backrest\":{" - "\"format\":5," + "\"format\":6," "\"version\":\"2.30\"" "}," "\"database\":{" diff --git a/test/src/module/command/repoTest.c b/test/src/module/command/repoTest.c index 4507f039fe..a71d7e5d11 100644 --- a/test/src/module/command/repoTest.c +++ b/test/src/module/command/repoTest.c @@ -386,7 +386,8 @@ testRun(void) "pg_data={\"path\":\"/var/lib/pgsql/12/data\",\"type\":\"path\"}\n" "\n" "[backrest]\n" - "backrest-checksum=\"31706010d1aa7e850191b4de9e76dc1ed13fb855\"\n"); + "backrest-checksum=\"ec7b178fd6d568bf592f16c36329ef6b533a2232\"\n" + "backrest-format=5\n"); const Buffer *backupLabelBuffer = BUFSTRDEF("BACKUP-LABEL"); diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index dea46a47fa..53d9fe0d3e 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -119,7 +119,7 @@ testManifestMinimal(const String *label, unsigned int pgVersion, const String *p OBJ_NEW_BASE_BEGIN(Manifest, .childQty = MEM_CONTEXT_QTY_MAX) { result = manifestNewInternal(); - result->pub.info = infoNew(NULL); + result->pub.info = infoNew(REPOSITORY_FORMAT_DEFAULT, NULL); result->pub.data.backupLabel = strDup(label); result->pub.data.backupTimestampStart = 1; @@ -2260,7 +2260,7 @@ testRun(void) OBJ_NEW_BASE_BEGIN(Manifest, .childQty = MEM_CONTEXT_QTY_MAX) { manifest = manifestNewInternal(); - manifest->pub.info = infoNew(NULL); + manifest->pub.info = infoNew(REPOSITORY_FORMAT_DEFAULT, NULL); manifest->pub.data.backupLabel = strNewZ(TEST_LABEL); manifest->pub.data.pgVersion = PG_VERSION_11; manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_11); @@ -2331,7 +2331,8 @@ testRun(void) // Write archive.info to the encrypted repo InfoArchive *infoArchive = infoArchiveNew( - PG_VERSION_11, 6569239123849665679, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); + PG_VERSION_11, 6569239123849665679, REPOSITORY_FORMAT_DEFAULT, + cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); infoArchiveSaveFile( infoArchive, storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS))); @@ -2378,7 +2379,7 @@ testRun(void) HRN_INFO_PUT(storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO "\n" TEST_RESTORE_BACKUP_INFO_DB); // Store archive.info to repo1 - repo1 will be selected because of the priority order - infoArchive = infoArchiveNew(PG_VERSION_11, 6569239123849665679, NULL); + infoArchive = infoArchiveNew(PG_VERSION_11, 6569239123849665679, REPOSITORY_FORMAT_DEFAULT, NULL); infoArchiveSaveFile( infoArchive, storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone()); @@ -2658,7 +2659,7 @@ testRun(void) OBJ_NEW_BASE_BEGIN(Manifest, .childQty = MEM_CONTEXT_QTY_MAX) { manifest = manifestNewInternal(); - manifest->pub.info = infoNew(NULL); + manifest->pub.info = infoNew(REPOSITORY_FORMAT_DEFAULT, NULL); manifest->pub.data.backupLabel = strNewZ(TEST_LABEL); manifest->pub.data.backupTimestampStart = 1482182860; manifest->pub.data.pgVersion = PG_VERSION_10; diff --git a/test/src/module/command/stanzaTest.c b/test/src/module/command/stanzaTest.c index 0154e0653b..1519a0603e 100644 --- a/test/src/module/command/stanzaTest.c +++ b/test/src/module/command/stanzaTest.c @@ -117,6 +117,9 @@ testRun(void) hrnCfgArgKeyRawStrId(argList, cfgOptRepoCipherType, 2, cipherTypeAes256Cbc); hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 2, "12345678"); hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 3, TEST_PATH "/repo3"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoFormat, 3, "6"); + // Request a format for repo1 as well, where the stanza already exists + hrnCfgArgKeyRawZ(argList, cfgOptRepoFormat, 1, "6"); hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 4, TEST_PATH "/repo4"); hrnCfgArgKeyRawStrId(argList, cfgOptRepoCipherType, 4, cipherTypeAes256Cbc); hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 4, "87654321"); @@ -150,7 +153,8 @@ testRun(void) bufEq(cipherSpecPass(infoArchiveCipherSpec(infoArchive)), cipherSpecPass(infoBackupCipherSpec(infoBackup))), false, "cipher sub different for archive and backup"); - // Confirm non-encrypted repo created successfully + // Confirm non-encrypted repo created successfully. This repo was created with an explicit format so also confirm that the + // requested format was stored rather than the default. TEST_ASSIGN( infoArchive, infoArchiveLoadFile(storageRepoIdx(2), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone()), @@ -158,6 +162,7 @@ testRun(void) TEST_RESULT_UINT( cipherSpecType(infoArchiveCipherSpec(infoArchive)), cipherTypeNone, "archive cipher sub not set on non-encrypted repo"); + TEST_RESULT_UINT(infoArchiveFormat(infoArchive), REPOSITORY_FORMAT_6, "archive info at requested format"); TEST_ASSIGN( infoBackup, infoBackupLoadFile(storageRepoIdx(2), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewNone()), @@ -165,6 +170,16 @@ testRun(void) TEST_RESULT_UINT( cipherSpecType(infoBackupCipherSpec(infoBackup)), cipherTypeNone, "backup cipher sub not set on non-encrypted repo"); + TEST_RESULT_UINT(infoBackupFormat(infoBackup), REPOSITORY_FORMAT_6, "backup info at requested format"); + + // The stanza on repo1 already existed, so the format requested for it was ignored. A stanza is migrated with + // stanza-upgrade rather than with this command, which sets the format only for a stanza it creates. + TEST_RESULT_UINT(infoArchiveFormat(infoArchive) != REPOSITORY_FORMAT_DEFAULT, true, "requested format is not the default"); + TEST_ASSIGN( + infoArchive, infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone()), + "load archive info from repo1"); + TEST_RESULT_UINT( + infoArchiveFormat(infoArchive), REPOSITORY_FORMAT_DEFAULT, "archive info on existing stanza still at default format"); // Confirm other repo encrypted with different password TEST_ASSIGN( @@ -990,6 +1005,214 @@ testRun(void) storageGetP(storageNewReadP(storageRepoIdx(0), INFO_BACKUP_PATH_FILE_STR)), storageGetP(storageNewReadP(storageHrn, STRDEF("test.info")))), true, "test and stanza backup info files are equal"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("stanza-upgrade - repository format"); + + // The format is left alone when repo-format is not specified, since the option default would otherwise downgrade a + // repository that has already been upgraded + argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format not requested"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 INFO: stanza 'db' on repo1 is already up to date"); + + TEST_RESULT_UINT( + infoArchiveFormat(infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone())), + REPOSITORY_FORMAT_DEFAULT, "archive info still at default format"); + + // A format in a configuration file is warned about and ignored, which is why the option is command line only. A format + // left in the configuration would otherwise migrate a stanza as a side effect of an upgrade run for another reason. + HRN_STORAGE_PUT_Z( + storageTest, "repo-format.conf", + "[global]\n" + "repo1-format=6\n"); + + argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptConfig, TEST_PATH "/repo-format.conf"); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_LOG("P00 WARN: configuration file contains command-line only option 'repo1-format'"); + + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format in configuration file"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 INFO: stanza 'db' on repo1 is already up to date"); + + TEST_RESULT_UINT( + infoArchiveFormat(infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone())), + REPOSITORY_FORMAT_DEFAULT, "archive info still at default format"); + + // Upgrade the format + argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + hrnCfgArgRawZ(argList, cfgOptRepoFormat, "6"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format 6"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 INFO: upgrade repository format from 5 to 6"); + + TEST_RESULT_UINT( + infoArchiveFormat(infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone())), + REPOSITORY_FORMAT_6, "archive info at format 6"); + TEST_RESULT_UINT( + infoBackupFormat(infoBackupLoadFile(storageRepoIdx(0), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewNone())), + REPOSITORY_FORMAT_6, "backup info at format 6"); + + // Requesting the format the repository is already at does nothing + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format 6 again"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 INFO: stanza 'db' on repo1 is already up to date"); + + // The save of archive.info and backup.info is not atomic, so an upgrade interrupted between the two leaves the info files + // at different formats. Running the upgrade again brings the lagging file forward. + HRN_INFO_PUT( + storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, + "[db]\n" + "db-catalog-version=202211111\n" + "db-control-version=1300\n" + "db-id=2\n" + "db-system-id=" HRN_PG_SYSTEMID_15_Z "\n" + "db-version=\"15\"\n" + "\n" + "[db:history]\n" + "1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665999" + ",\"db-version\":\"9.6\"}\n" + "2={\"db-catalog-version\":202211111,\"db-control-version\":1300,\"db-system-id\":" HRN_PG_SYSTEMID_15_Z + ",\"db-version\":\"15\"}\n", + .comment = "put backup info at prior format to simulate an interrupted upgrade"); + + TEST_RESULT_UINT( + infoBackupFormat(infoBackupLoadFile(storageRepoIdx(0), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewNone())), + REPOSITORY_FORMAT_DEFAULT, "backup info back at default format"); + + // An upgrade that does not request a format brings the lagging file forward since the higher of the two formats is the + // only target that does not downgrade a file + argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format mismatch between info files"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 WARN: repository format mismatch from an interrupted stanza-upgrade will be repaired\n" + "P00 INFO: upgrade repository format from 5 to 6"); + + TEST_RESULT_UINT( + infoBackupFormat(infoBackupLoadFile(storageRepoIdx(0), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewNone())), + REPOSITORY_FORMAT_6, "backup info brought forward to format 6"); + + // The lagging file is normally backup.info since archive.info is saved first, but the format the upgrade reports is the + // lower of the two whichever file it is + InfoArchive *const infoArchivePrior = infoArchiveLoadFile( + storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone()); + infoArchiveFormatSet(infoArchivePrior, REPOSITORY_FORMAT_5); + infoArchiveSaveFile(infoArchivePrior, storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone()); + + // A downgrade is measured against the file that is ahead, so requesting the format the archive info is at does not put + // the backup info back to it + argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + hrnCfgArgRawZ(argList, cfgOptRepoFormat, "5"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_ERROR( + cmdStanzaUpgrade(), FormatError, + "unable to downgrade repository format from 6 to 5\n" + "HINT: backups and archives already written at format 6 would not be readable by a version that only supports" + " format 5."); + TEST_RESULT_LOG("P00 INFO: stanza-upgrade for stanza 'db' on repo1"); + + argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + hrnCfgArgRawZ(argList, cfgOptRepoFormat, "6"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - archive info behind backup info"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 WARN: repository format mismatch from an interrupted stanza-upgrade will be repaired\n" + "P00 INFO: upgrade repository format from 5 to 6"); + + TEST_RESULT_UINT( + infoBackupFormat(infoBackupLoadFile(storageRepoIdx(0), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewNone())), + REPOSITORY_FORMAT_6, "backup info brought forward to format 6"); + TEST_RESULT_UINT( + infoArchiveFormat(infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone())), + REPOSITORY_FORMAT_6, "archive info still at format 6"); + + // The format cannot be downgraded + argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + hrnCfgArgRawZ(argList, cfgOptRepoFormat, "5"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_ERROR( + cmdStanzaUpgrade(), FormatError, + "unable to downgrade repository format from 6 to 5\n" + "HINT: backups and archives already written at format 6 would not be readable by a version that only supports" + " format 5."); + TEST_RESULT_LOG("P00 INFO: stanza-upgrade for stanza 'db' on repo1"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("stanza-upgrade - repository format from the environment"); + + // Put the repository back to the default format so it can be migrated again. This is done directly since the command + // refuses to downgrade. + InfoArchive *const infoArchiveReset = infoArchiveLoadFile( + storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone()); + infoArchiveFormatSet(infoArchiveReset, REPOSITORY_FORMAT_DEFAULT); + infoArchiveSaveFile(infoArchiveReset, storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewNone()); + + InfoBackup *const infoBackupReset = infoBackupLoadFile(storageRepoIdx(0), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewNone()); + infoBackupFormatSet(infoBackupReset, REPOSITORY_FORMAT_DEFAULT); + infoBackupSaveFile(infoBackupReset, storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewNone()); + + // The option cannot be set in a configuration file but it can be set in the environment, where it migrates a stanza + // without appearing in the command that ran. That is how the environment works for every option so it is allowed, but the + // migration must still happen. + argList = strLstDup(argListBase); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + hrnCfgEnvKeyRawZ(cfgOptRepoFormat, 1, "6"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format from the environment"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 INFO: upgrade repository format from 5 to 6"); + + hrnCfgEnvKeyRemoveRaw(cfgOptRepoFormat, 1); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("stanza-upgrade - every format that can be read can be requested"); + + // The allow list for repo-format in build/config.yaml and REPOSITORY_FORMAT_MIN/MAX in version.h are declared separately, + // so make sure they say the same thing + for (unsigned int format = REPOSITORY_FORMAT_MIN; format <= REPOSITORY_FORMAT_MAX; format++) + { + argList = strLstDup(argListBase); + hrnCfgArgKeyRawFmt(argList, cfgOptRepoFormat, 1, "%u", format); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_UINT(cfgOptionIdxUInt(cfgOptRepoFormat, 0), format, "format allowed"); + } + + const unsigned int formatUnsupported = REPOSITORY_FORMAT_MAX + 1; + + argList = strLstDup(argListBase); + hrnCfgArgKeyRawFmt(argList, cfgOptRepoFormat, 1, "%u", formatUnsupported); + + TEST_ERROR_FMT( + hrnCfgLoadP(cfgCmdStanzaUpgrade, argList), OptionInvalidValueError, + "'%u' is not allowed for 'repo1-format' option\n" + "HINT: allowed values are '5', '6'", formatUnsupported); } // ***************************************************************************************************************************** diff --git a/test/src/module/command/verifyTest.c b/test/src/module/command/verifyTest.c index 51d6f42571..92740965aa 100644 --- a/test/src/module/command/verifyTest.c +++ b/test/src/module/command/verifyTest.c @@ -2381,8 +2381,10 @@ testRun(void) "P00 DETAIL: path '11-2/0000000500000007' does not contain any valid WAL to be processed\n" "P00 DETAIL: path '11-2/0000000500000008' does not contain any valid WAL to be processed\n" "P00 DETAIL: path '11-2/0000000500000009' does not contain any valid WAL to be processed\n" - "P00 DETAIL: expected format 5 but found 1234\n" - "P00 DETAIL: expected format 5 but found 1234"); + "P00 DETAIL: repository format 1234 requires a newer version of pgBackRest\n" + " HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6.\n" + "P00 DETAIL: repository format 1234 requires a newer version of pgBackRest\n" + " HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); } if (testBegin("cmdBackup() and verifyProcess()")) diff --git a/test/src/module/config/parseTest.c b/test/src/module/config/parseTest.c index cbe7643d51..13497d0df7 100644 --- a/test/src/module/config/parseTest.c +++ b/test/src/module/config/parseTest.c @@ -1952,6 +1952,8 @@ testRun(void) TEST_RESULT_UINT(cfgOptionGroupIdxToKey(cfgOptGrpPg, 1), 2, "pg2 is index 2"); TEST_RESULT_Z(cfgOptionIdxName(cfgOptPgPath, 0), "pg1-path", "pg1-path option name"); TEST_RESULT_Z(cfgOptionIdxName(cfgOptPgPath, 1), "pg2-path", "pg2-path option name"); + TEST_RESULT_Z( + cfgOptionIdxName(cfgOptRepoFormat, 1), "repo2-format", "repo2-format option name when not valid for command"); TEST_RESULT_Z(cfgOptionGroupName(cfgOptGrpPg, 1), "pg2", "pg2 group display"); TEST_RESULT_Z(cfgOptionGroupName(cfgOptGrpPg, 0), "pg1", "pg1 group display (cached)"); TEST_RESULT_STR_Z(cfgOptionStr(cfgOptPgPath), "/path/to/db", "default pg-path"); diff --git a/test/src/module/info/infoArchiveTest.c b/test/src/module/info/infoArchiveTest.c index c6e0c37bec..5a33e9a516 100644 --- a/test/src/module/info/infoArchiveTest.c +++ b/test/src/module/info/infoArchiveTest.c @@ -61,11 +61,18 @@ testRun(void) // Create the same content by creating a new object TEST_ASSIGN( - info, infoArchiveNew(PG_VERSION_96, 6569239123849665679, NULL), "infoArchiveNew() - no sub cipher"); + info, infoArchiveNew( + PG_VERSION_96, 6569239123849665679, REPOSITORY_FORMAT_DEFAULT, NULL), "infoArchiveNew() - no sub cipher"); TEST_RESULT_STR_Z(infoArchiveId(info), "9.6-1", "archiveId set"); TEST_RESULT_PTR(infoArchivePg(info), info->pub.infoPg, "infoPg set"); TEST_RESULT_UINT(cipherSpecType(infoArchiveCipherSpec(info)), cipherTypeNone, "no cipher sub"); TEST_RESULT_INT(infoPgDataTotal(infoArchivePg(info)), 1, "history set"); + TEST_RESULT_UINT(infoArchiveFormat(info), REPOSITORY_FORMAT_DEFAULT, "format set"); + + // Set the format and put it back so the save below is not affected + TEST_RESULT_VOID(infoArchiveFormatSet(info, REPOSITORY_FORMAT_6), "set format"); + TEST_RESULT_UINT(infoArchiveFormat(info), REPOSITORY_FORMAT_6, "format updated"); + TEST_RESULT_VOID(infoArchiveFormatSet(info, REPOSITORY_FORMAT_DEFAULT), "reset format"); Buffer *contentCompare = bufNew(0); @@ -79,7 +86,7 @@ testRun(void) TEST_ASSIGN( info, infoArchiveNew( - PG_VERSION_10, 6569239123849665999, + PG_VERSION_10, 6569239123849665999, REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), "infoArchiveNew() - cipher sub"); @@ -158,7 +165,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("save and load archive info file"); - InfoArchive *infoArchive = infoArchiveNew(PG_VERSION_10, 6569239123849665999, NULL); + InfoArchive *infoArchive = infoArchiveNew(PG_VERSION_10, 6569239123849665999, REPOSITORY_FORMAT_DEFAULT, NULL); TEST_RESULT_VOID( infoArchiveSaveFile(infoArchive, storageTest, STRDEF(INFO_ARCHIVE_FILE), cipherSpecNewNone()), "save archive info"); diff --git a/test/src/module/info/infoBackupTest.c b/test/src/module/info/infoBackupTest.c index 12de0bba6c..ab1e8a5cee 100644 --- a/test/src/module/info/infoBackupTest.c +++ b/test/src/module/info/infoBackupTest.c @@ -63,7 +63,8 @@ testRun(void) Buffer *contentCompare = bufNew(0); TEST_ASSIGN( - infoBackup, infoBackupNew(PG_VERSION_96, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_96), NULL), + infoBackup, infoBackupNew( + PG_VERSION_96, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, NULL), "infoBackupNew() - no cipher sub"); TEST_RESULT_VOID(infoBackupSave(infoBackup, ioBufferWriteNew(contentCompare)), "save backup info from new"); TEST_RESULT_STR(strNewBuf(contentCompare), strNewBuf(contentSave), "check save"); @@ -72,6 +73,13 @@ testRun(void) TEST_RESULT_PTR(infoBackupPg(infoBackup), infoBackup->pub.infoPg, "infoPg set"); TEST_RESULT_UINT(cipherSpecType(infoBackupCipherSpec(infoBackup)), cipherTypeNone, "cipher sub not set"); TEST_RESULT_INT(infoBackupDataTotal(infoBackup), 0, "infoBackupDataTotal returns 0"); + TEST_RESULT_UINT(infoBackupFormat(infoBackup), REPOSITORY_FORMAT_DEFAULT, "format set"); + + // Setting the format marks the info as updated so it will be saved + TEST_RESULT_BOOL(infoBackup->pub.updated, false, "not updated after load"); + TEST_RESULT_VOID(infoBackupFormatSet(infoBackup, REPOSITORY_FORMAT_6), "set format"); + TEST_RESULT_UINT(infoBackupFormat(infoBackup), REPOSITORY_FORMAT_6, "format updated"); + TEST_RESULT_BOOL(infoBackup->pub.updated, true, "updated after format set"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("check cipher pass"); @@ -79,7 +87,7 @@ testRun(void) TEST_ASSIGN( infoBackup, infoBackupNew( - PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), + PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), "infoBackupNew() - cipher sub"); @@ -330,7 +338,7 @@ testRun(void) TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 1, "backup added to current"); TEST_ASSIGN(backupData, infoBackupData(infoBackup, 0), "get added backup"); TEST_RESULT_STR_Z(backupData.backupLabel, "20190818-084502F", "backup label set"); - TEST_RESULT_UINT(backupData.backrestFormat, REPOSITORY_FORMAT, "backrest format"); + TEST_RESULT_UINT(backupData.backrestFormat, REPOSITORY_FORMAT_DEFAULT, "backrest format"); TEST_RESULT_STR_Z(backupData.backrestVersion, PROJECT_VERSION, "backuprest version"); TEST_RESULT_INT(backupData.backupPgId, 1, "pg id"); TEST_RESULT_STR(backupData.backupArchiveStart, NULL, "archive start NULL"); @@ -440,7 +448,7 @@ testRun(void) TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, "backup added to current"); TEST_ASSIGN(backupData, infoBackupData(infoBackup, 1), "get added backup"); TEST_RESULT_STR_Z(backupData.backupLabel, "20190818-084502F_20190820-084502I", "backup label set"); - TEST_RESULT_UINT(backupData.backrestFormat, REPOSITORY_FORMAT, "backrest format"); + TEST_RESULT_UINT(backupData.backrestFormat, REPOSITORY_FORMAT_DEFAULT, "backrest format"); TEST_RESULT_STR_Z(backupData.backrestVersion, PROJECT_VERSION, "backuprest version"); TEST_RESULT_STR_Z(backupData.backupArchiveStart, "000000030000028500000089", "archive start set"); TEST_RESULT_STR_Z(backupData.backupArchiveStop, "000000030000028500000090", "archive stop set"); @@ -899,7 +907,8 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("save and load backup info file"); - InfoBackup *infoBackup = infoBackupNew(PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), NULL); + InfoBackup *infoBackup = infoBackupNew( + PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), REPOSITORY_FORMAT_DEFAULT, NULL); TEST_RESULT_VOID( infoBackupSaveFile( infoBackup, storageTest, STRDEF(INFO_BACKUP_FILE), cipherSpecNewNone()), "save backup info"); diff --git a/test/src/module/info/infoPgTest.c b/test/src/module/info/infoPgTest.c index ed6a0cb7fb..13a04301b9 100644 --- a/test/src/module/info/infoPgTest.c +++ b/test/src/module/info/infoPgTest.c @@ -36,13 +36,14 @@ testRun(void) { InfoPg *infoPg = NULL; - TEST_ASSIGN(infoPg, infoPgNew(infoPgBackup, NULL), "infoPgNew(cipherTypeNone, NULL)"); + TEST_ASSIGN(infoPg, infoPgNew(infoPgBackup, REPOSITORY_FORMAT_DEFAULT, NULL), "infoPgNew(cipherTypeNone, NULL)"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 0, " 0 history"); TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(infoPgInfo(infoPg))), cipherTypeNone, " cipher spec none"); TEST_RESULT_INT(infoPgDataCurrentId(infoPg), 0, " 0 historyCurrent"); TEST_ASSIGN( - infoPg, infoPgNew(infoPgArchive, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + infoPg, + infoPgNew(infoPgArchive, REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), "infoPgNew(cipherTypeAes256Cbc, 123xyz)"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 0, " 0 history"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(infoPgInfo(infoPg)))), "123xyz", " cipherPass set"); @@ -52,7 +53,8 @@ testRun(void) TEST_ASSIGN( infoPg, infoPgSet( - infoPgNew(infoPgArchive, NULL), PG_VERSION_18, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_18)), + infoPgNew(infoPgArchive, REPOSITORY_FORMAT_DEFAULT, NULL), PG_VERSION_18, 6569239123849665679, + hrnPgCatalogVersion(PG_VERSION_18)), "infoPgSet - infoPgArchive"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 1, " 1 history"); TEST_RESULT_INT(infoPgDataCurrentId(infoPg), 0, " 0 historyCurrent"); @@ -78,8 +80,8 @@ testRun(void) TEST_ASSIGN( infoPg, infoPgSet( - infoPgNew(infoPgBackup, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), PG_VERSION_18, - 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_18)), + infoPgNew(infoPgBackup, REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + PG_VERSION_18, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_18)), "infoPgSet - infoPgBackup"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 1, " 1 history"); TEST_RESULT_INT(infoPgDataCurrentId(infoPg), 0, " 0 historyCurrent"); diff --git a/test/src/module/info/infoTest.c b/test/src/module/info/infoTest.c index 82aeea4bc7..2bd306e5d3 100644 --- a/test/src/module/info/infoTest.c +++ b/test/src/module/info/infoTest.c @@ -78,10 +78,12 @@ testRun(void) { Info *info = NULL; - TEST_ASSIGN(info, infoNew(cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), "infoNew(cipher)"); + TEST_ASSIGN( + info, infoNew(REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + "infoNew(cipher)"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(info))), "123xyz", " cipherPass is set"); - TEST_ASSIGN(info, infoNew(NULL), "infoNew(NULL)"); + TEST_ASSIGN(info, infoNew(REPOSITORY_FORMAT_DEFAULT, NULL), "infoNew(NULL)"); TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(info)), cipherTypeNone, " cipher spec is none"); } @@ -98,9 +100,23 @@ testRun(void) TEST_ERROR( infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), - FormatError, "expected format 5 but found 4"); + FormatError, + "repository format 4 is no longer supported by pgBackRest\n" + "HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); + // Format newer than supported + // ------------------------------------------------------------------------------------------------------------------------- + contentLoad = BUFSTRDEF( + "[backrest]\n" + "backrest-format=7\n"); + + TEST_ERROR( + infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + FormatError, + "repository format 7 requires a newer version of pgBackRest\n" + "HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); + // Checksum not found // ------------------------------------------------------------------------------------------------------------------------- contentLoad = BUFSTRDEF( @@ -126,6 +142,20 @@ testRun(void) ChecksumError, "invalid checksum, actual 'fe989a75dcf7a0261e57d210707c0db741462763' but expected 'BOGUS'"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); + // Format not found. The checksum must be valid since it is verified before the format is checked. + // ------------------------------------------------------------------------------------------------------------------------- + contentLoad = BUFSTRDEF( + "[backrest]\n" + "backrest-checksum=\"be4f04bf9a8346d387bb254c48aeee2cbb5d46d0\"\n" + "backrest-version=\"2.17\"\n"); + + TEST_ERROR( + infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + FormatError, + "repository format not found\n" + "HINT: is this a valid pgBackRest info file?"); + TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); + // Crypto expected // ------------------------------------------------------------------------------------------------------------------------- contentLoad = BUFSTRDEF( @@ -181,6 +211,35 @@ testRun(void) TEST_RESULT_VOID(infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "info save"); TEST_RESULT_STR(strNewBuf(contentSave), strNewBuf(contentLoad), " check save"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_DEFAULT, " check format"); + + // The format is read from the file rather than assumed, and written back out as read + // ------------------------------------------------------------------------------------------------------------------------- + contentLoad = harnessInfoChecksumFormat( + REPOSITORY_FORMAT_6, + STRDEF( + "[c]\n" + "key=1\n" + "\n" + "[d]\n" + "key=1\n")); + + callbackContent = strNew(); + + TEST_ASSIGN( + info, infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + "info format 6"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); + + contentSave = bufNew(0); + + TEST_RESULT_VOID(infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "info save"); + TEST_RESULT_STR(strNewBuf(contentSave), strNewBuf(contentLoad), " check save preserves format"); + + // Set the format + // ------------------------------------------------------------------------------------------------------------------------- + TEST_RESULT_VOID(infoFormatSet(info, REPOSITORY_FORMAT_5), "set format"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_5, " check format"); // File with content and cipher // ------------------------------------------------------------------------------------------------------------------------- diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index 277ab17ba0..1d97cb2382 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -289,8 +289,8 @@ testRun(void) // Test tablespace error TEST_ERROR( manifestNewBuild( - storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), 1565282120, false, false, false, false, NULL, - exclusionList, pckWriteResult(tablespaceList)), + storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, 1565282120, false, false, + false, false, NULL, exclusionList, pckWriteResult(tablespaceList)), AssertError, "tablespace with oid 1 not found in tablespace map\n" "HINT: was a tablespace created or dropped during the backup?"); @@ -314,8 +314,8 @@ testRun(void) TEST_ASSIGN( manifest, manifestNewBuild( - storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), 1565282120, false, false, false, false, NULL, NULL, - pckWriteResult(tablespaceList)), + storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, 1565282120, false, false, + false, false, NULL, NULL, pckWriteResult(tablespaceList)), "build manifest"); TEST_RESULT_VOID(manifestBackupLabelSet(manifest, STRDEF("20190818-084502F")), "backup label set"); @@ -411,8 +411,8 @@ testRun(void) TEST_ASSIGN( manifest, manifestNewBuild( - storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), 1565282120, true, false, false, false, NULL, NULL, - NULL), + storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, 1565282120, true, false, + false, false, NULL, NULL, NULL), "build manifest"); contentSave = bufNew(0); @@ -485,7 +485,8 @@ testRun(void) TEST_ERROR( manifestNewBuild( - storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), 1, false, false, false, false, NULL, NULL, NULL), + storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, 1, false, false, false, + false, NULL, NULL, NULL), LinkDestinationError, "link 'pg_xlog/wal' (" TEST_PATH "/wal) destination is the same directory as link 'pg_xlog' (" TEST_PATH "/wal)"); @@ -541,8 +542,8 @@ testRun(void) TEST_ASSIGN( manifest, manifestNewBuild( - storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), 1565282120, false, true, false, false, NULL, NULL, - NULL), + storagePg, PG_VERSION_96, hrnPgCatalogVersion(PG_VERSION_96), REPOSITORY_FORMAT_DEFAULT, 1565282120, false, true, + false, false, NULL, NULL, NULL), "build manifest"); contentSave = bufNew(0); @@ -636,8 +637,8 @@ testRun(void) // Tablespace link errors when correct version not found TEST_ERROR( manifestNewBuild( - storagePg, PG_VERSION_12, hrnPgCatalogVersion(PG_VERSION_12), 1565282120, false, false, false, false, NULL, NULL, - NULL), + storagePg, PG_VERSION_12, hrnPgCatalogVersion(PG_VERSION_12), REPOSITORY_FORMAT_DEFAULT, 1565282120, false, false, + false, false, NULL, NULL, NULL), FileOpenError, "unable to get info for missing path/file '" TEST_PATH "/pg/pg_tblspc/1/PG_12_201909212'"); // Remove the link inside pg/pg_tblspc @@ -656,8 +657,8 @@ testRun(void) TEST_ASSIGN( manifest, manifestNewBuild( - storagePg, PG_VERSION_12, hrnPgCatalogVersion(PG_VERSION_12), 1565282120, true, false, true, false, NULL, NULL, - NULL), + storagePg, PG_VERSION_12, hrnPgCatalogVersion(PG_VERSION_12), REPOSITORY_FORMAT_DEFAULT, 1565282120, true, false, + true, false, NULL, NULL, NULL), "build manifest"); contentSave = bufNew(0); @@ -780,8 +781,8 @@ testRun(void) TEST_ASSIGN( manifest, manifestNewBuild( - storagePg, PG_VERSION_13, hrnPgCatalogVersion(PG_VERSION_13), 1570000000, false, false, true, true, - &manifestBuildBlockIncrMap, NULL, NULL), + storagePg, PG_VERSION_13, hrnPgCatalogVersion(PG_VERSION_13), REPOSITORY_FORMAT_DEFAULT, 1570000000, false, false, + true, true, &manifestBuildBlockIncrMap, NULL, NULL), "build manifest"); contentSave = bufNew(0); @@ -856,7 +857,8 @@ testRun(void) TEST_ERROR( manifestNewBuild( - storagePg, PG_VERSION_14, hrnPgCatalogVersion(PG_VERSION_14), 1, false, false, false, false, NULL, NULL, NULL), + storagePg, PG_VERSION_14, hrnPgCatalogVersion(PG_VERSION_14), REPOSITORY_FORMAT_DEFAULT, 1, false, false, false, + false, NULL, NULL, NULL), LinkDestinationError, "link 'link' destination '" TEST_PATH "/pg/base' is in PGDATA"); THROW_ON_SYS_ERROR(unlink(TEST_PATH "/pg/link") == -1, FileRemoveError, "unable to remove symlink"); @@ -868,7 +870,8 @@ testRun(void) TEST_ERROR( manifestNewBuild( - storagePg, PG_VERSION_15, hrnPgCatalogVersion(PG_VERSION_15), 1, false, false, false, false, NULL, NULL, NULL), + storagePg, PG_VERSION_15, hrnPgCatalogVersion(PG_VERSION_15), REPOSITORY_FORMAT_DEFAULT, 1, false, false, false, + false, NULL, NULL, NULL), LinkExpectedError, "'pg_data/pg_tblspc/somedir' is not a symlink - pg_tblspc should contain only symlinks"); HRN_STORAGE_PATH_REMOVE(storagePgWrite, MANIFEST_TARGET_PGTBLSPC "/somedir"); @@ -880,7 +883,8 @@ testRun(void) TEST_ERROR( manifestNewBuild( - storagePg, PG_VERSION_16, hrnPgCatalogVersion(PG_VERSION_16), 1, false, false, false, false, NULL, NULL, NULL), + storagePg, PG_VERSION_16, hrnPgCatalogVersion(PG_VERSION_16), REPOSITORY_FORMAT_DEFAULT, 1, false, false, false, + false, NULL, NULL, NULL), LinkExpectedError, "'pg_data/pg_tblspc/somefile' is not a symlink - pg_tblspc should contain only symlinks"); TEST_STORAGE_EXISTS(storagePgWrite, MANIFEST_TARGET_PGTBLSPC "/somefile", .remove = true); @@ -892,7 +896,8 @@ testRun(void) TEST_ERROR( manifestNewBuild( - storagePg, PG_VERSION_17, hrnPgCatalogVersion(PG_VERSION_17), 1, false, true, false, false, NULL, NULL, NULL), + storagePg, PG_VERSION_17, hrnPgCatalogVersion(PG_VERSION_17), REPOSITORY_FORMAT_DEFAULT, 1, false, true, false, + false, NULL, NULL, NULL), FileOpenError, "unable to get info for missing path/file '" TEST_PATH "/pg/link-to-link'"); THROW_ON_SYS_ERROR(unlink(TEST_PATH "/pg/link-to-link") == -1, FileRemoveError, "unable to remove symlink"); @@ -908,7 +913,8 @@ testRun(void) TEST_ERROR( manifestNewBuild( - storagePg, PG_VERSION_18, hrnPgCatalogVersion(PG_VERSION_18), 1, false, false, false, false, NULL, NULL, NULL), + storagePg, PG_VERSION_18, hrnPgCatalogVersion(PG_VERSION_18), REPOSITORY_FORMAT_DEFAULT, 1, false, false, false, + false, NULL, NULL, NULL), LinkDestinationError, "link '" TEST_PATH "/pg/linktolink' cannot reference another link '" TEST_PATH "/linktest'"); #undef TEST_MANIFEST_HEADER @@ -1026,7 +1032,7 @@ testRun(void) OBJ_NEW_BASE_BEGIN(Manifest, .childQty = MEM_CONTEXT_QTY_MAX) { manifest = manifestNewInternal(); - manifest->pub.info = infoNew(NULL); + manifest->pub.info = infoNew(REPOSITORY_FORMAT_DEFAULT, NULL); manifest->pub.data.backupTimestampStart = 1482182860; manifest->pub.data.pgVersion = PG_VERSION_96; manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_96); @@ -1414,55 +1420,57 @@ testRun(void) { Manifest *manifest = NULL; - // Manifest with minimal features - const Buffer *contentLoad = harnessInfoChecksumZ( - "[backup]\n" - "backup-bundle=true\n" - "backup-bundle-raw=true\n" - "backup-label=\"20190808-163540F\"\n" - "backup-reference=\"20190808-163540F\"\n" - "backup-timestamp-copy-start=1565282141\n" - "backup-timestamp-start=1565282140\n" - "backup-timestamp-stop=1565282142\n" - "backup-type=\"full\"\n" - "\n" - "[backup:db]\n" - "db-catalog-version=201608131\n" - "db-control-version=960\n" - "db-id=1\n" - "db-system-id=1000000000000000094\n" - "db-version=\"9.6\"\n" - "\n" - "[backup:option]\n" - "option-archive-check=true\n" - "option-archive-copy=true\n" - "option-compress=false\n" - "option-compress-type=\"none\"\n" - "option-hardlink=false\n" - "option-online=false\n" - "\n" - "[backup:target]\n" - "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" - "\n" - "[cipher]\n" - "cipher-pass=\"somepass\"\n" - "\n" - "[target:file]\n" - "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"reference\":\"20190808-163540F\"" - ",\"size\":4,\"timestamp\":1565282114}\n" - "\n" - "[target:file:default]\n" - "group=\"group1\"\n" - "mode=\"0600\"\n" - "user=\"user1\"\n" - "\n" - "[target:path]\n" - "pg_data={}\n" - "\n" - "[target:path:default]\n" - "group=\"group1\"\n" - "mode=\"0700\"\n" - "user=\"user1\"\n"); + // Manifest with minimal features, written at a format other than the default so the format is read from the file + const Buffer *contentLoad = harnessInfoChecksumFormat( + REPOSITORY_FORMAT_6, + STRDEF( + "[backup]\n" + "backup-bundle=true\n" + "backup-bundle-raw=true\n" + "backup-label=\"20190808-163540F\"\n" + "backup-reference=\"20190808-163540F\"\n" + "backup-timestamp-copy-start=1565282141\n" + "backup-timestamp-start=1565282140\n" + "backup-timestamp-stop=1565282142\n" + "backup-type=\"full\"\n" + "\n" + "[backup:db]\n" + "db-catalog-version=201608131\n" + "db-control-version=960\n" + "db-id=1\n" + "db-system-id=1000000000000000094\n" + "db-version=\"9.6\"\n" + "\n" + "[backup:option]\n" + "option-archive-check=true\n" + "option-archive-copy=true\n" + "option-compress=false\n" + "option-compress-type=\"none\"\n" + "option-hardlink=false\n" + "option-online=false\n" + "\n" + "[backup:target]\n" + "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" + "\n" + "[cipher]\n" + "cipher-pass=\"somepass\"\n" + "\n" + "[target:file]\n" + "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"reference\":\"20190808-163540F\"" + ",\"size\":4,\"timestamp\":1565282114}\n" + "\n" + "[target:file:default]\n" + "group=\"group1\"\n" + "mode=\"0600\"\n" + "user=\"user1\"\n" + "\n" + "[target:path]\n" + "pg_data={}\n" + "\n" + "[target:path:default]\n" + "group=\"group1\"\n" + "mode=\"0700\"\n" + "user=\"user1\"\n")); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("manifest move"); @@ -1482,6 +1490,7 @@ testRun(void) TEST_ERROR( manifestTargetFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest target list"); TEST_RESULT_STR_Z(manifestData(manifest)->backupLabel, "20190808-163540F", "check manifest data"); + TEST_RESULT_UINT(manifestFormat(manifest), REPOSITORY_FORMAT_6, "check manifest format"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(manifestCipherSpec(manifest))), "somepass", "check cipher subpass"); diff --git a/test/src/module/integration/allTest.c b/test/src/module/integration/allTest.c index e152fc2097..7d1cce5022 100644 --- a/test/src/module/integration/allTest.c +++ b/test/src/module/integration/allTest.c @@ -198,6 +198,10 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("standby full backup and expire"); { + // Update repo 1 to format 6 when there is more than one repo + if (hrnHostRepoTotal() == 2) + TEST_HOST_BR(repo, CFGCMD_STANZA_UPGRADE, .option = "--repo1-format=6"); + // Check backups before the backup so we know how many will exist after const InfoBackup *infoBackup = infoBackupLoadFile( hrnHostRepo1Storage(repo), STRDEF("backup/" HRN_STANZA "/backup.info"), hrnHostCipherSpec()); diff --git a/test/src/module/performance/typeTest.c b/test/src/module/performance/typeTest.c index b558a7f6a1..c379fbb76d 100644 --- a/test/src/module/performance/typeTest.c +++ b/test/src/module/performance/typeTest.c @@ -279,7 +279,9 @@ testRun(void) MEM_CONTEXT_BEGIN(testContext) { TEST_ASSIGN( - manifest, manifestNewBuild(storagePg, PG_VERSION_15, 999999999, 0, false, false, false, false, NULL, NULL, NULL), + manifest, manifestNewBuild( + storagePg, PG_VERSION_15, 999999999, REPOSITORY_FORMAT_DEFAULT, 0, false, false, false, false, NULL, NULL, + NULL), "build files"); } MEM_CONTEXT_END(); From 5f76e6863f4b726f783099ea6cbd133b13014f73 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 09:45:11 +0700 Subject: [PATCH 02/15] Derive the cipher key with SHA-256 at repository format 6. Repositories written before format 6 derive the key from the passphrase with SHA-1 and continue to. Format 6 derives with SHA-256. The digest is chosen along with the pass rather than at each use, so it now travels with the pass in CipherSpec and defaults to SHA-256, and the places that must keep deriving with SHA-1 say so explicitly. A reader cannot learn which digest a file needs from the file itself, since the format is stored in the content the pass decrypts. Encrypted info files at format 6 therefore begin with a plaintext header, PGBR followed by the three-digit format and a byte held back for whatever the header turns out to need, in place of the OpenSSL Salted__ magic. A file that begins with the magic was written at format 5, the only format there was before the header. The header format is validated before anything is decrypted, since this version cannot know what a newer format expects, and is compared against the format read from the content afterward so a file assembled from parts of two files is rejected rather than half read. --- doc/xml/reference.xml | 2 +- src/command/backup/backup.c | 2 +- src/command/check/common.c | 2 +- src/command/repo/get.c | 15 +- src/command/stanza/common.c | 10 +- src/command/stanza/common.h | 3 +- src/command/stanza/create.c | 5 +- src/command/verify/verify.c | 14 +- src/common/crypto/cipherBlock.c | 247 +++++++++++++++++++++++-- src/common/crypto/cipherBlock.h | 9 + src/common/crypto/spec.c | 28 ++- src/common/crypto/spec.h | 42 ++++- src/config/config.c | 2 +- src/info/info.c | 143 +++++++++++--- src/info/info.h | 35 +++- src/info/infoArchive.c | 7 +- src/info/infoBackup.c | 7 +- src/info/infoPg.c | 3 +- src/info/manifest/manifest.c | 2 +- src/info/manifest/serialize.c.inc | 4 +- test/src/harness/config.h | 2 +- test/src/harness/host.c | 2 +- test/src/harness/info.c | 80 ++++++++ test/src/harness/info.h | 19 +- test/src/harness/storage.c | 19 +- test/src/module/command/backupTest.c | 2 +- test/src/module/command/restoreTest.c | 12 +- test/src/module/command/stanzaTest.c | 8 +- test/src/module/command/verifyTest.c | 43 +++-- test/src/module/common/cryptoTest.c | 198 ++++++++++++++++++-- test/src/module/info/infoArchiveTest.c | 12 +- test/src/module/info/infoBackupTest.c | 14 +- test/src/module/info/infoPgTest.c | 4 +- test/src/module/info/infoTest.c | 147 +++++++++++++-- test/src/module/info/manifestTest.c | 7 +- test/src/module/storage/remoteTest.c | 4 +- 36 files changed, 998 insertions(+), 157 deletions(-) diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index e6b8228f40..4bfcc3521c 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -2929,7 +2929,7 @@ Format supported by all versions since 1.00 - Adds no features yet and is reserved for features in development + Derives repository encryption keys with SHA-256 rather than SHA-1

An existing stanza is migrated to a newer format with the stanza-upgrade command rather than with this command, which sets the format only for a stanza it creates.

diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c index a4e564be37..42b2f9f144 100644 --- a/src/command/backup/backup.c +++ b/src/command/backup/backup.c @@ -241,7 +241,7 @@ cmdBackup(void) // Build an incremental backup if type is not full (manifestPrior will be freed in this call) if (!backupBuildIncr(manifest, manifestPrior, backupStartResult.walSegmentName)) - manifestCipherSpecSet(manifest, cipherSpecGen(cfgOptionStrId(cfgOptRepoCipherType))); + manifestCipherSpecSet(manifest, cipherSpecGen(cfgOptionStrId(cfgOptRepoCipherType), manifestFormat(manifest))); // Set delta if it is not already set and the manifest requires it if (!cfgOptionBool(cfgOptDelta) && varBool(manifestData(manifest)->backupOptionDelta)) diff --git a/src/command/check/common.c b/src/command/check/common.c index af6f86857d..2541291875 100644 --- a/src/command/check/common.c +++ b/src/command/check/common.c @@ -123,7 +123,7 @@ checkStanzaInfo(const unsigned int repoIdx, const InfoPg *const archiveInfoPg, c } // Error if the info files are at different repository formats. The formats are written together but stored apart, so an - // upgrade interrupted between the two saves leaves them here. + // upgrade interrupted between the two saves leaves them mismatched. if (infoPgFormat(archiveInfoPg) != infoPgFormat(backupInfoPg)) { const unsigned int formatArchive = infoPgFormat(archiveInfoPg); diff --git a/src/command/repo/get.c b/src/command/repo/get.c index 81a42e28a6..84a9a39f73 100644 --- a/src/command/repo/get.c +++ b/src/command/repo/get.c @@ -16,6 +16,7 @@ Repository Get Command #include "config/config.h" #include "storage/helper.h" +#include "info/info.h" #include "info/infoArchive.h" #include "info/infoBackup.h" @@ -47,6 +48,9 @@ storageGetProcess(IoWrite *const destination) IoRead *const source = storageReadIo( storageNewReadP(storageRepo(), file, .ignoreMissing = cfgOptionBool(cfgOptIgnoreMissing))); + // Is the file an info file, i.e. one that carries a header in front of its content? + bool fileIsInfo = false; + // Add decryption if needed if (!cfgOptionBool(cfgOptRaw)) { @@ -96,6 +100,9 @@ storageGetProcess(IoWrite *const destination) cfgCipherSpecMain()); cipherSpec = infoArchiveCipherSpec(info); } + // Else the file is the archive info, which the repo passphrase opens + else + fileIsInfo = true; } // Backup path @@ -128,6 +135,9 @@ storageGetProcess(IoWrite *const destination) else cipherSpec = cipherSpecManifest; } + // Else the file is the backup info, which the repo passphrase opens + else + fileIsInfo = true; } } @@ -137,8 +147,9 @@ storageGetProcess(IoWrite *const destination) ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); - // Add encryption filter - cipherBlockFilterGroupAdd(ioReadFilterGroup(source), cipherModeDecrypt, cipherSpec); + // Add the decryption filter. An info file is read with a header, which the cipher consumes. + ioFilterGroupAdd( + ioReadFilterGroup(source), cipherBlockNewP(cipherModeDecrypt, cipherSpec, .header = fileIsInfo)); } } diff --git a/src/command/stanza/common.c b/src/command/stanza/common.c index cdb7feb380..cf802ae03b 100644 --- a/src/command/stanza/common.c +++ b/src/command/stanza/common.c @@ -16,10 +16,11 @@ Stanza Commands Handler /**********************************************************************************************************************************/ FN_EXTERN CipherSpec * -cipherSpecGen(const CipherType cipherType) +cipherSpecGen(const CipherType cipherType, const unsigned int format) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING_ID, cipherType); + FUNCTION_TEST_PARAM(UINT, format); FUNCTION_TEST_END(); CipherSpec *result; @@ -33,8 +34,11 @@ cipherSpecGen(const CipherType cipherType) uint8_t buffer[48]; // 48 is the amount of entropy needed to get a 64 base key cryptoRandomBytes(buffer, sizeof(buffer)); - // The pass is the encoded text rather than the bytes it encodes, so it is stored and derived from as that text - result = cipherSpecNew(cipherType, BUFSTR(strNewEncode(encodingBase64, BUF(buffer, sizeof(buffer))))); + // The pass is the encoded text rather than the bytes it encodes, so it is stored and derived from as that text. The + // digest is the one the file it will be stored in is read with, since that is what will derive it later. + result = cipherSpecNewP( + cipherType, BUFSTR(strNewEncode(encodingBase64, BUF(buffer, sizeof(buffer)))), + .digest = infoFormatDigest(format)); cipherSpecMove(result, memContextPrior()); } MEM_CONTEXT_TEMP_END(); diff --git a/src/command/stanza/common.h b/src/command/stanza/common.h index 225ca5ee34..e763789d6e 100644 --- a/src/command/stanza/common.h +++ b/src/command/stanza/common.h @@ -11,7 +11,8 @@ Stanza Commands Handler Functions ***********************************************************************************************************************************/ // Generate cipher spec with a new pass, none when the type is none -FN_EXTERN CipherSpec *cipherSpecGen(CipherType cipherType); +// Generate a sub pass for a file at this format, which sets the digest since that is what will derive it when it is read back +FN_EXTERN CipherSpec *cipherSpecGen(CipherType cipherType, unsigned int format); // Validate and return database information FN_EXTERN PgControl pgValidate(void); diff --git a/src/command/stanza/create.c b/src/command/stanza/create.c index 856ffe5f6e..61d1585711 100644 --- a/src/command/stanza/create.c +++ b/src/command/stanza/create.c @@ -78,7 +78,7 @@ cmdStanzaCreate(void) const unsigned int format = cfgOptionIdxUInt(cfgOptRepoFormat, repoIdx); // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent archive files - const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx)); + const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), format); // Create and save archive info infoArchive = infoArchiveNew(pgControl.version, pgControl.systemId, format, cipherSpecArchive); @@ -86,7 +86,8 @@ cmdStanzaCreate(void) infoArchiveSaveFile(infoArchive, storageRepoWriteStanza, INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx)); // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent manifests - const CipherSpec *const cipherSpecManifest = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx)); + const CipherSpec *const cipherSpecManifest = cipherSpecGen( + cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), format); // Create and save backup info infoBackup = infoBackupNew( diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c index f30d40fe3e..e158d7b47f 100644 --- a/src/command/verify/verify.c +++ b/src/command/verify/verify.c @@ -167,11 +167,12 @@ verifyInvalidFileAdd(List *const invalidFileList, const VerifyResult reason, con Load a file into memory ***********************************************************************************************************************************/ static StorageRead * -verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherSpec) +verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherSpec, const bool decrypt) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING, pathFileName); // Fully qualified path/file name FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); // Cipher spec to open file if encrypted + FUNCTION_TEST_PARAM(BOOL, decrypt); // Decrypt here rather than leaving it to the load? FUNCTION_TEST_END(); ASSERT(pathFileName != NULL); @@ -182,7 +183,12 @@ verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherS // *read points to a location within result so update result with contents based on necessary filters IoRead *const read = storageReadIo(result); - cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec); + // An info file or manifest is decrypted by the load, which has to read the header before it knows what to decrypt with, so the + // checksum is over the file as it is stored. The file and its copy are written from the same bytes, which is all the checksum + // is used to compare. + if (decrypt) + cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec); + ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1)); // If the file is compressed, add a decompression filter @@ -214,7 +220,7 @@ verifyInfoFile(const String *const pathFileName, const bool keepFile, const Ciph { TRY_BEGIN() { - IoRead *const infoRead = storageReadIo(verifyFileLoad(pathFileName, cipherSpec)); + IoRead *const infoRead = storageReadIo(verifyFileLoad(pathFileName, cipherSpec, false)); // If directed to keep the loaded file in memory, then move the file into the result, else drain the io and close it if (keepFile) @@ -827,7 +833,7 @@ verifyArchive(VerifyJobData *const jobData) strNewFmt( STORAGE_REPO_ARCHIVE "/%s/%s/%s", strZ(archiveResult->archiveId), strZ(walPath), strZ(strLstGet(jobData->walFileList, 0))), - jobData->cipherSpecArchive); + jobData->cipherSpecArchive, true); const PgWal walInfo = pgWalFromBuffer( storageGetP(walRead, .exactSize = PG_WAL_HEADER_SIZE), cfgOptionStrNull(cfgOptPgVersionForce)); diff --git a/src/common/crypto/cipherBlock.c b/src/common/crypto/cipherBlock.c index bd3ff35577..a374323480 100644 --- a/src/common/crypto/cipherBlock.c +++ b/src/common/crypto/cipherBlock.c @@ -3,6 +3,7 @@ Block Cipher ***********************************************************************************************************************************/ #include +#include #include #include @@ -13,19 +14,98 @@ Block Cipher #include "common/debug.h" #include "common/io/filter/filter.h" #include "common/log.h" +#include "common/type/convert.h" #include "common/type/object.h" +#include "version.h" /*********************************************************************************************************************************** -Header constants and sizes +Magic constant for salted encrypt, written before the salt unless the cipher is raw. Only salted encrypt is done here, but this +constant is required for compatibility with the openssl command-line tool. ***********************************************************************************************************************************/ -// Magic constant for salted encrypt. Only salted encrypt is done here, but this constant is required for compatibility with the -// openssl command-line tool. #define CIPHER_BLOCK_MAGIC "Salted__" #define CIPHER_BLOCK_MAGIC_SIZE (sizeof(CIPHER_BLOCK_MAGIC) - 1) +/*********************************************************************************************************************************** +Format header + +A file written with a header begins with fixed-size plaintext naming the repository format it was written with. The header exists +because the digest the pass derives with follows the format, and the format is recorded inside the file that the pass encrypts. A +reader that could not see the format in advance would have to decrypt to learn what it should have decrypted with. + +The header takes the place of the salted magic, which is why a file that carries one is written raw. Both are eight bytes followed +by the salt, so the eight bytes are consumed either way and what follows begins with the salt no matter which was there. It also +means a file of either kind is opened with the openssl command-line tool the same way: replace the first eight bytes with the magic +that tool expects. + +A file that begins with the magic rather than the header was written at format 5, the only format there was before the header, so +that start is not an error when a header was expected. + +Of the four bytes after the header magic, the first three are the format and the last is held back for whatever the header turns +out to need, e.g. naming which key the file was encrypted with once a repository can hold more than one. The format comes first so +that it is always at the same place, which is what lets a version work out whether it can read the file at all. Only once the +format turns out to be one this version knows is the spare byte examined, and then it must be the underscore this version writes. + +The header is not part of the file content. This filter adds it on encrypt and consumes it on decrypt, so nothing on either side +sees anything but the content. +***********************************************************************************************************************************/ +#define CIPHER_BLOCK_HEADER_MAGIC "PGBR" +#define CIPHER_BLOCK_HEADER_MAGIC_SIZE (sizeof(CIPHER_BLOCK_HEADER_MAGIC) - 1) +#define CIPHER_BLOCK_HEADER_RESERVED '_' +#define CIPHER_BLOCK_HEADER_FORMAT_SIZE 3 + // Total length of cipher header #define CIPHER_BLOCK_HEADER_SIZE (CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN) +/*********************************************************************************************************************************** +Digest the pass derives the key with at a format. A format that predates the header derived with SHA-1, which is why a file with no +header is read with it. +***********************************************************************************************************************************/ +static const EVP_MD * +cipherBlockFormatDigest(const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + // Both accessors always return a digest, so unlike a lookup by name this cannot fail + FUNCTION_TEST_RETURN_TYPE_CONST_P(EVP_MD, format >= REPOSITORY_FORMAT_6 ? EVP_sha256() : EVP_sha1()); +} + +/*********************************************************************************************************************************** +Error when the format read from a header cannot be read by this version. This is checked before anything is decrypted since +decrypting requires knowing what the format expects and this version does not know what a newer format expects. +***********************************************************************************************************************************/ +static void +cipherBlockFormatValidate(const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + // A format newer than this version can read requires an upgrade. Do not suggest a version since this version cannot know which + // version added the format. + if (format > REPOSITORY_FORMAT_MAX) + { + THROW_FMT( + FormatError, + "repository format %u requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + // A format older than this version can read requires an older version to migrate the repository + if (format < REPOSITORY_FORMAT_MIN) + { + THROW_FMT( + FormatError, + "repository format %u is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + FUNCTION_TEST_RETURN_VOID(); +} + /*********************************************************************************************************************************** Object type ***********************************************************************************************************************************/ @@ -33,6 +113,8 @@ typedef struct CipherBlock { CipherMode mode; // Mode encrypt/decrypt bool raw; // Omit header magic to save space + bool headerFormat; // Read/write the format header + unsigned int format; // Repository format, zero until a read header gives one bool saltDone; // Has the salt been read/generated? bool processDone; // Has any data been processed? const Buffer *pass; // Passphrase used to generate encryption key @@ -131,8 +213,28 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s // On encrypt the salt is generated if (this->mode == cipherModeEncrypt) { - // Add magic to the destination buffer so openssl knows the file is salted - if (!this->raw) + // Add the header to the destination buffer in place of the magic. Both are the same size so what follows begins with + // the salt either way. + if (this->headerFormat) + { + memcpy(destination, CIPHER_BLOCK_HEADER_MAGIC, CIPHER_BLOCK_HEADER_MAGIC_SIZE); + + // Write the format right-aligned in the digits it gets so it is always at the same place + unsigned int format = this->format; + + for (unsigned int digitIdx = CIPHER_BLOCK_HEADER_FORMAT_SIZE; digitIdx > 0; digitIdx--) + { + destination[CIPHER_BLOCK_HEADER_MAGIC_SIZE + digitIdx - 1] = (uint8_t)('0' + format % 10); + format /= 10; + } + + destination[CIPHER_BLOCK_MAGIC_SIZE - 1] = CIPHER_BLOCK_HEADER_RESERVED; + + destination += CIPHER_BLOCK_MAGIC_SIZE; + destinationSize += CIPHER_BLOCK_MAGIC_SIZE; + } + // Else add magic to the destination buffer so openssl knows the file is salted + else if (!this->raw) { memcpy(destination, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE); destination += CIPHER_BLOCK_MAGIC_SIZE; @@ -161,9 +263,43 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s source += headerExpected - this->headerSize; sourceSize -= headerExpected - this->headerSize; - // The first bytes of the file to decrypt should be equal to the magic. If not then this is not an encrypted file, - // or at least not in a format we recognize. - if (!this->raw && memcmp(this->header, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) + // Read the format from the header. A file that begins with the magic instead was written at format 5, the only + // format there was before the header, so that start is not an error here. + if (this->headerFormat) + { + const char *const headerZ = (const char *)this->header; + unsigned int format = REPOSITORY_FORMAT_5; + + if (memcmp(headerZ, CIPHER_BLOCK_HEADER_MAGIC, CIPHER_BLOCK_HEADER_MAGIC_SIZE) == 0) + { + for (unsigned int digitIdx = 0; digitIdx < CIPHER_BLOCK_HEADER_FORMAT_SIZE; digitIdx++) + { + if (!isdigit((unsigned char)headerZ[CIPHER_BLOCK_HEADER_MAGIC_SIZE + digitIdx])) + THROW(FormatError, "invalid cipher header"); + } + + format = cvtZSubNToUInt(headerZ, CIPHER_BLOCK_HEADER_MAGIC_SIZE, CIPHER_BLOCK_HEADER_FORMAT_SIZE); + + // Error on a format this version cannot read before anything is decrypted + cipherBlockFormatValidate(format); + + // The format is one this version knows, so the byte held back for later must be the one this version writes + if (headerZ[CIPHER_BLOCK_MAGIC_SIZE - 1] != CIPHER_BLOCK_HEADER_RESERVED) + THROW(FormatError, "invalid cipher header"); + } + // Else the bytes must be the magic, since that is all a file with no header can begin with + else if (memcmp(headerZ, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) + THROW(CryptoError, "cipher header invalid"); + + // Error when the format the caller expected is not the one the file was written with + if (this->format != 0 && this->format != format) + THROW_FMT(FormatError, "expected repository format %u but found %u", this->format, format); + + this->format = format; + } + // Else the first bytes of the file to decrypt should be equal to the magic. If not then this is not an encrypted + // file, or at least not in a format we recognize. + else if (!this->raw && memcmp(this->header, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) THROW(CryptoError, "cipher header invalid"); } // Else copy what was provided into the header buffer and return 0 @@ -180,6 +316,10 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s // If salt generation/read is done if (salt) { + // Resolve the digest now that the format is known, which for a header that was read is only true here + if (this->digest == NULL) + this->digest = cipherBlockFormatDigest(this->format); + // Generate key and initialization vector uint8_t key[EVP_MAX_KEY_LENGTH]; uint8_t initVector[EVP_MAX_IV_LENGTH]; @@ -382,6 +522,49 @@ cipherBlockInputSame(const THIS_VOID) FUNCTION_TEST_RETURN(BOOL, this->inputSame); } +/*********************************************************************************************************************************** +Report the format the header gave +***********************************************************************************************************************************/ +static Pack * +cipherBlockResult(THIS_VOID) +{ + THIS(CipherBlock); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_BLOCK, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + Pack *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + PackWrite *const packWrite = pckWriteNewP(); + + pckWriteU32P(packWrite, this->format); + pckWriteEndP(packWrite); + + result = pckMove(pckWriteResult(packWrite), memContextPrior()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_TEST_RETURN(PACK, result); +} + +/**********************************************************************************************************************************/ +FN_EXTERN unsigned int +cipherBlockFormat(PackRead *const cipherBlockResult) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, cipherBlockResult); + FUNCTION_TEST_END(); + + ASSERT(cipherBlockResult != NULL); + + FUNCTION_TEST_RETURN(UINT, pckReadU32P(cipherBlockResult)); +} + /**********************************************************************************************************************************/ FN_EXTERN IoFilter * cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const CipherBlockNewParam param) @@ -390,12 +573,21 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const FUNCTION_LOG_PARAM(STRING_ID, mode); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); FUNCTION_LOG_PARAM(BOOL, param.raw); + FUNCTION_LOG_PARAM(BOOL, param.header); + FUNCTION_LOG_PARAM(UINT, param.format); FUNCTION_LOG_END(); ASSERT(cipherSpec != NULL); ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); ASSERT(cipherSpecPass(cipherSpec) != NULL && !bufEmpty(cipherSpecPass(cipherSpec))); + // The header takes the place of the magic, so a file that carries one is never also raw + ASSERT(!param.header || !param.raw); + + // The format must be known to write a header. On decrypt it is optional since the header is what says which format it is, but + // when it is given the header must agree with it. + ASSERT(mode != cipherModeEncrypt || !param.header || param.format != 0); + // Init crypto subsystem cryptoInit(); @@ -409,16 +601,37 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const zFree(cipherTypeZ); + // Lookup digest. A header that has yet to be read is what says which format the file is, and the format is what says which + // digest, so in that case the lookup waits until the header has been read. + const EVP_MD *digest = NULL; + + if (!param.header || mode == cipherModeEncrypt) + { + // A format says which digest, otherwise it comes from the spec and must be one openssl knows + if (param.format != 0) + digest = cipherBlockFormatDigest(param.format); + else + { + char digestZ[STRID_MAX + 1]; + strIdToZ(cipherSpecDigest(cipherSpec), digestZ); + + digest = EVP_get_digestbyname(digestZ); + + if (!digest) + THROW_FMT(AssertError, "unable to load digest '%s'", digestZ); + } + } + OBJ_NEW_BEGIN(CipherBlock, .childQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1) { *this = (CipherBlock) { .mode = mode, .raw = param.raw, + .headerFormat = param.header, + .format = param.format, .cipher = cipher, - - // The key is always derived with SHA-1. Deriving it with anything else is not supported yet. - .digest = EVP_sha1(), + .digest = digest, .pass = bufDup(cipherSpecPass(cipherSpec)), }; } @@ -434,6 +647,8 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const pckWriteU64P(packWrite, mode); cipherSpecPack(packWrite, cipherSpec); pckWriteBoolP(packWrite, param.raw); + pckWriteBoolP(packWrite, param.header); + pckWriteU32P(packWrite, param.format); pckWriteEndP(packWrite); paramList = pckMove(pckWriteResult(packWrite), memContextPrior()); @@ -444,7 +659,10 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const IO_FILTER, ioFilterNewP( CIPHER_BLOCK_FILTER_TYPE, this, paramList, .done = cipherBlockDone, .inOut = cipherBlockProcess, - .inputSame = cipherBlockInputSame)); + .inputSame = cipherBlockInputSame, + + // Only a filter that reads a header has a format to report + .result = param.header && mode == cipherModeDecrypt ? cipherBlockResult : NULL)); } FN_EXTERN IoFilter * @@ -458,8 +676,11 @@ cipherBlockNewPack(const Pack *const paramList) const CipherMode cipherMode = (CipherMode)pckReadU64P(paramListPack); const CipherSpec *const cipherSpec = cipherSpecNewPack(paramListPack); const bool raw = pckReadBoolP(paramListPack); + const bool header = pckReadBoolP(paramListPack); + const unsigned int format = pckReadU32P(paramListPack); - result = ioFilterMove(cipherBlockNewP(cipherMode, cipherSpec, .raw = raw), memContextPrior()); + result = ioFilterMove( + cipherBlockNewP(cipherMode, cipherSpec, .raw = raw, .header = header, .format = format), memContextPrior()); } MEM_CONTEXT_TEMP_END(); diff --git a/src/common/crypto/cipherBlock.h b/src/common/crypto/cipherBlock.h index ad28a26149..360b5de796 100644 --- a/src/common/crypto/cipherBlock.h +++ b/src/common/crypto/cipherBlock.h @@ -19,6 +19,8 @@ typedef struct CipherBlockNewParam { VAR_PARAM_HEADER; bool raw; // Omit header magic to save space + bool header; // Read/write the format header + unsigned int format; // Repository format, required to write a header } CipherBlockNewParam; #define cipherBlockNewP(mode, cipherSpec, ...) \ @@ -27,6 +29,13 @@ typedef struct CipherBlockNewParam FN_EXTERN IoFilter *cipherBlockNew(CipherMode mode, const CipherSpec *cipherSpec, CipherBlockNewParam param); FN_EXTERN IoFilter *cipherBlockNewPack(const Pack *paramList); +/*********************************************************************************************************************************** +Filter result + +The format the file was written with, which is what the header was read for. Only a filter that read a header has one to report. +***********************************************************************************************************************************/ +FN_EXTERN unsigned int cipherBlockFormat(PackRead *cipherBlockResult); + /*********************************************************************************************************************************** Helper functions ***********************************************************************************************************************************/ diff --git a/src/common/crypto/spec.c b/src/common/crypto/spec.c index 76627b8af9..6e32ffb5a9 100644 --- a/src/common/crypto/spec.c +++ b/src/common/crypto/spec.c @@ -17,11 +17,12 @@ struct CipherSpec /**********************************************************************************************************************************/ FN_EXTERN CipherSpec * -cipherSpecNew(const CipherType type, const Buffer *const pass) +cipherSpecNew(const CipherType type, const Buffer *const pass, const CipherSpecNewParam param) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING_ID, type); FUNCTION_TEST_PARAM(BUFFER, pass); + FUNCTION_TEST_PARAM(STRING_ID, param.digest); FUNCTION_TEST_END(); ASSERT((type == cipherTypeNone) == (pass == NULL)); @@ -32,7 +33,14 @@ cipherSpecNew(const CipherType type, const Buffer *const pass) *this = (CipherSpec){.pub = {.type = type}}; if (this->pub.type != cipherTypeNone) + { + if (param.digest == 0) + this->pub.digest = hashTypeSha256; + else + this->pub.digest = param.digest; + this->pub.pass = bufDup(pass); + } } OBJ_NEW_END(); @@ -56,7 +64,10 @@ cipherSpecNewPack(PackRead *const packRead) // Nothing else was written when there is no cipher. The pass is read here rather than copied in since this context is the // one it belongs in. if (this->pub.type != cipherTypeNone) + { + this->pub.digest = (HashType)pckReadStrIdP(packRead); this->pub.pass = pckReadBinP(packRead); + } } OBJ_NEW_END(); @@ -79,7 +90,10 @@ cipherSpecPack(PackWrite *const packWrite, const CipherSpec *const this) // Nothing else is needed when there is no cipher if (cipherSpecType(this) != cipherTypeNone) + { + pckWriteStrIdP(packWrite, cipherSpecDigest(this)); pckWriteBinP(packWrite, cipherSpecPass(this)); + } FUNCTION_TEST_RETURN_VOID(); } @@ -91,6 +105,14 @@ cipherSpecToLog(const CipherSpec *const this, StringStatic *const debugLog) char typeZ[STRID_MAX + 1]; strIdToZ(cipherSpecType(this), typeZ); - // The pass is never logged - strStcFmt(debugLog, "{type: %s}", typeZ); + // There is no digest when there is no cipher. The pass is never logged. + if (cipherSpecType(this) == cipherTypeNone) + strStcFmt(debugLog, "{type: %s}", typeZ); + else + { + char digestZ[STRID_MAX + 1]; + strIdToZ(cipherSpecDigest(this), digestZ); + + strStcFmt(debugLog, "{type: %s, digest: %s}", typeZ, digestZ); + } } diff --git a/src/common/crypto/spec.h b/src/common/crypto/spec.h index 003c9ca2ba..ca6e5e37fa 100644 --- a/src/common/crypto/spec.h +++ b/src/common/crypto/spec.h @@ -4,8 +4,19 @@ Cipher Spec Everything needed to encrypt or decrypt, kept together so that adding to it does not mean changing every function and protocol message that carries it. -The passphrase (pass) contains the bytes the key is derived from. There is no pass when the type is none, and the pass is never -logged. +The pass is the bytes the key is derived from rather than the text it was stored as. Whatever reads a pass from the repository +decides how to interpret it and builds a cipher spec from the result, so nothing downstream needs to know how it was stored. The +digest travels with the pass because the two are chosen together and deriving with the wrong digest produces a wrong key rather +than an error. + +The pass is a buffer rather than a string so an absent pass is simply NULL, which saves callers from guarding a conversion that +cannot represent one. It is copied into the object, so the caller is free to release whatever it was read from. + +The digest defaults to SHA-256, so a caller that has no reason to choose gets the digest new work should use. Deriving with SHA-1 +is what every repository did before repository format 6 and is now specified explicitly, which also marks the places that are +waiting on a way to tell an old pass from a new one. + +There is no digest or pass when the type is none, and the pass is never logged. ***********************************************************************************************************************************/ #ifndef COMMON_CRYPTO_SPEC_H #define COMMON_CRYPTO_SPEC_H @@ -23,14 +34,23 @@ typedef struct CipherSpec CipherSpec; /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -// Create from a pass -FN_EXTERN CipherSpec *cipherSpecNew(CipherType type, const Buffer *pass); +// Create from a pass, which is the key bytes or the passphrase text rather than what either was stored as +typedef struct CipherSpecNewParam +{ + VAR_PARAM_HEADER; + HashType digest; // Digest to derive the key with instead of SHA-256 +} CipherSpecNewParam; + +#define cipherSpecNewP(type, pass, ...) \ + cipherSpecNew(type, pass, (CipherSpecNewParam){VAR_PARAM_INIT, __VA_ARGS__}) -// Create for no encryption +FN_EXTERN CipherSpec *cipherSpecNew(CipherType type, const Buffer *pass, CipherSpecNewParam param); + +// Create for data that is not encrypted FN_INLINE_ALWAYS CipherSpec * cipherSpecNewNone(void) { - return cipherSpecNew(cipherTypeNone, NULL); + return cipherSpecNewP(cipherTypeNone, NULL); } // Create from a pack written by cipherSpecPack() @@ -42,6 +62,7 @@ Getters/Setters typedef struct CipherSpecPub { CipherType type; // Cipher type, none when not encrypted + HashType digest; // Digest the pass derives the key with const Buffer *pass; // Passphrase text or key bytes } CipherSpecPub; @@ -52,6 +73,13 @@ cipherSpecType(const CipherSpec *const this) return THIS_PUB(CipherSpec)->type; } +// Digest the pass derives the key with +FN_INLINE_ALWAYS HashType +cipherSpecDigest(const CipherSpec *const this) +{ + return THIS_PUB(CipherSpec)->digest; +} + // Passphrase text or key bytes FN_INLINE_ALWAYS const Buffer * cipherSpecPass(const CipherSpec *const this) @@ -66,7 +94,7 @@ Functions FN_INLINE_ALWAYS CipherSpec * cipherSpecDup(const CipherSpec *const this) { - return cipherSpecNew(cipherSpecType(this), cipherSpecPass(this)); + return cipherSpecNewP(cipherSpecType(this), cipherSpecPass(this), .digest = cipherSpecDigest(this)); } // Write to a pack so it can be passed over a protocol diff --git a/src/config/config.c b/src/config/config.c index de49e46c3b..7b49f979d2 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1117,7 +1117,7 @@ cfgCipherSpecMainIdx(const unsigned int repoIdx) { const CipherType cipherType = cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx); - configLocal->cipherSpecMain[repoIdx] = cipherSpecNew( + configLocal->cipherSpecMain[repoIdx] = cipherSpecNewP( cipherType, cipherType == cipherTypeNone ? NULL : BUFSTR(cfgOptionIdxStr(cfgOptRepoCipherPass, repoIdx))); } MEM_CONTEXT_END(); diff --git a/src/info/info.c b/src/info/info.c index d31941f1b1..3453f9122a 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -3,14 +3,19 @@ Info Handler ***********************************************************************************************************************************/ #include +#include #include #include #include +#include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" #include "common/ini.h" +#include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" #include "common/io/filter/filter.h" +#include "common/io/io.h" #include "common/log.h" #include "common/type/convert.h" #include "common/type/json.h" @@ -117,6 +122,50 @@ infoNew(const unsigned int format, const CipherSpec *const cipherSpecSub) FUNCTION_LOG_RETURN(INFO, this); } +// Error when the format cannot be read by this version. Called for the format in the header before anything is decrypted and again +// for the format in the content, since the two are written together but stored apart. +static void +infoFormatValidate(const uint64_t format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT64, format); + FUNCTION_TEST_END(); + + // A format newer than this version can read requires an upgrade. Do not suggest a version since this version cannot know which + // version added the format. + if (format > REPOSITORY_FORMAT_MAX) + { + THROW_FMT( + FormatError, + "repository format %" PRIu64 " requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + // A format older than this version can read requires an older version to migrate the repository + if (format < REPOSITORY_FORMAT_MIN) + { + THROW_FMT( + FormatError, + "repository format %" PRIu64 " is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN HashType +infoFormatDigest(const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RETURN(STRING_ID, format >= REPOSITORY_FORMAT_6 ? hashTypeSha256 : hashTypeSha1); +} + /**********************************************************************************************************************************/ #define INFO_SECTION_BACKREST "backrest" #define INFO_KEY_CHECKSUM "backrest-checksum" @@ -125,13 +174,15 @@ infoNew(const unsigned int format, const CipherSpec *const cipherSpecSub) FN_EXTERN Info * infoNewLoad( - IoRead *const read, const CipherSpec *const cipherSpec, InfoLoadNewCallback *const callbackFunction, void *const callbackData) + IoRead *const read, const CipherSpec *const cipherSpec, InfoLoadNewCallback *const callbackFunction, + void *const callbackData, const InfoNewLoadParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(IO_READ, read); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); FUNCTION_LOG_PARAM(FUNCTIONP, callbackFunction); FUNCTION_LOG_PARAM_P(VOID, callbackData); + FUNCTION_LOG_PARAM(BOOL, param.header); FUNCTION_LOG_END(); FUNCTION_AUDIT_CALLBACK(); @@ -150,12 +201,22 @@ infoNewLoad( String *const sectionLast = strNew(); // The last section seen during load IoFilter *const checksumActualFilter = cryptoHashNew(hashTypeSha1); // Checksum calculated from the file const String *checksumExpected = NULL; // Checksum found in ini file + unsigned int formatHeader = 0; // Format the header gave, 0 when there is none + IoRead *contentRead = read; // Read the content comes from INFO_CHECKSUM_BEGIN(checksumActualFilter); TRY_BEGIN() { - Ini *const ini = iniNewP(read, .strict = true); + // The content is decrypted as it is parsed. A file that may carry a header is read with one, which the cipher + // consumes and reports the format of once the read is done. + if (cipherSpecType(cipherSpec) != cipherTypeNone) + { + ioFilterGroupAdd( + ioReadFilterGroup(read), cipherBlockNewP(cipherModeDecrypt, cipherSpec, .header = param.header)); + } + + Ini *const ini = iniNewP(contentRead, .strict = true); MEM_CONTEXT_TEMP_RESET_BEGIN() { @@ -187,27 +248,7 @@ infoNewLoad( if (strEqZ(value->key, INFO_KEY_FORMAT)) { const uint64_t format = varUInt64(jsonToVar(value->value)); - - // A format newer than this version can read requires an upgrade. Do not suggest a version since - // this version cannot know which version added the format. - if (format > REPOSITORY_FORMAT_MAX) - { - THROW_FMT( - FormatError, - "repository format %" PRIu64 " requires a newer version of " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } - - // A format older than this version can read requires an older version to migrate the repository - if (format < REPOSITORY_FORMAT_MIN) - { - THROW_FMT( - FormatError, - "repository format %" PRIu64 " is no longer supported by " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } + infoFormatValidate(format); this->pub.format = (unsigned int)format; } @@ -238,9 +279,12 @@ infoNewLoad( { MEM_CONTEXT_OBJ_BEGIN(this) { - // The dependent files are encrypted with the same cipher type as this one - this->pub.cipherSpec = cipherSpecNew( - cipherSpecType(cipherSpec), BUFSTR(varStr(jsonToVar(value->value)))); + // The dependent files are encrypted with the same cipher type as this one and derive with the + // digest that goes with the format this file was written at. The format is read before this + // since the sections come out in order and backrest sorts before cipher. + this->pub.cipherSpec = cipherSpecNewP( + cipherSpecType(cipherSpec), BUFSTR(varStr(jsonToVar(value->value))), + .digest = infoFormatDigest(this->pub.format)); } MEM_CONTEXT_OBJ_END(); } @@ -281,6 +325,22 @@ infoNewLoad( // format is zero until the key is found and the value stored, so if we got here then the key was not found. if (infoFormat(this) == 0) THROW(FormatError, "repository format not found\nHINT: is this a valid " PROJECT_NAME " info file?"); + + // Only a cipher that read a header reports a format, so a result here is what says the file had one. The header is + // written from the same format as the content, so a file where they disagree has been damaged or put together from + // parts of two files. + PackRead *const cipherResult = ioFilterGroupResultP(ioReadFilterGroup(read), CIPHER_BLOCK_FILTER_TYPE); + + if (cipherResult != NULL) + { + formatHeader = cipherBlockFormat(cipherResult); + + if (this->pub.format != formatHeader) + { + THROW_FMT( + FormatError, "repository format %u does not match header format %u", this->pub.format, formatHeader); + } + } } MEM_CONTEXT_TEMP_END(); @@ -293,6 +353,37 @@ infoNewLoad( FUNCTION_LOG_RETURN(INFO, this); } +/**********************************************************************************************************************************/ +FN_EXTERN IoWrite * +infoWriteNew(Buffer *const buffer, const unsigned int format, const CipherSpec *const cipherSpec) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(BUFFER, buffer); + FUNCTION_LOG_PARAM(UINT, format); + FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_LOG_END(); + + FUNCTION_AUDIT_HELPER(); + + ASSERT(buffer != NULL); + ASSERT(format >= REPOSITORY_FORMAT_MIN && format <= REPOSITORY_FORMAT_MAX); + ASSERT(cipherSpec != NULL); + + IoWrite *const result = ioBufferWriteNew(buffer); + + // The cipher writes the header and derives the pass with the digest the format calls for. Format 5 gets no header since it is + // the format a reader assumes when there is nothing to say otherwise. + if (cipherSpecType(cipherSpec) != cipherTypeNone) + { + ioFilterGroupAdd( + ioWriteFilterGroup(result), + cipherBlockNewP( + cipherModeEncrypt, cipherSpec, .header = format >= REPOSITORY_FORMAT_6, .format = format)); + } + + FUNCTION_LOG_RETURN(IO_WRITE, result); +} + /**********************************************************************************************************************************/ FN_EXTERN bool infoSaveSection(InfoSave *const infoSaveData, const char *const section, const String *const sectionNext) diff --git a/src/info/info.h b/src/info/info.h index 3af5818afc..9fbc4a78f9 100644 --- a/src/info/info.h +++ b/src/info/info.h @@ -41,10 +41,22 @@ Constructors ***********************************************************************************************************************************/ FN_EXTERN Info *infoNew(unsigned int format, const CipherSpec *cipherSpecSub); -// Create new object and load contents from a file. The cipher spec the file is read with supplies the type for the cipher spec -// built from the pass stored in it. +// Create new object and load contents from a file. Decryption is added here rather than by the caller because a file that carries +// a header cannot be decrypted until the header has been read. The cipher spec supplies the type and pass; the digest comes from +// the format the file turns out to be at. The same spec supplies the type for the cipher spec built from the pass stored in the +// file. +typedef struct InfoNewLoadParam +{ + VAR_PARAM_HEADER; + bool header; // Does the file begin with a header? +} InfoNewLoadParam; + +#define infoNewLoadP(read, cipherSpec, callbackFunction, callbackData, ...) \ + infoNewLoad(read, cipherSpec, callbackFunction, callbackData, (InfoNewLoadParam){VAR_PARAM_INIT, __VA_ARGS__}) + FN_EXTERN Info *infoNewLoad( - IoRead *read, const CipherSpec *cipherSpec, InfoLoadNewCallback *callbackFunction, void *callbackData); + IoRead *read, const CipherSpec *cipherSpec, InfoLoadNewCallback *callbackFunction, void *callbackData, + InfoNewLoadParam param); /*********************************************************************************************************************************** Getters/Setters @@ -86,6 +98,18 @@ infoBackrestVersion(const Info *const this) /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ +// Read the header of an encrypted info file and return a read of the content behind it. The read must already be open since the +// header is read from it immediately, and it is consumed and closed here so that filters the caller added to it produce their +// results. The header has to be read before the digest is known, by which time an encryption filter can no longer be added, so the +// content is decrypted into a buffer here rather than as it is read. An info file is small enough for that to be reasonable, and is +// already built whole in a buffer when it is saved so that it can be written twice. The format the header gives is returned when +// format is not NULL, so that the content can be checked against it. + +// Create a write to save an info file into a buffer. The header is written and the encryption filter added according to the +// format, so the caller has only to save into the write it gets back. The write side is the one that knows the format, so unlike +// the load there is nothing to work out first. +FN_EXTERN IoWrite *infoWriteNew(Buffer *buffer, unsigned int format, const CipherSpec *cipherSpec); + // Save to file FN_EXTERN void infoSave(Info *this, IoWrite *write, InfoSaveCallback *callbackFunction, void *callbackData); @@ -101,6 +125,11 @@ Helper functions // Load info file(s) and throw error for each attempt if none are successful FN_EXTERN void infoLoad(const String *error, InfoLoadCallback *callbackFunction, void *callbackData); +// Digest a pass stored in a file at this format derives the key with. SHA-1 is what every repository used before format 6 and is +// kept for those, so a repository that has not been migrated is read and written exactly as it was. A pass is generated with the +// digest of the file it will be stored in, since that is what a reader will derive it with. +FN_EXTERN HashType infoFormatDigest(unsigned int format); + /*********************************************************************************************************************************** Macros for function logging ***********************************************************************************************************************************/ diff --git a/src/info/infoArchive.c b/src/info/infoArchive.c index 4e38a60038..d6839154c3 100644 --- a/src/info/infoArchive.c +++ b/src/info/infoArchive.c @@ -250,10 +250,8 @@ infoArchiveLoadFileCallback(void *const data, const unsigned int try) // Construct filename based on try const String *const fileName = try == 0 ? loadData->fileName : strNewFmt("%s" INFO_COPY_EXT, strZ(loadData->fileName)); - // Attempt to load the file + // Attempt to load the file. Decryption is added during load since the header has to be read before the digest is known. IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); - cipherBlockFilterGroupAdd( - ioReadFilterGroup(read), cipherModeDecrypt, loadData->cipherSpec); MEM_CONTEXT_BEGIN(loadData->memContext) { @@ -339,8 +337,7 @@ infoArchiveSaveFile( { // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); - IoWrite *const write = ioBufferWriteNew(buffer); - cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec); + IoWrite *const write = infoWriteNew(buffer, infoArchiveFormat(infoArchive), cipherSpec); infoArchiveSave(infoArchive, write); // Save the file and make a copy diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index 5385097a2f..c10ce3f843 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -714,10 +714,8 @@ infoBackupLoadFileCallback(void *const data, const unsigned int try) // Construct filename based on try const String *const fileName = try == 0 ? loadData->fileName : strNewFmt("%s" INFO_COPY_EXT, strZ(loadData->fileName)); - // Attempt to load the file + // Attempt to load the file. Decryption is added during load since the header has to be read before the digest is known. IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); - cipherBlockFilterGroupAdd( - ioReadFilterGroup(read), cipherModeDecrypt, loadData->cipherSpec); MEM_CONTEXT_BEGIN(loadData->memContext) { @@ -938,8 +936,7 @@ infoBackupSaveFile( { // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); - IoWrite *const write = ioBufferWriteNew(buffer); - cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec); + IoWrite *const write = infoWriteNew(buffer, infoBackupFormat(infoBackup), cipherSpec); infoBackupSave(infoBackup, write); // Save the file and make a copy diff --git a/src/info/infoPg.c b/src/info/infoPg.c index ebced1fc02..840b00c6e1 100644 --- a/src/info/infoPg.c +++ b/src/info/infoPg.c @@ -189,7 +189,8 @@ infoPgNewLoad( .infoPg = this, }; - this->pub.info = infoNewLoad(read, cipherSpec, infoPgLoadCallback, &loadData); + // The archive.info and backup.info files are the only users of this object and both carry a header + this->pub.info = infoNewLoadP(read, cipherSpec, infoPgLoadCallback, &loadData, .header = true); CHECK(FormatError, !lstEmpty(this->pub.history), "history is missing"); CHECK(FormatError, loadData.currentId > 0, "current id is missing"); diff --git a/src/info/manifest/manifest.c b/src/info/manifest/manifest.c index ebd3ea1678..b46ab149f3 100644 --- a/src/info/manifest/manifest.c +++ b/src/info/manifest/manifest.c @@ -705,7 +705,7 @@ manifestNewLoad(IoRead *const read, const CipherSpec *const cipherSpec) } MEM_CONTEXT_END(); - this->pub.info = infoNewLoad(read, cipherSpec, manifestLoadCallback, &loadData); + this->pub.info = infoNewLoadP(read, cipherSpec, manifestLoadCallback, &loadData); this->pub.data.backrestVersion = infoBackrestVersion(this->pub.info); // Add the label to the reference list in case the manifest was created before 2.42 when the explicit reference list was diff --git a/src/info/manifest/serialize.c.inc b/src/info/manifest/serialize.c.inc index 64bf889b79..97b19469a6 100644 --- a/src/info/manifest/serialize.c.inc +++ b/src/info/manifest/serialize.c.inc @@ -1037,9 +1037,9 @@ manifestLoadFileCallback(void *const data, const unsigned int try) // Construct filename based on try const String *const fileName = try == 0 ? loadData->fileName : strNewFmt("%s" INFO_COPY_EXT, strZ(loadData->fileName)); - // Attempt to load the file + // Attempt to load the file. Decryption is added during the load so that everything an info file is read with is in one + // place, even though a manifest carries no header. IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); - cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, loadData->cipherSpec); MEM_CONTEXT_BEGIN(loadData->memContext) { diff --git a/test/src/harness/config.h b/test/src/harness/config.h index c129316457..ea25041232 100644 --- a/test/src/harness/config.h +++ b/test/src/harness/config.h @@ -18,7 +18,7 @@ Config option constants #define TEST_CIPHER_PASS_ARCHIVE "xarchivex" // Cipher spec for a test pass, defaulting to the main pass, since most tests encrypt with that -#define TEST_CIPHER_SPEC_PASS(pass) cipherSpecNew(cipherTypeAes256Cbc, BUFSTRZ(pass)) +#define TEST_CIPHER_SPEC_PASS(pass) cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRZ(pass)) #define TEST_CIPHER_SPEC TEST_CIPHER_SPEC_PASS(TEST_CIPHER_PASS) /*********************************************************************************************************************************** diff --git a/test/src/harness/host.c b/test/src/harness/host.c index d82554a0d5..6492c946d8 100644 --- a/test/src/harness/host.c +++ b/test/src/harness/host.c @@ -1232,7 +1232,7 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si hrnHostLocal.storage = strIdFromZ(testDef->stg); hrnHostLocal.compressType = compressTypeEnum(strIdFromZ(testDef->cmp)); hrnHostLocal.cipherSpecMain = - testDef->enc ? cipherSpecNew(cipherTypeAes256Cbc, BUFSTRZ(HRN_CIPHER_PASSPHRASE)) : cipherSpecNewNone(); + testDef->enc ? cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRZ(HRN_CIPHER_PASSPHRASE)) : cipherSpecNewNone(); hrnHostLocal.repoTotal = testDef->rt; hrnHostLocal.tls = testDef->tls; hrnHostLocal.bundle = testDef->bnd; diff --git a/test/src/harness/info.c b/test/src/harness/info.c index d99f178e4c..9f56b5910c 100644 --- a/test/src/harness/info.c +++ b/test/src/harness/info.c @@ -6,13 +6,16 @@ Harness for Loading Test Configurations #include #include "common/assert.h" +#include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" #include "common/io/filter/filter.h" #include "common/type/json.h" #include "info/info.h" #include "version.h" +#include "harness/config.h" #include "harness/debug.h" #include "harness/info.h" @@ -116,6 +119,83 @@ harnessInfoChecksum(const String *const info) FUNCTION_HARNESS_RETURN(BUFFER, harnessInfoChecksumFormat(REPOSITORY_FORMAT_DEFAULT, info)); } +/**********************************************************************************************************************************/ +void +hrnInfoPut(const Storage *const storage, const char *const file, const char *const info, HrnInfoPutParam param) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STORAGE, storage); + FUNCTION_HARNESS_PARAM(STRINGZ, file); + FUNCTION_HARNESS_PARAM(STRINGZ, info); + FUNCTION_HARNESS_PARAM(UINT, param.format); + FUNCTION_HARNESS_PARAM(BOOL, param.header); + FUNCTION_HARNESS_PARAM(CIPHER_SPEC, param.cipherSpec); + FUNCTION_HARNESS_PARAM(STRINGZ, param.comment); + FUNCTION_HARNESS_END(); + + ASSERT(info != NULL); + + // Default to the format a new repository is created at + if (param.format == 0) + param.format = REPOSITORY_FORMAT_DEFAULT; + + const Buffer *content = harnessInfoChecksumFormat(param.format, STR(info)); + + // Encrypt the way the format stores the file. A file that carries a header gets it in place of the magic the cipher writes, + // and from format 6 the pass derives with SHA-256 rather than SHA-1. + if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) + { + const bool header = param.header && param.format >= REPOSITORY_FORMAT_6; + Buffer *const encrypted = bufNew(0); + + if (header) + bufCat(encrypted, BUFSTR(strNewFmt("PGBR%03u_", param.format))); + + IoWrite *const write = ioBufferWriteNew(encrypted); + ioFilterGroupAdd( + ioWriteFilterGroup(write), + cipherBlockNewP( + cipherModeEncrypt, + cipherSpecNewP( + cipherSpecType(param.cipherSpec), cipherSpecPass(param.cipherSpec), + .digest = param.format >= REPOSITORY_FORMAT_6 ? hashTypeSha256 : hashTypeSha1), + .raw = header)); + + ioWriteOpen(write); + ioWrite(write, content); + ioWriteClose(write); + + content = encrypted; + } + + hrnStoragePut(storage, file, content, "put info", (HrnStoragePutParam){VAR_PARAM_INIT, .comment = param.comment}); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +Buffer * +harnessInfoEncrypt(const Buffer *const content, const CipherSpec *const cipherSpec) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(BUFFER, content); + FUNCTION_HARNESS_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_HARNESS_END(); + + ASSERT(content != NULL); + ASSERT(cipherSpec != NULL); + + Buffer *const result = bufNew(0); + IoWrite *const write = ioBufferWriteNew(result); + ioFilterGroupAdd(ioWriteFilterGroup(write), cipherBlockNewP(cipherModeEncrypt, cipherSpec)); + + ioWriteOpen(write); + ioWrite(write, content); + ioWriteClose(write); + + FUNCTION_HARNESS_RETURN(BUFFER, result); +} + /*********************************************************************************************************************************** Test callback that logs the results to a string ***********************************************************************************************************************************/ diff --git a/test/src/harness/info.h b/test/src/harness/info.h index e1457ee7df..7682bff1ea 100644 --- a/test/src/harness/info.h +++ b/test/src/harness/info.h @@ -13,17 +13,27 @@ know this since the option supplies it everywhere else. #define REPOSITORY_FORMAT_DEFAULT REPOSITORY_FORMAT_5 /*********************************************************************************************************************************** -Write info to a file and add the checksum +Write info to a file and add the checksum, storing it the way the format stores it when it is encrypted ***********************************************************************************************************************************/ +typedef struct HrnInfoPutParam +{ + VAR_PARAM_HEADER; + unsigned int format; // Repository format, default format when zero + bool header; // Does the file carry a header, i.e. is it an info file? + const CipherSpec *cipherSpec; // Cipher spec when the file is encrypted, digest set by format + const char *comment; // Comment +} HrnInfoPutParam; + #define HRN_INFO_PUT(storage, file, info, ...) \ do \ { \ hrnTestLogPrefix(__LINE__); \ - hrnStoragePut( \ - storage, file, harnessInfoChecksumZ(info), "put info", (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__}); \ + hrnInfoPut(storage, file, info, (HrnInfoPutParam){VAR_PARAM_INIT, __VA_ARGS__}); \ } \ while (0) +void hrnInfoPut(const Storage *storage, const char *file, const char *info, HrnInfoPutParam param); + /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ @@ -31,4 +41,7 @@ Buffer *harnessInfoChecksum(const String *info); Buffer *harnessInfoChecksumFormat(unsigned int format, const String *info); Buffer *harnessInfoChecksumZ(const char *info); +// Encrypt content the way a file that carries no header is stored, e.g. a manifest +Buffer *harnessInfoEncrypt(const Buffer *content, const CipherSpec *cipherSpec); + void harnessInfoLoadNewCallback(void *callbackData, const String *section, const String *key, JsonRead *json); diff --git a/test/src/harness/storage.c b/test/src/harness/storage.c index 89a5dbdd7f..e096315207 100644 --- a/test/src/harness/storage.c +++ b/test/src/harness/storage.c @@ -69,7 +69,14 @@ testStorageGet(const Storage *const storage, const char *const file, const char // Add decrypt filter if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) { - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, param.cipherSpec)); + // Derive with SHA-1 since the harness reads and writes files the way a repository at the format these tests build stores + // them, which is the format that had no header to say anything else + ioFilterGroupAdd( + filterGroup, + cipherBlockNewP( + cipherModeDecrypt, + cipherSpecNewP( + cipherSpecType(param.cipherSpec), cipherSpecPass(param.cipherSpec), .digest = hashTypeSha1))); strCatFmt( filter, "enc[%s,%s] ", zNewStrId(cipherSpecType(param.cipherSpec)), @@ -393,7 +400,15 @@ hrnStoragePut( // Add encrypted filter if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, param.cipherSpec)); + { + // Derive with SHA-1 to match how the harness reads these files back + ioFilterGroupAdd( + filterGroup, + cipherBlockNewP( + cipherModeEncrypt, + cipherSpecNewP( + cipherSpecType(param.cipherSpec), cipherSpecPass(param.cipherSpec), .digest = hashTypeSha1))); + } // Add file name printf( diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index 853726305b..69afbb2bd4 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -1331,7 +1331,7 @@ testRun(void) blockIncrNew( 3, 3, 8, 2, 4, 5, NULL, compressFilterP(compressTypeGz, 1, .raw = true), cipherBlockNewP( - cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS)), .raw = true)))), + cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS)), .raw = true)))), "block incr pack"); } diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index 53d9fe0d3e..8487a0a7ad 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -311,7 +311,8 @@ testRun(void) TEST_ERROR( restoreFile( strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strZ(repoFileReferenceFull), strZ(repoFile1)), repoIdx, compressTypeGz, - 0, false, false, false, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("badpass")), NULL, fileList), + 0, false, false, false, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("badpass"), .digest = hashTypeSha1), NULL, + fileList), ChecksumError, "error restoring 'normal': actual checksum 'd1cd8a7d11daa26814b93eb604e1d49ab4b43770' does not match expected checksum" " 'ffffffffffffffffffffffffffffffffffffffff'"); @@ -2309,7 +2310,7 @@ testRun(void) Manifest *manifestEncrypted = manifestLoadFile( storageRepoIdxWrite(0), STRDEF(STORAGE_REPO_BACKUP "/" TEST_LABEL "/" BACKUP_MANIFEST_FILE), cipherSpecNewNone()); - manifestCipherSpecSet(manifestEncrypted, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); + manifestCipherSpecSet(manifestEncrypted, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); // Open file for write IoWrite *write = storageWriteIo( @@ -2321,7 +2322,8 @@ testRun(void) #define TEST_CIPHER_PASS_MANIFEST "backpass" cipherBlockFilterGroupAdd( ioWriteFilterGroup(write), cipherModeEncrypt, - cipherSpecNew(cfgOptionIdxStrId(cfgOptRepoCipherType, 1), BUFSTRDEF(TEST_CIPHER_PASS_MANIFEST))); + cipherSpecNewP( + cfgOptionIdxStrId(cfgOptRepoCipherType, 1), BUFSTRDEF(TEST_CIPHER_PASS_MANIFEST), .digest = hashTypeSha1)); manifestSave(manifestEncrypted, write); // Write backup.info to the encrypted repo @@ -2332,10 +2334,10 @@ testRun(void) // Write archive.info to the encrypted repo InfoArchive *infoArchive = infoArchiveNew( PG_VERSION_11, 6569239123849665679, REPOSITORY_FORMAT_DEFAULT, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); infoArchiveSaveFile( infoArchive, storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE_STR, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS))); + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS))); TEST_RESULT_VOID(hrnCmdRestore(), "successful restore"); diff --git a/test/src/module/command/stanzaTest.c b/test/src/module/command/stanzaTest.c index 1519a0603e..ba7bc1a6c7 100644 --- a/test/src/module/command/stanzaTest.c +++ b/test/src/module/command/stanzaTest.c @@ -137,7 +137,7 @@ testRun(void) TEST_ASSIGN( infoArchive, infoArchiveLoadFile( - storageRepoIdx(1), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), + storageRepoIdx(1), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), "load archive info from encrypted repo2"); TEST_RESULT_UINT(cipherSpecType(infoArchiveCipherSpec(infoArchive)), cipherTypeAes256Cbc, "cipher sub set"); @@ -145,7 +145,7 @@ testRun(void) TEST_ASSIGN( infoBackup, infoBackupLoadFile( - storageRepoIdx(1), INFO_BACKUP_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), + storageRepoIdx(1), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), "load backup info from encrypted repo2"); TEST_RESULT_UINT(cipherSpecType(infoBackupCipherSpec(infoBackup)), cipherTypeAes256Cbc, "cipher sub set"); @@ -185,14 +185,14 @@ testRun(void) TEST_ASSIGN( infoArchive, infoArchiveLoadFile( - storageRepoIdx(3), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), + storageRepoIdx(3), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), "load archive info from encrypted repo4"); TEST_RESULT_UINT(cipherSpecType(infoArchiveCipherSpec(infoArchive)), cipherTypeAes256Cbc, "cipher sub set"); TEST_ASSIGN( infoBackup, infoBackupLoadFile( - storageRepoIdx(3), INFO_BACKUP_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), + storageRepoIdx(3), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), "load backup info from encrypted repo4"); TEST_RESULT_UINT(cipherSpecType(infoBackupCipherSpec(infoBackup)), cipherTypeAes256Cbc, "cipher sub set"); diff --git a/test/src/module/command/verifyTest.c b/test/src/module/command/verifyTest.c index 92740965aa..e4a3abd123 100644 --- a/test/src/module/command/verifyTest.c +++ b/test/src/module/command/verifyTest.c @@ -876,22 +876,43 @@ testRun(void) hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 1, TEST_CIPHER_PASS); HRN_CFG_LOAD(cfgCmdVerify, argList); + // The copy is a copy of the file rather than a second encryption of the same content, which is how the info files are + // saved, i.e. the content is built once in a buffer and that buffer is written twice HRN_INFO_PUT( - storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .cipherSpec = TEST_CIPHER_SPEC, - .comment = "encrypted backup.info"); + storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .header = true, + .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted backup.info"); + HRN_STORAGE_COPY( + storageRepo(), INFO_BACKUP_PATH_FILE, storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted backup.info.copy"); HRN_INFO_PUT( - storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, TEST_NO_CURRENT_BACKUP, - .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted backup.info.copy"); - HRN_INFO_PUT( - storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, .header = true, .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted archive.info"); - HRN_INFO_PUT( - storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, - .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted archive.info.copy"); + HRN_STORAGE_COPY( + storageRepo(), INFO_ARCHIVE_PATH_FILE, storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted archive.info.copy"); TEST_RESULT_VOID(cmdVerify(), "usable encrypted backup and archive info files"); TEST_RESULT_LOG(""); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("encrypted info files at format 6"); + + HRN_INFO_PUT( + storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .format = REPOSITORY_FORMAT_6, .header = true, + .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted backup.info at format 6"); + HRN_STORAGE_COPY( + storageRepo(), INFO_BACKUP_PATH_FILE, storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted backup.info.copy at format 6"); + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, .format = REPOSITORY_FORMAT_6, + .header = true, .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted archive.info at format 6"); + HRN_STORAGE_COPY( + storageRepo(), INFO_ARCHIVE_PATH_FILE, storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted archive.info.copy at format 6"); + + TEST_RESULT_VOID(cmdVerify(), "usable encrypted backup and archive info files at format 6"); + TEST_RESULT_LOG(""); + hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 1); harnessLogLevelReset(); } @@ -941,12 +962,12 @@ testRun(void) TEST_RESULT_UINT( verifyFile( filePathName, 0, NULL, compressTypeGz, fileChecksum, fileSize, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("pass"))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("pass"), .digest = hashTypeSha1)), verifyOk, "file encrypted compressed ok"); TEST_RESULT_UINT( verifyFile( filePathName, 0, NULL, compressTypeGz, bufNewDecode(encodingHex, STRDEF("aa")), fileSize, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("pass"))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("pass"), .digest = hashTypeSha1)), verifyChecksumMismatch, "file encrypted compressed checksum mismatch"); } diff --git a/test/src/module/common/cryptoTest.c b/test/src/module/common/cryptoTest.c index d3d42e1429..1879cde127 100644 --- a/test/src/module/common/cryptoTest.c +++ b/test/src/module/common/cryptoTest.c @@ -2,6 +2,7 @@ Test Block Cipher ***********************************************************************************************************************************/ #include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" #include "common/io/filter/filter.h" #include "common/io/io.h" #include "common/type/json.h" @@ -72,21 +73,26 @@ testRun(void) // ***************************************************************************************************************************** if (testBegin("CipherBlock")) { - // Cipher error + // Cipher and digest errors // ------------------------------------------------------------------------------------------------------------------------- TEST_ERROR( - cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(strIdFromZ(BOGUS_STR), testPass)), AssertError, + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(strIdFromZ(BOGUS_STR), testPass)), AssertError, "unable to load cipher 'BOGUS'"); + TEST_ERROR( + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass, .digest = strIdFromZ(BOGUS_STR))), + AssertError, "unable to load digest 'BOGUS'"); // Initialization of object // ------------------------------------------------------------------------------------------------------------------------- - // Build from a duplicate to show the copy carries the type and pass of the original + // Build from a duplicate to show the copy carries the type, digest, and pass of the original TEST_RESULT_UINT(cipherSpecType(cipherSpecDup(cipherSpecNewNone())), cipherTypeNone, "dup of none"); - const CipherSpec *const cipherSpec = cipherSpecDup(cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS))); + const CipherSpec *const cipherSpec = cipherSpecDup(cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS))); - TEST_RESULT_UINT(cipherSpecType(cipherSpec), cipherTypeAes256Cbc, "dup type"); - TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(cipherSpec)), TEST_PASS, "dup pass"); + TEST_RESULT_UINT(cipherSpecDigest(cipherSpec), hashTypeSha256, "dup default digest"); + TEST_RESULT_UINT( + cipherSpecDigest(cipherSpecDup(cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS), .digest = hashTypeSha1))), + hashTypeSha1, "dup digest"); // A pack carries nothing but the type when there is no cipher PackWrite *packWrite = pckWriteNewP(); @@ -97,17 +103,171 @@ testRun(void) TEST_RESULT_UINT( cipherSpecType(cipherSpecNewPack(pckReadNew(pckWriteResult(packWrite)))), cipherTypeNone, "unpack none"); - // Else it carries the type and pass + // Else it carries the type, digest, and pass. Pack a digest that is not the default so that a pack which loses the digest + // cannot pass by falling back to the default. packWrite = pckWriteNewP(); - cipherSpecPack(packWrite, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + cipherSpecPack(packWrite, cipherSpecNewP(cipherTypeAes256Cbc, testPass, .digest = hashTypeSha1)); pckWriteEndP(packWrite); const CipherSpec *const cipherSpecUnpack = cipherSpecNewPack(pckReadNew(pckWriteResult(packWrite))); TEST_RESULT_UINT(cipherSpecType(cipherSpecUnpack), cipherTypeAes256Cbc, "unpack type"); + TEST_RESULT_UINT(cipherSpecDigest(cipherSpecUnpack), hashTypeSha1, "unpack digest"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(cipherSpecUnpack)), TEST_PASS, "unpack pass"); + // Format header + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("write a header and read the format back from it"); + + Buffer *headerBuffer = bufNew(TEST_BUFFER_SIZE); + IoWrite *headerWrite = ioBufferWriteNew(headerBuffer); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerWrite), + cipherBlockNewP( + cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true, + .format = REPOSITORY_FORMAT_6)); + ioWriteOpen(headerWrite); + ioWrite(headerWrite, testPlainText); + ioWriteClose(headerWrite); + + TEST_RESULT_BOOL( + memcmp(bufPtrConst(headerBuffer), CIPHER_BLOCK_HEADER_MAGIC "006_", CIPHER_BLOCK_MAGIC_SIZE) == 0, true, + "header names the format"); + + // The format is not given on decrypt, so it comes from the header and is what the pass derives with + Buffer *headerResult = bufNew(TEST_BUFFER_SIZE); + IoWrite *headerRead = ioBufferWriteNew(headerResult); + IoFilterGroup *headerFilterGroup = ioWriteFilterGroup(headerRead); + + ioFilterGroupAdd( + headerFilterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true)); + ioWriteOpen(headerRead); + ioWrite(headerRead, headerBuffer); + ioWriteClose(headerRead); + + TEST_RESULT_STR_Z(strNewBuf(headerResult), TEST_PLAINTEXT, "content decrypted with the digest the header called for"); + TEST_RESULT_UINT( + cipherBlockFormat(ioFilterGroupResultP(headerFilterGroup, CIPHER_BLOCK_FILTER_TYPE)), REPOSITORY_FORMAT_6, + "filter reports the format"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a file that begins with the magic was written before there was a header"); + + headerBuffer = bufNew(TEST_BUFFER_SIZE); + headerWrite = ioBufferWriteNew(headerBuffer); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerWrite), + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass, .digest = hashTypeSha1))); + ioWriteOpen(headerWrite); + ioWrite(headerWrite, testPlainText); + ioWriteClose(headerWrite); + + headerResult = bufNew(TEST_BUFFER_SIZE); + headerRead = ioBufferWriteNew(headerResult); + headerFilterGroup = ioWriteFilterGroup(headerRead); + + ioFilterGroupAdd( + headerFilterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true)); + ioWriteOpen(headerRead); + ioWrite(headerRead, headerBuffer); + ioWriteClose(headerRead); + + TEST_RESULT_STR_Z(strNewBuf(headerResult), TEST_PLAINTEXT, "content decrypted"); + TEST_RESULT_UINT( + cipherBlockFormat(ioFilterGroupResultP(headerFilterGroup, CIPHER_BLOCK_FILTER_TYPE)), REPOSITORY_FORMAT_5, + "filter reports the format before the header"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a format given on decrypt must be the one the header names"); + + headerBuffer = bufNew(TEST_BUFFER_SIZE); + headerWrite = ioBufferWriteNew(headerBuffer); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerWrite), + cipherBlockNewP( + cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true, + .format = REPOSITORY_FORMAT_6)); + ioWriteOpen(headerWrite); + ioWrite(headerWrite, testPlainText); + ioWriteClose(headerWrite); + + IoWrite *const headerMismatch = ioBufferWriteNew(bufNew(TEST_BUFFER_SIZE)); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerMismatch), + cipherBlockNewP( + cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true, + .format = REPOSITORY_FORMAT_5)); + ioWriteOpen(headerMismatch); + + TEST_ERROR( + ioWrite(headerMismatch, headerBuffer), FormatError, "expected repository format 5 but found 6"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a format given on decrypt that the header agrees with is accepted"); + + headerResult = bufNew(TEST_BUFFER_SIZE); + headerRead = ioBufferWriteNew(headerResult); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerRead), + cipherBlockNewP( + cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true, + .format = REPOSITORY_FORMAT_6)); + ioWriteOpen(headerRead); + ioWrite(headerRead, headerBuffer); + ioWriteClose(headerRead); + + TEST_RESULT_STR_Z(strNewBuf(headerResult), TEST_PLAINTEXT, "content decrypted"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("damaged headers"); + + // Decrypt the buffer above after damaging a byte of the header, which is the same buffer for each case + #define TEST_HEADER_DAMAGE(damageIdx, damageChar, errorType, errorMessage) \ + do \ + { \ + Buffer *const damaged = bufDup(headerBuffer); \ + bufPtr(damaged)[damageIdx] = damageChar; \ + \ + IoWrite *const write = ioBufferWriteNew(bufNew(TEST_BUFFER_SIZE)); \ + ioFilterGroupAdd( \ + ioWriteFilterGroup(write), \ + cipherBlockNewP( \ + cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true)); \ + ioWriteOpen(write); \ + \ + TEST_ERROR(ioWrite(write, damaged), errorType, errorMessage); \ + } \ + while (0) + + // Where the format should be, so this version cannot tell what it is looking at + TEST_HEADER_DAMAGE(CIPHER_BLOCK_HEADER_MAGIC_SIZE, 'X', FormatError, "invalid cipher header"); + + // The byte held back for later, which must be the one this version writes + TEST_HEADER_DAMAGE(CIPHER_BLOCK_MAGIC_SIZE - 1, 'X', FormatError, "invalid cipher header"); + + // A format newer than this version can read, reported before anything is decrypted + TEST_HEADER_DAMAGE( + CIPHER_BLOCK_MAGIC_SIZE - 2, '7', FormatError, + "repository format 7 requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + + // A format older than this version can read + TEST_HEADER_DAMAGE( + CIPHER_BLOCK_MAGIC_SIZE - 2, '4', FormatError, + "repository format 4 is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + + // Neither a header nor the magic, which is what a file that was never encrypted looks like from here + TEST_HEADER_DAMAGE(0, 'X', CryptoError, "cipher header invalid"); + + #undef TEST_HEADER_DAMAGE + CipherBlock *cipherBlock = (CipherBlock *)ioFilterDriver(cipherBlockNewP(cipherModeEncrypt, cipherSpec)); TEST_RESULT_UINT(cipherBlock->mode, cipherModeEncrypt, "mode is valid"); TEST_RESULT_UINT(bufSize(cipherBlock->pass), strlen(TEST_PASS), "passphrase size is valid"); @@ -123,7 +283,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- Buffer *encryptBuffer = bufNew(TEST_BUFFER_SIZE); - IoFilter *blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + IoFilter *blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockEncryptFilter = cipherBlockNewPack(ioFilterParamList(blockEncryptFilter)); CipherBlock *blockEncrypt = (CipherBlock *)ioFilterDriver(blockEncryptFilter); @@ -174,7 +334,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- Buffer *decryptBuffer = bufNew(TEST_BUFFER_SIZE); - IoFilter *blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + IoFilter *blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecryptFilter = cipherBlockNewPack(ioFilterParamList(blockDecryptFilter)); CipherBlock *blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); @@ -194,7 +354,7 @@ testRun(void) // Decrypt in small chunks to test buffering // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -235,7 +395,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("encrypt zero byte file with no magic"); - blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass), .raw = true); + blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .raw = true); blockEncrypt = (CipherBlock *)ioFilterDriver(blockEncryptFilter); bufUsedZero(encryptBuffer); @@ -248,13 +408,13 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("error on decrypt expecting magic"); - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); TEST_ERROR(ioFilterProcessInOut(blockDecryptFilter, encryptBuffer, decryptBuffer), CryptoError, "cipher header invalid"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("decrypt zero byte file with no magic"); - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass), .raw = true); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .raw = true); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -268,7 +428,7 @@ testRun(void) // Invalid cipher header // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); TEST_ERROR( @@ -279,7 +439,7 @@ testRun(void) // Invalid encrypted data cannot be flushed // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -293,7 +453,7 @@ testRun(void) // File with no header should not flush // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -304,7 +464,7 @@ testRun(void) // File with header only should error // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -325,7 +485,7 @@ testRun(void) TEST_RESULT_VOID( cipherBlockFilterGroupAdd( - filterGroup, cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("X"))), " filter add"); + filterGroup, cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("X"))), " filter add"); TEST_RESULT_UINT(ioFilterGroupSize(filterGroup), 1, " check filter add"); } diff --git a/test/src/module/info/infoArchiveTest.c b/test/src/module/info/infoArchiveTest.c index 5a33e9a516..6f826221c3 100644 --- a/test/src/module/info/infoArchiveTest.c +++ b/test/src/module/info/infoArchiveTest.c @@ -87,16 +87,20 @@ testRun(void) info, infoArchiveNew( PG_VERSION_10, 6569239123849665999, REPOSITORY_FORMAT_DEFAULT, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), "infoArchiveNew() - cipher sub"); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + contentSave = bufNew(0); - TEST_RESULT_VOID(infoArchiveSave(info, ioBufferWriteNew(contentSave)), "save new with cipher"); + TEST_RESULT_VOID( + infoArchiveSave(info, infoWriteNew(contentSave, REPOSITORY_FORMAT_DEFAULT, cipherSpec)), "save new with cipher"); + TEST_RESULT_BOOL( + strBeginsWithZ(strNewBuf(contentSave), "PGBR"), false, "no header before the format that added it"); TEST_ASSIGN( - info, infoArchiveNewLoad(ioBufferReadNew(contentSave), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))), - "load encrypted archive info"); + info, infoArchiveNewLoad(ioBufferReadNew(contentSave), cipherSpec), "load encrypted archive info"); TEST_RESULT_STR_Z(infoArchiveId(info), "10-1", "archiveId set"); TEST_RESULT_PTR(infoArchivePg(info), infoArchivePg(info), "infoPg set"); TEST_RESULT_STR_Z( diff --git a/test/src/module/info/infoBackupTest.c b/test/src/module/info/infoBackupTest.c index ab1e8a5cee..83886ebb17 100644 --- a/test/src/module/info/infoBackupTest.c +++ b/test/src/module/info/infoBackupTest.c @@ -87,18 +87,22 @@ testRun(void) TEST_ASSIGN( infoBackup, infoBackupNew( - PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), REPOSITORY_FORMAT_DEFAULT, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), + PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), REPOSITORY_FORMAT_6, + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), "infoBackupNew() - cipher sub"); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + contentSave = bufNew(0); - TEST_RESULT_VOID(infoBackupSave(infoBackup, ioBufferWriteNew(contentSave)), "save new with cipher sub"); + TEST_RESULT_VOID( + infoBackupSave(infoBackup, infoWriteNew(contentSave, REPOSITORY_FORMAT_6, cipherSpec)), "save new with cipher sub"); + TEST_RESULT_STR_Z( + strNewZN((const char *)bufPtrConst(contentSave), 8), "PGBR006_", "header names the format"); infoBackup = NULL; TEST_ASSIGN( - infoBackup, infoBackupNewLoad(ioBufferReadNew(contentSave), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))), - "load backup info with cipher sub"); + infoBackup, infoBackupNewLoad(ioBufferReadNew(contentSave), cipherSpec), "load backup info with cipher sub"); TEST_RESULT_PTR(infoBackupPg(infoBackup), infoBackupPg(infoBackup), "infoPg set"); TEST_RESULT_STR_Z( strNewBuf(cipherSpecPass(infoBackupCipherSpec(infoBackup))), diff --git a/test/src/module/info/infoPgTest.c b/test/src/module/info/infoPgTest.c index 13a04301b9..18b7578ada 100644 --- a/test/src/module/info/infoPgTest.c +++ b/test/src/module/info/infoPgTest.c @@ -43,7 +43,7 @@ testRun(void) TEST_ASSIGN( infoPg, - infoPgNew(infoPgArchive, REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + infoPgNew(infoPgArchive, REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), "infoPgNew(cipherTypeAes256Cbc, 123xyz)"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 0, " 0 history"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(infoPgInfo(infoPg)))), "123xyz", " cipherPass set"); @@ -80,7 +80,7 @@ testRun(void) TEST_ASSIGN( infoPg, infoPgSet( - infoPgNew(infoPgBackup, REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + infoPgNew(infoPgBackup, REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), PG_VERSION_18, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_18)), "infoPgSet - infoPgBackup"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 1, " 1 history"); diff --git a/test/src/module/info/infoTest.c b/test/src/module/info/infoTest.c index 2bd306e5d3..d7f8ea33e3 100644 --- a/test/src/module/info/infoTest.c +++ b/test/src/module/info/infoTest.c @@ -67,6 +67,22 @@ testInfoSaveCallback(void *data, const String *sectionNext, InfoSave *infoSaveDa infoSaveValue(infoSaveData, "d", "key", (String *)data); } +/*********************************************************************************************************************************** +Store info file content the way it would be stored at a format, i.e. with the header and digest that go with the format +***********************************************************************************************************************************/ +static Buffer * +testInfoEncrypt(const Buffer *const content, const unsigned int format, const CipherSpec *const cipherSpec) +{ + Buffer *const result = bufNew(0); + IoWrite *const write = infoWriteNew(result, format, cipherSpec); + + ioWriteOpen(write); + ioWrite(write, content); + ioWriteClose(write); + + return result; +} + /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -79,7 +95,7 @@ testRun(void) Info *info = NULL; TEST_ASSIGN( - info, infoNew(REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + info, infoNew(REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), "infoNew(cipher)"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(info))), "123xyz", " cipherPass is set"); @@ -99,7 +115,7 @@ testRun(void) String *callbackContent = strNew(); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), FormatError, "repository format 4 is no longer supported by pgBackRest\n" "HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); @@ -112,7 +128,7 @@ testRun(void) "backrest-format=7\n"); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), FormatError, "repository format 7 requires a newer version of pgBackRest\n" "HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); @@ -124,7 +140,7 @@ testRun(void) "backrest-format=5\n"); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), ChecksumError, "invalid checksum, actual 'a3765a8c2c1e5d35274a0b0ce118f4031faff0bd' but no checksum found"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); @@ -138,7 +154,7 @@ testRun(void) "bogus=\"BOGUS\"\n"); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), ChecksumError, "invalid checksum, actual 'fe989a75dcf7a0261e57d210707c0db741462763' but expected 'BOGUS'"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); @@ -150,7 +166,7 @@ testRun(void) "backrest-version=\"2.17\"\n"); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), FormatError, "repository format not found\n" "HINT: is this a valid pgBackRest info file?"); @@ -167,10 +183,10 @@ testRun(void) IoRead *read = ioBufferReadNew(contentLoad); ioFilterGroupAdd( ioReadFilterGroup(read), - cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("X")))); + cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("X")))); TEST_ERROR( - infoNewLoad(read, cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), CryptoError, + infoNewLoadP(read, cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), CryptoError, "cipher header invalid\n" "HINT: is or was the repo encrypted?"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); @@ -185,7 +201,7 @@ testRun(void) callbackContent = strNew(); TEST_ASSIGN( - info, infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + info, infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), "info with other cipher"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(info)), cipherTypeNone, " check cipher pass not set"); @@ -202,7 +218,7 @@ testRun(void) callbackContent = strNew(); TEST_ASSIGN( - info, infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + info, infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), "info with content"); TEST_RESULT_STR_Z(callbackContent, "[c] key=1\n[d] key=1\n", " check callback content"); TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(info)), cipherTypeNone, " check cipher pass not set"); @@ -227,7 +243,7 @@ testRun(void) callbackContent = strNew(); TEST_ASSIGN( - info, infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + info, infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), "info format 6"); TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); @@ -255,20 +271,125 @@ testRun(void) callbackContent = strNew(); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + + // A file with no header, e.g. a manifest, is decrypted with the spec as it was given since there is nothing to say the + // digest should be anything else TEST_ASSIGN( info, - infoNewLoad( - ioBufferReadNew(contentLoad), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")), harnessInfoLoadNewCallback, + infoNewLoadP( + ioBufferReadNew(harnessInfoEncrypt(contentLoad, cipherSpec)), cipherSpec, harnessInfoLoadNewCallback, callbackContent), "info with content and cipher"); TEST_RESULT_STR_Z(callbackContent, "[c] key=1\n[d] key=1\n", " check callback content"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(info))), "somepass", " check cipher pass set"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); TEST_RESULT_STR_Z(infoBackrestVersion(info), PROJECT_VERSION, " check backrest version"); contentSave = bufNew(0); TEST_RESULT_VOID(infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "info save"); TEST_RESULT_STR(strNewBuf(contentSave), strNewBuf(contentLoad), " check save"); + + // Header + // ------------------------------------------------------------------------------------------------------------------------- + // An unencrypted file has no header no matter the format, since the format is read from the content + contentSave = bufNew(0); + + TEST_RESULT_VOID( + infoSave( + info, infoWriteNew(contentSave, REPOSITORY_FORMAT_6, cipherSpecNewNone()), testInfoSaveCallback, strNewZ("1")), + "info save"); + TEST_RESULT_BOOL(strBeginsWithZ(strNewBuf(contentSave), "PGBR"), false, " check no header"); + + contentLoad = harnessInfoChecksumFormat( + REPOSITORY_FORMAT_6, + STRDEF( + "[cipher]\n" + "cipher-pass=\"somepass\"\n")); + + callbackContent = strNew(); + + TEST_ASSIGN( + info, + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, callbackContent, .header = true), + "info with header"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha256, " check cipher sub digest"); + + // The content on its own, which is how a caller that wants the file rather than the values in it reads an info file + IoRead *const infoRead = ioBufferReadNew(testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec)); + + ioFilterGroupAdd(ioReadFilterGroup(infoRead), cipherBlockNewP(cipherModeDecrypt, cipherSpec, .header = true)); + ioReadOpen(infoRead); + + TEST_RESULT_STR(strNewBuf(ioReadBuf(infoRead)), strNewBuf(contentLoad), "info content read"); + + // A file written before the header existed is read as the format that had none + contentLoad = harnessInfoChecksumZ("[c]\nkey=1\n"); + + TEST_ASSIGN( + info, + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_5, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, callbackContent, .header = true), + "info with no header"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_5, " check format"); + + // A file too short to hold a header cannot have one + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(BUFSTRDEF("PGBR")), cipherSpec, harnessInfoLoadNewCallback, callbackContent, .header = true), + CryptoError, + "cipher header missing\n" + "HINT: is or was the repo encrypted?"); + + // A file with neither header was never encrypted, which is reported the way the cipher reports it + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(BUFSTRDEF("[backrest]\nbackrest-format=5\n")), cipherSpec, harnessInfoLoadNewCallback, + callbackContent, .header = true), + CryptoError, + "cipher header invalid\n" + "HINT: is or was the repo encrypted?"); + + // Header uses the byte held back for later, so this version does not know what it is looking at + Buffer *contentHeader = testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec); + bufPtr(contentHeader)[7] = 'X'; + + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(contentHeader), cipherSpec, harnessInfoLoadNewCallback, callbackContent, .header = true), + FormatError, "invalid cipher header"); + + // Header damaged where the format should be + contentHeader = testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec); + bufPtr(contentHeader)[5] = 'X'; + + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(contentHeader), cipherSpec, harnessInfoLoadNewCallback, callbackContent, .header = true), + FormatError, "invalid cipher header"); + + // Header names a format this version cannot read, which is reported before anything is decrypted + contentHeader = testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec); + bufPtr(contentHeader)[6] = '7'; + + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(contentHeader), cipherSpec, harnessInfoLoadNewCallback, callbackContent, .header = true), + FormatError, + "repository format 7 requires a newer version of pgBackRest\n" + "HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); + + // Header and content disagree about the format + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, callbackContent, .header = true), + FormatError, "repository format 5 does not match header format 6"); } // ***************************************************************************************************************************** diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index 1d97cb2382..af48de8f8f 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -1478,7 +1478,10 @@ testRun(void) MEM_CONTEXT_TEMP_BEGIN() { TEST_ASSIGN( - manifest, manifestNewLoad(ioBufferReadNew(contentLoad), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))), + manifest, + manifestNewLoad( + ioBufferReadNew(harnessInfoEncrypt(contentLoad, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x"))), "load manifest"); TEST_RESULT_VOID(manifestMove(manifest, memContextPrior()), "move manifest"); } @@ -1781,7 +1784,7 @@ testRun(void) TEST_RESULT_UINT(cipherSpecType(manifestCipherSpec(manifest)), cipherTypeNone, "check cipher subpass"); TEST_RESULT_VOID( - manifestCipherSpecSet(manifest, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("supersecret"))), + manifestCipherSpecSet(manifest, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("supersecret"))), "cipher subpass set"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(manifestCipherSpec(manifest))), "supersecret", "check cipher subpass"); diff --git a/test/src/module/storage/remoteTest.c b/test/src/module/storage/remoteTest.c index b3047f305e..c22ff4d7de 100644 --- a/test/src/module/storage/remoteTest.c +++ b/test/src/module/storage/remoteTest.c @@ -381,8 +381,8 @@ testRun(void) IoFilterGroup *filterGroup = ioReadFilterGroup(storageReadIo(fileRead)); ioFilterGroupAdd(filterGroup, ioSizeNew()); ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1)); - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")))); - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")))); + ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")))); + ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")))); ioFilterGroupAdd(filterGroup, compressFilterP(compressTypeGz, 3)); ioFilterGroupAdd(filterGroup, decompressFilterP(compressTypeGz)); From 8bd03f64d64f57acb3f6f1eee738a0519d6e257e Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 10:03:53 +0700 Subject: [PATCH 03/15] Minor fixes. --- src/info/info.c | 6 ++---- src/info/info.h | 7 ------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/info/info.c b/src/info/info.c index 3453f9122a..cd407b28bc 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -201,8 +201,6 @@ infoNewLoad( String *const sectionLast = strNew(); // The last section seen during load IoFilter *const checksumActualFilter = cryptoHashNew(hashTypeSha1); // Checksum calculated from the file const String *checksumExpected = NULL; // Checksum found in ini file - unsigned int formatHeader = 0; // Format the header gave, 0 when there is none - IoRead *contentRead = read; // Read the content comes from INFO_CHECKSUM_BEGIN(checksumActualFilter); @@ -216,7 +214,7 @@ infoNewLoad( ioReadFilterGroup(read), cipherBlockNewP(cipherModeDecrypt, cipherSpec, .header = param.header)); } - Ini *const ini = iniNewP(contentRead, .strict = true); + Ini *const ini = iniNewP(read, .strict = true); MEM_CONTEXT_TEMP_RESET_BEGIN() { @@ -333,7 +331,7 @@ infoNewLoad( if (cipherResult != NULL) { - formatHeader = cipherBlockFormat(cipherResult); + const unsigned int formatHeader = cipherBlockFormat(cipherResult); if (this->pub.format != formatHeader) { diff --git a/src/info/info.h b/src/info/info.h index 9fbc4a78f9..cd147d9d36 100644 --- a/src/info/info.h +++ b/src/info/info.h @@ -98,13 +98,6 @@ infoBackrestVersion(const Info *const this) /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ -// Read the header of an encrypted info file and return a read of the content behind it. The read must already be open since the -// header is read from it immediately, and it is consumed and closed here so that filters the caller added to it produce their -// results. The header has to be read before the digest is known, by which time an encryption filter can no longer be added, so the -// content is decrypted into a buffer here rather than as it is read. An info file is small enough for that to be reasonable, and is -// already built whole in a buffer when it is saved so that it can be written twice. The format the header gives is returned when -// format is not NULL, so that the content can be checked against it. - // Create a write to save an info file into a buffer. The header is written and the encryption filter added according to the // format, so the caller has only to save into the write it gets back. The write side is the one that knows the format, so unlike // the load there is nothing to work out first. From 79d0c1e43873efcd5cde4b5a2d386f4020d09ce0 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 10:06:09 +0700 Subject: [PATCH 04/15] Clarify. --- src/command/check/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/command/check/common.c b/src/command/check/common.c index af6f86857d..2541291875 100644 --- a/src/command/check/common.c +++ b/src/command/check/common.c @@ -123,7 +123,7 @@ checkStanzaInfo(const unsigned int repoIdx, const InfoPg *const archiveInfoPg, c } // Error if the info files are at different repository formats. The formats are written together but stored apart, so an - // upgrade interrupted between the two saves leaves them here. + // upgrade interrupted between the two saves leaves them mismatched. if (infoPgFormat(archiveInfoPg) != infoPgFormat(backupInfoPg)) { const unsigned int formatArchive = infoPgFormat(archiveInfoPg); From fc36cafcbc895000b29da11baa150f2f4090371e Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 11:20:35 +0700 Subject: [PATCH 05/15] Update from review. --- build/config.yaml | 2 +- src/command/repo/get.c | 2 +- src/command/stanza/common.c | 3 +- src/common/crypto/cipherBlock.c | 93 ++++++++--------------------- src/common/crypto/spec.h | 17 +++--- src/common/format.c | 51 ++++++++++++++++ src/common/format.h | 37 ++++++++++++ src/info/info.c | 53 ++-------------- src/info/info.h | 5 -- src/info/manifest/serialize.c.inc | 3 +- src/meson.build | 1 + src/version.h | 14 ----- test/define.yaml | 7 +++ test/src/harness/info.c | 1 + test/src/harness/info.h | 1 + test/src/module/common/cryptoTest.c | 1 + test/src/module/common/formatTest.c | 38 ++++++++++++ 17 files changed, 179 insertions(+), 150 deletions(-) create mode 100644 src/common/format.c create mode 100644 src/common/format.h create mode 100644 test/src/module/common/formatTest.c diff --git a/build/config.yaml b/build/config.yaml index 6b500af5a5..66a442d5f0 100644 --- a/build/config.yaml +++ b/build/config.yaml @@ -1691,7 +1691,7 @@ option: # Command-line only so that a format left in a configuration file cannot upgrade a repository as a side effect of an unrelated # stanza-upgrade. This does not cover the environment, which can set any option that is valid for the command, so a format left # there does migrate on the next stanza-upgrade and the option reference says so. The allow list must be kept in sync with - # REPOSITORY_FORMAT_MIN/MAX in version.h + # REPOSITORY_FORMAT_MIN/MAX in common/format.h repo-format: group: repo type: integer diff --git a/src/command/repo/get.c b/src/command/repo/get.c index 84a9a39f73..789aa39c05 100644 --- a/src/command/repo/get.c +++ b/src/command/repo/get.c @@ -48,7 +48,7 @@ storageGetProcess(IoWrite *const destination) IoRead *const source = storageReadIo( storageNewReadP(storageRepo(), file, .ignoreMissing = cfgOptionBool(cfgOptIgnoreMissing))); - // Is the file an info file, i.e. one that carries a header in front of its content? + // Is the file an info file, i.e. one that has a header in front of its content? bool fileIsInfo = false; // Add decryption if needed diff --git a/src/command/stanza/common.c b/src/command/stanza/common.c index cf802ae03b..05c2063b41 100644 --- a/src/command/stanza/common.c +++ b/src/command/stanza/common.c @@ -6,6 +6,7 @@ Stanza Commands Handler #include "command/check/common.h" #include "command/stanza/common.h" #include "common/debug.h" +#include "common/format.h" #include "common/log.h" #include "config/config.h" #include "db/helper.h" @@ -38,7 +39,7 @@ cipherSpecGen(const CipherType cipherType, const unsigned int format) // digest is the one the file it will be stored in is read with, since that is what will derive it later. result = cipherSpecNewP( cipherType, BUFSTR(strNewEncode(encodingBase64, BUF(buffer, sizeof(buffer)))), - .digest = infoFormatDigest(format)); + .digest = repoFormatDigest(format)); cipherSpecMove(result, memContextPrior()); } MEM_CONTEXT_TEMP_END(); diff --git a/src/common/crypto/cipherBlock.c b/src/common/crypto/cipherBlock.c index a374323480..1ddf856a5d 100644 --- a/src/common/crypto/cipherBlock.c +++ b/src/common/crypto/cipherBlock.c @@ -12,11 +12,11 @@ Block Cipher #include "common/crypto/cipherBlock.h" #include "common/crypto/common.h" #include "common/debug.h" +#include "common/format.h" #include "common/io/filter/filter.h" #include "common/log.h" #include "common/type/convert.h" #include "common/type/object.h" -#include "version.h" /*********************************************************************************************************************************** Magic constant for salted encrypt, written before the salt unless the cipher is raw. Only salted encrypt is done here, but this @@ -32,18 +32,18 @@ A file written with a header begins with fixed-size plaintext naming the reposit because the digest the pass derives with follows the format, and the format is recorded inside the file that the pass encrypts. A reader that could not see the format in advance would have to decrypt to learn what it should have decrypted with. -The header takes the place of the salted magic, which is why a file that carries one is written raw. Both are eight bytes followed +The header takes the place of the salted magic, which is why a file that contains one is written raw. Both are eight bytes followed by the salt, so the eight bytes are consumed either way and what follows begins with the salt no matter which was there. It also means a file of either kind is opened with the openssl command-line tool the same way: replace the first eight bytes with the magic that tool expects. A file that begins with the magic rather than the header was written at format 5, the only format there was before the header, so -that start is not an error when a header was expected. +that is not an error when a header was expected. -Of the four bytes after the header magic, the first three are the format and the last is held back for whatever the header turns -out to need, e.g. naming which key the file was encrypted with once a repository can hold more than one. The format comes first so -that it is always at the same place, which is what lets a version work out whether it can read the file at all. Only once the -format turns out to be one this version knows is the spare byte examined, and then it must be the underscore this version writes. +For the four bytes after the header magic, the first three are the format and the last is reserved for future use, e.g. naming which +key the file was encrypted with once a repository can hold more than one. The format comes first so that it is always at the same +place, which is what lets a version work out whether it can read the file at all. Once the format is identified as compatible with +this version, the spare byte is examined, and it must only be the underscore this version writes. The header is not part of the file content. This filter adds it on encrypt and consumes it on decrypt, so nothing on either side sees anything but the content. @@ -57,53 +57,24 @@ sees anything but the content. #define CIPHER_BLOCK_HEADER_SIZE (CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN) /*********************************************************************************************************************************** -Digest the pass derives the key with at a format. A format that predates the header derived with SHA-1, which is why a file with no -header is read with it. +Digest the pass derives the key with. The lookup is by name, so a digest must be one openssl knows. ***********************************************************************************************************************************/ static const EVP_MD * -cipherBlockFormatDigest(const unsigned int format) +cipherBlockDigest(const HashType type) { FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_PARAM(STRING_ID, type); FUNCTION_TEST_END(); - // Both accessors always return a digest, so unlike a lookup by name this cannot fail - FUNCTION_TEST_RETURN_TYPE_CONST_P(EVP_MD, format >= REPOSITORY_FORMAT_6 ? EVP_sha256() : EVP_sha1()); -} - -/*********************************************************************************************************************************** -Error when the format read from a header cannot be read by this version. This is checked before anything is decrypted since -decrypting requires knowing what the format expects and this version does not know what a newer format expects. -***********************************************************************************************************************************/ -static void -cipherBlockFormatValidate(const unsigned int format) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(UINT, format); - FUNCTION_TEST_END(); + char typeZ[STRID_MAX + 1]; + strIdToZ(type, typeZ); - // A format newer than this version can read requires an upgrade. Do not suggest a version since this version cannot know which - // version added the format. - if (format > REPOSITORY_FORMAT_MAX) - { - THROW_FMT( - FormatError, - "repository format %u requires a newer version of " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } + const EVP_MD *const result = EVP_get_digestbyname(typeZ); - // A format older than this version can read requires an older version to migrate the repository - if (format < REPOSITORY_FORMAT_MIN) - { - THROW_FMT( - FormatError, - "repository format %u is no longer supported by " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } + if (result == NULL) + THROW_FMT(AssertError, "unable to load digest '%s'", typeZ); - FUNCTION_TEST_RETURN_VOID(); + FUNCTION_TEST_RETURN_TYPE_CONST_P(EVP_MD, result); } /*********************************************************************************************************************************** @@ -219,14 +190,11 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s { memcpy(destination, CIPHER_BLOCK_HEADER_MAGIC, CIPHER_BLOCK_HEADER_MAGIC_SIZE); - // Write the format right-aligned in the digits it gets so it is always at the same place - unsigned int format = this->format; - - for (unsigned int digitIdx = CIPHER_BLOCK_HEADER_FORMAT_SIZE; digitIdx > 0; digitIdx--) - { - destination[CIPHER_BLOCK_HEADER_MAGIC_SIZE + digitIdx - 1] = (uint8_t)('0' + format % 10); - format /= 10; - } + // Write the format zero-padded so it is always the same size. The terminator lands on the reserved byte, which is + // written next. + snprintf( + (char *)destination + CIPHER_BLOCK_HEADER_MAGIC_SIZE, CIPHER_BLOCK_HEADER_FORMAT_SIZE + 1, "%0*u", + CIPHER_BLOCK_HEADER_FORMAT_SIZE, this->format); destination[CIPHER_BLOCK_MAGIC_SIZE - 1] = CIPHER_BLOCK_HEADER_RESERVED; @@ -281,7 +249,7 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s format = cvtZSubNToUInt(headerZ, CIPHER_BLOCK_HEADER_MAGIC_SIZE, CIPHER_BLOCK_HEADER_FORMAT_SIZE); // Error on a format this version cannot read before anything is decrypted - cipherBlockFormatValidate(format); + repoFormatValidate(format); // The format is one this version knows, so the byte held back for later must be the one this version writes if (headerZ[CIPHER_BLOCK_MAGIC_SIZE - 1] != CIPHER_BLOCK_HEADER_RESERVED) @@ -318,7 +286,7 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s { // Resolve the digest now that the format is known, which for a header that was read is only true here if (this->digest == NULL) - this->digest = cipherBlockFormatDigest(this->format); + this->digest = cipherBlockDigest(repoFormatDigest(this->format)); // Generate key and initialization vector uint8_t key[EVP_MAX_KEY_LENGTH]; @@ -607,19 +575,8 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const if (!param.header || mode == cipherModeEncrypt) { - // A format says which digest, otherwise it comes from the spec and must be one openssl knows - if (param.format != 0) - digest = cipherBlockFormatDigest(param.format); - else - { - char digestZ[STRID_MAX + 1]; - strIdToZ(cipherSpecDigest(cipherSpec), digestZ); - - digest = EVP_get_digestbyname(digestZ); - - if (!digest) - THROW_FMT(AssertError, "unable to load digest '%s'", digestZ); - } + // A format says which digest, otherwise it comes from the spec + digest = cipherBlockDigest(param.format != 0 ? repoFormatDigest(param.format) : cipherSpecDigest(cipherSpec)); } OBJ_NEW_BEGIN(CipherBlock, .childQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1) diff --git a/src/common/crypto/spec.h b/src/common/crypto/spec.h index ca6e5e37fa..782db77db0 100644 --- a/src/common/crypto/spec.h +++ b/src/common/crypto/spec.h @@ -4,17 +4,14 @@ Cipher Spec Everything needed to encrypt or decrypt, kept together so that adding to it does not mean changing every function and protocol message that carries it. -The pass is the bytes the key is derived from rather than the text it was stored as. Whatever reads a pass from the repository -decides how to interpret it and builds a cipher spec from the result, so nothing downstream needs to know how it was stored. The -digest travels with the pass because the two are chosen together and deriving with the wrong digest produces a wrong key rather -than an error. +The pass holds the bytes used to derive the key. The digest travels with the pass because the two are chosen together and deriving +with the wrong digest gives a wrong key instead of an error. -The pass is a buffer rather than a string so an absent pass is simply NULL, which saves callers from guarding a conversion that -cannot represent one. It is copied into the object, so the caller is free to release whatever it was read from. +The pass is a buffer rather than a string because it may be binary or it may be text and nothing here needs to know which. It is +copied into the object, so the caller can release whatever it read the pass from. -The digest defaults to SHA-256, so a caller that has no reason to choose gets the digest new work should use. Deriving with SHA-1 -is what every repository did before repository format 6 and is now specified explicitly, which also marks the places that are -waiting on a way to tell an old pass from a new one. +The digest defaults to SHA-256, so a caller with no preference gets the digest new work should use. Repositories before format 6 +derived with SHA-1, so a caller that works with that format must pass SHA-1 rather than taking the default. There is no digest or pass when the type is none, and the pass is never logged. ***********************************************************************************************************************************/ @@ -34,7 +31,7 @@ typedef struct CipherSpec CipherSpec; /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -// Create from a pass, which is the key bytes or the passphrase text rather than what either was stored as +// Create from a pass, which is the key bytes or the passphrase text typedef struct CipherSpecNewParam { VAR_PARAM_HEADER; diff --git a/src/common/format.c b/src/common/format.c new file mode 100644 index 0000000000..aba51f5710 --- /dev/null +++ b/src/common/format.c @@ -0,0 +1,51 @@ +/*********************************************************************************************************************************** +Repository Format +***********************************************************************************************************************************/ +#include + +#include "common/debug.h" +#include "common/format.h" +#include "version.h" + +/**********************************************************************************************************************************/ +FN_EXTERN void +repoFormatValidate(const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + // A format newer than this version can read requires an upgrade. Do not suggest a version since this version cannot know which + // version added the format. + if (format > REPOSITORY_FORMAT_MAX) + { + THROW_FMT( + FormatError, + "repository format %u requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + // A format older than this version can read requires an older version to migrate the repository + if (format < REPOSITORY_FORMAT_MIN) + { + THROW_FMT( + FormatError, + "repository format %u is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN HashType +repoFormatDigest(const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RETURN(STRING_ID, format >= REPOSITORY_FORMAT_6 ? hashTypeSha256 : hashTypeSha1); +} diff --git a/src/common/format.h b/src/common/format.h new file mode 100644 index 0000000000..385c15bbff --- /dev/null +++ b/src/common/format.h @@ -0,0 +1,37 @@ +/*********************************************************************************************************************************** +Repository Format + +Defines format for info and manifest files as well as on-disk structure. Each info file and manifest stores the format it was +written with, so a repository may contain more than one format while older backups and archives expire. + +A constant is defined for each format so that code which varies by format can be explicit about the format it applies to. +REPOSITORY_FORMAT_MIN/MAX are the range that can be read. The allow list for repo-format in build/config.yaml must be kept in sync +with MIN/MAX and its default is the format used for new repositories. +***********************************************************************************************************************************/ +#ifndef COMMON_FORMAT_H +#define COMMON_FORMAT_H + +#include "common/crypto/common.h" + +/*********************************************************************************************************************************** +Format numbers +***********************************************************************************************************************************/ +#define REPOSITORY_FORMAT_5 5 +#define REPOSITORY_FORMAT_6 6 + +#define REPOSITORY_FORMAT_MIN REPOSITORY_FORMAT_5 +#define REPOSITORY_FORMAT_MAX REPOSITORY_FORMAT_6 + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Error when a format cannot be read by this version. The format in a cipher header is checked before anything is decrypted, since +// decrypting requires knowing what the format expects and this version does not know what a newer format expects. +FN_EXTERN void repoFormatValidate(unsigned int format); + +// Digest a pass stored in a file at this format derives the key with. SHA-1 is what every repository used before format 6 and is +// kept for those, so a repository that has not been migrated is read and written exactly as it was. A pass is generated with the +// digest of the file it will be stored in, since that is what a reader will derive it with. +FN_EXTERN HashType repoFormatDigest(unsigned int format); + +#endif diff --git a/src/info/info.c b/src/info/info.c index cd407b28bc..b34227de49 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -11,6 +11,7 @@ Info Handler #include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" +#include "common/format.h" #include "common/ini.h" #include "common/io/bufferRead.h" #include "common/io/bufferWrite.h" @@ -122,50 +123,6 @@ infoNew(const unsigned int format, const CipherSpec *const cipherSpecSub) FUNCTION_LOG_RETURN(INFO, this); } -// Error when the format cannot be read by this version. Called for the format in the header before anything is decrypted and again -// for the format in the content, since the two are written together but stored apart. -static void -infoFormatValidate(const uint64_t format) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(UINT64, format); - FUNCTION_TEST_END(); - - // A format newer than this version can read requires an upgrade. Do not suggest a version since this version cannot know which - // version added the format. - if (format > REPOSITORY_FORMAT_MAX) - { - THROW_FMT( - FormatError, - "repository format %" PRIu64 " requires a newer version of " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } - - // A format older than this version can read requires an older version to migrate the repository - if (format < REPOSITORY_FORMAT_MIN) - { - THROW_FMT( - FormatError, - "repository format %" PRIu64 " is no longer supported by " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } - - FUNCTION_TEST_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -FN_EXTERN HashType -infoFormatDigest(const unsigned int format) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(UINT, format); - FUNCTION_TEST_END(); - - FUNCTION_TEST_RETURN(STRING_ID, format >= REPOSITORY_FORMAT_6 ? hashTypeSha256 : hashTypeSha1); -} - /**********************************************************************************************************************************/ #define INFO_SECTION_BACKREST "backrest" #define INFO_KEY_CHECKSUM "backrest-checksum" @@ -245,10 +202,10 @@ infoNewLoad( // Validate and store format if (strEqZ(value->key, INFO_KEY_FORMAT)) { - const uint64_t format = varUInt64(jsonToVar(value->value)); - infoFormatValidate(format); + const unsigned int format = jsonReadUInt(jsonReadNew(value->value)); + repoFormatValidate(format); - this->pub.format = (unsigned int)format; + this->pub.format = format; } // Store pgBackRest version else if (strEqZ(value->key, INFO_KEY_VERSION)) @@ -282,7 +239,7 @@ infoNewLoad( // since the sections come out in order and backrest sorts before cipher. this->pub.cipherSpec = cipherSpecNewP( cipherSpecType(cipherSpec), BUFSTR(varStr(jsonToVar(value->value))), - .digest = infoFormatDigest(this->pub.format)); + .digest = repoFormatDigest(this->pub.format)); } MEM_CONTEXT_OBJ_END(); } diff --git a/src/info/info.h b/src/info/info.h index cd147d9d36..6790ac47d3 100644 --- a/src/info/info.h +++ b/src/info/info.h @@ -118,11 +118,6 @@ Helper functions // Load info file(s) and throw error for each attempt if none are successful FN_EXTERN void infoLoad(const String *error, InfoLoadCallback *callbackFunction, void *callbackData); -// Digest a pass stored in a file at this format derives the key with. SHA-1 is what every repository used before format 6 and is -// kept for those, so a repository that has not been migrated is read and written exactly as it was. A pass is generated with the -// digest of the file it will be stored in, since that is what a reader will derive it with. -FN_EXTERN HashType infoFormatDigest(unsigned int format); - /*********************************************************************************************************************************** Macros for function logging ***********************************************************************************************************************************/ diff --git a/src/info/manifest/serialize.c.inc b/src/info/manifest/serialize.c.inc index 97b19469a6..02c0a58f31 100644 --- a/src/info/manifest/serialize.c.inc +++ b/src/info/manifest/serialize.c.inc @@ -1037,8 +1037,7 @@ manifestLoadFileCallback(void *const data, const unsigned int try) // Construct filename based on try const String *const fileName = try == 0 ? loadData->fileName : strNewFmt("%s" INFO_COPY_EXT, strZ(loadData->fileName)); - // Attempt to load the file. Decryption is added during the load so that everything an info file is read with is in one - // place, even though a manifest carries no header. + // Attempt to load the file IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); MEM_CONTEXT_BEGIN(loadData->memContext) diff --git a/src/meson.build b/src/meson.build index 67793b0d86..ff92b09944 100644 --- a/src/meson.build +++ b/src/meson.build @@ -126,6 +126,7 @@ src_pgbackrest = [ 'common/error/retry.c', 'common/exec.c', 'common/fork.c', + 'common/format.c', 'common/ini.c', 'common/io/bufferRead.c', 'common/io/bufferWrite.c', diff --git a/src/version.h b/src/version.h index d1e078206e..eaa579b5ff 100644 --- a/src/version.h +++ b/src/version.h @@ -24,20 +24,6 @@ Config include path name. The parent path will vary based on configuration. ***********************************************************************************************************************************/ #define PROJECT_CONFIG_INCLUDE_PATH "conf.d" -/*********************************************************************************************************************************** -Format Number -- defines format for info and manifest files as well as on-disk structure. Each info file and manifest stores the -format it was written with, so a repository may contain more than one format while older backups and archives expire. - -A constant is defined for each format so that code which varies by format can be explicit about the format it applies to. -REPOSITORY_FORMAT_MIN/MAX are the range that can be read. The allow list for repo-format in build/config.yaml must be kept in sync -with MIN/MAX and its default is the format used for new repositories. -***********************************************************************************************************************************/ -#define REPOSITORY_FORMAT_5 5 -#define REPOSITORY_FORMAT_6 6 - -#define REPOSITORY_FORMAT_MIN REPOSITORY_FORMAT_5 -#define REPOSITORY_FORMAT_MAX REPOSITORY_FORMAT_6 - /*********************************************************************************************************************************** Project version components. PROJECT_VERSION and PROJECT_VERSION_NUM are automatically generated from the component parts. ***********************************************************************************************************************************/ diff --git a/test/define.yaml b/test/define.yaml index 36ea93f94a..1b0777d710 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -358,6 +358,13 @@ unit: - storage/storage - storage/write + # -------------------------------------------------------------------------------------------------------------------------------- + - name: common/format + total: 2 + + coverage: + - common/format + # -------------------------------------------------------------------------------------------------------------------------------- - name: common/crypto total: 4 diff --git a/test/src/harness/info.c b/test/src/harness/info.c index 9f56b5910c..c92bdab4db 100644 --- a/test/src/harness/info.c +++ b/test/src/harness/info.c @@ -8,6 +8,7 @@ Harness for Loading Test Configurations #include "common/assert.h" #include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" +#include "common/format.h" #include "common/io/bufferRead.h" #include "common/io/bufferWrite.h" #include "common/io/filter/filter.h" diff --git a/test/src/harness/info.h b/test/src/harness/info.h index 7682bff1ea..541732676c 100644 --- a/test/src/harness/info.h +++ b/test/src/harness/info.h @@ -1,6 +1,7 @@ /*********************************************************************************************************************************** Harness for Generating Test Info Files ***********************************************************************************************************************************/ +#include "common/format.h" #include "common/type/buffer.h" #include "info/info.h" diff --git a/test/src/module/common/cryptoTest.c b/test/src/module/common/cryptoTest.c index 1879cde127..44141da655 100644 --- a/test/src/module/common/cryptoTest.c +++ b/test/src/module/common/cryptoTest.c @@ -6,6 +6,7 @@ Test Block Cipher #include "common/io/filter/filter.h" #include "common/io/io.h" #include "common/type/json.h" +#include "version.h" /*********************************************************************************************************************************** Data for testing diff --git a/test/src/module/common/formatTest.c b/test/src/module/common/formatTest.c new file mode 100644 index 0000000000..059dadba71 --- /dev/null +++ b/test/src/module/common/formatTest.c @@ -0,0 +1,38 @@ +/*********************************************************************************************************************************** +Test Repository Format +***********************************************************************************************************************************/ +#include "version.h" + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +static void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + // ***************************************************************************************************************************** + if (testBegin("repoFormatValidate()")) + { + TEST_ERROR( + repoFormatValidate(REPOSITORY_FORMAT_MIN - 1), FormatError, + "repository format 4 is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + TEST_ERROR( + repoFormatValidate(REPOSITORY_FORMAT_MAX + 1), FormatError, + "repository format 7 requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + + TEST_RESULT_VOID(repoFormatValidate(REPOSITORY_FORMAT_5), "format 5 is readable"); + TEST_RESULT_VOID(repoFormatValidate(REPOSITORY_FORMAT_6), "format 6 is readable"); + } + + // ***************************************************************************************************************************** + if (testBegin("repoFormatDigest()")) + { + TEST_RESULT_UINT(repoFormatDigest(REPOSITORY_FORMAT_5), hashTypeSha1, "format 5 derives with sha1"); + TEST_RESULT_UINT(repoFormatDigest(REPOSITORY_FORMAT_6), hashTypeSha256, "format 6 derives with sha256"); + } + + FUNCTION_HARNESS_RETURN_VOID(); +} From 02efae9337140f65b3b95de9a435d692b39cd1f6 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 11:33:20 +0700 Subject: [PATCH 06/15] Comment fixes. --- build/config.yaml | 2 +- src/command/stanza/upgrade.c | 4 ++-- src/common/crypto/cipherBlock.c | 14 +++++++------- src/common/crypto/spec.h | 2 +- src/config/config.h | 2 +- src/info/info.c | 10 +++++----- src/info/info.h | 2 +- src/info/infoPg.c | 2 +- test/src/harness/info.c | 2 +- test/src/harness/info.h | 4 ++-- test/src/harness/storage.c | 2 +- test/src/module/command/infoTest.c | 2 +- test/src/module/command/stanzaTest.c | 2 +- test/src/module/common/cryptoTest.c | 6 +++--- test/src/module/info/infoTest.c | 4 ++-- 15 files changed, 30 insertions(+), 30 deletions(-) diff --git a/build/config.yaml b/build/config.yaml index 66a442d5f0..30f635d07e 100644 --- a/build/config.yaml +++ b/build/config.yaml @@ -1690,7 +1690,7 @@ option: # Command-line only so that a format left in a configuration file cannot upgrade a repository as a side effect of an unrelated # stanza-upgrade. This does not cover the environment, which can set any option that is valid for the command, so a format left - # there does migrate on the next stanza-upgrade and the option reference says so. The allow list must be kept in sync with + # there does migrate on the next stanza-upgrade and the option reference documents it. The allow list must be kept in sync with # REPOSITORY_FORMAT_MIN/MAX in common/format.h repo-format: group: repo diff --git a/src/command/stanza/upgrade.c b/src/command/stanza/upgrade.c index 0db843e7e7..21d7a81d38 100644 --- a/src/command/stanza/upgrade.c +++ b/src/command/stanza/upgrade.c @@ -104,13 +104,13 @@ cmdStanzaUpgrade(void) if (format != formatArchive || format != formatBackup) { // Report a repository that was found at two formats. It is repaired here but a prior upgrade did not finish, which - // the user has not been told about since the run it happened on did not get far enough to say so. + // the user has not been told about since the run it happened on did not get far enough to report it. if (formatArchive != formatBackup) LOG_WARN("repository format mismatch from an interrupted " CFGCMD_STANZA_UPGRADE " will be repaired"); // Log the format the repository is migrating from, which is the lower of the two when an interrupted upgrade left // them at different formats. This cannot be undone and a version that does not support the new format will no - // longer be able to read the stanza, so say so rather than migrating silently. + // longer be able to read the stanza, so report it rather than migrating silently. LOG_INFO_FMT( "upgrade repository format from %u to %u", formatArchive < formatBackup ? formatArchive : formatBackup, format); diff --git a/src/common/crypto/cipherBlock.c b/src/common/crypto/cipherBlock.c index 1ddf856a5d..8d0c8ef84d 100644 --- a/src/common/crypto/cipherBlock.c +++ b/src/common/crypto/cipherBlock.c @@ -491,7 +491,7 @@ cipherBlockInputSame(const THIS_VOID) } /*********************************************************************************************************************************** -Report the format the header gave +Report the format the header contained ***********************************************************************************************************************************/ static Pack * cipherBlockResult(THIS_VOID) @@ -549,11 +549,11 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); ASSERT(cipherSpecPass(cipherSpec) != NULL && !bufEmpty(cipherSpecPass(cipherSpec))); - // The header takes the place of the magic, so a file that carries one is never also raw + // The header takes the place of the magic, so a file that contains one is never also raw ASSERT(!param.header || !param.raw); - // The format must be known to write a header. On decrypt it is optional since the header is what says which format it is, but - // when it is given the header must agree with it. + // The format must be known to write a header. On decrypt it is optional since the header defines the format, but when it is + // given the header must agree with it. ASSERT(mode != cipherModeEncrypt || !param.header || param.format != 0); // Init crypto subsystem @@ -569,13 +569,13 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const zFree(cipherTypeZ); - // Lookup digest. A header that has yet to be read is what says which format the file is, and the format is what says which - // digest, so in that case the lookup waits until the header has been read. + // Lookup digest. A header that has yet to be read is what defines the format of the file, and the format defines the digest, so + // in that case the lookup waits until the header has been read. const EVP_MD *digest = NULL; if (!param.header || mode == cipherModeEncrypt) { - // A format says which digest, otherwise it comes from the spec + // A format defines the digest, otherwise it comes from the spec digest = cipherBlockDigest(param.format != 0 ? repoFormatDigest(param.format) : cipherSpecDigest(cipherSpec)); } diff --git a/src/common/crypto/spec.h b/src/common/crypto/spec.h index 782db77db0..919503b8da 100644 --- a/src/common/crypto/spec.h +++ b/src/common/crypto/spec.h @@ -2,7 +2,7 @@ Cipher Spec Everything needed to encrypt or decrypt, kept together so that adding to it does not mean changing every function and protocol -message that carries it. +message that contains it. The pass holds the bytes used to derive the key. The digest travels with the pass because the two are chosen together and deriving with the wrong digest gives a wrong key instead of an error. diff --git a/src/config/config.h b/src/config/config.h index 7d65723aa2..65c9e5f570 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -243,7 +243,7 @@ FN_EXTERN bool cfgOptionIdxTest(ConfigOption optionId, unsigned int optionIdx); Cipher Functions Main cipher spec for a repository, built from the cipher options and cached per repo since the options cannot change while a command -runs. This is the top of the chain. The info files are read with it and each of those carries the cipher spec for the files that +runs. This is the top of the chain. The info files are read with it and each of those contains the cipher spec for the files that depend on it, so nothing further down needs to consult the options again. ***********************************************************************************************************************************/ FN_EXTERN const CipherSpec *cfgCipherSpecMainIdx(unsigned int repoIdx); diff --git a/src/info/info.c b/src/info/info.c index b34227de49..c5fa15aedf 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -163,7 +163,7 @@ infoNewLoad( TRY_BEGIN() { - // The content is decrypted as it is parsed. A file that may carry a header is read with one, which the cipher + // The content is decrypted as it is parsed. A file that may contain a header is read with one, which the cipher // consumes and reports the format of once the read is done. if (cipherSpecType(cipherSpec) != cipherTypeNone) { @@ -281,9 +281,9 @@ infoNewLoad( if (infoFormat(this) == 0) THROW(FormatError, "repository format not found\nHINT: is this a valid " PROJECT_NAME " info file?"); - // Only a cipher that read a header reports a format, so a result here is what says the file had one. The header is - // written from the same format as the content, so a file where they disagree has been damaged or put together from - // parts of two files. + // Only a cipher that read a header reports a format, so a result here means the file had one. The header is written + // from the same format as the content, so a file where they disagree has been damaged or put together from parts of two + // files. PackRead *const cipherResult = ioFilterGroupResultP(ioReadFilterGroup(read), CIPHER_BLOCK_FILTER_TYPE); if (cipherResult != NULL) @@ -327,7 +327,7 @@ infoWriteNew(Buffer *const buffer, const unsigned int format, const CipherSpec * IoWrite *const result = ioBufferWriteNew(buffer); // The cipher writes the header and derives the pass with the digest the format calls for. Format 5 gets no header since it is - // the format a reader assumes when there is nothing to say otherwise. + // the format a reader assumes when no header defines one. if (cipherSpecType(cipherSpec) != cipherTypeNone) { ioFilterGroupAdd( diff --git a/src/info/info.h b/src/info/info.h index 6790ac47d3..91104735e5 100644 --- a/src/info/info.h +++ b/src/info/info.h @@ -41,7 +41,7 @@ Constructors ***********************************************************************************************************************************/ FN_EXTERN Info *infoNew(unsigned int format, const CipherSpec *cipherSpecSub); -// Create new object and load contents from a file. Decryption is added here rather than by the caller because a file that carries +// Create new object and load contents from a file. Decryption is added here rather than by the caller because a file that contains // a header cannot be decrypted until the header has been read. The cipher spec supplies the type and pass; the digest comes from // the format the file turns out to be at. The same spec supplies the type for the cipher spec built from the pass stored in the // file. diff --git a/src/info/infoPg.c b/src/info/infoPg.c index 840b00c6e1..ea1124c173 100644 --- a/src/info/infoPg.c +++ b/src/info/infoPg.c @@ -189,7 +189,7 @@ infoPgNewLoad( .infoPg = this, }; - // The archive.info and backup.info files are the only users of this object and both carry a header + // The archive.info and backup.info files are the only users of this object and both contain a header this->pub.info = infoNewLoadP(read, cipherSpec, infoPgLoadCallback, &loadData, .header = true); CHECK(FormatError, !lstEmpty(this->pub.history), "history is missing"); diff --git a/test/src/harness/info.c b/test/src/harness/info.c index c92bdab4db..dd44041e75 100644 --- a/test/src/harness/info.c +++ b/test/src/harness/info.c @@ -142,7 +142,7 @@ hrnInfoPut(const Storage *const storage, const char *const file, const char *con const Buffer *content = harnessInfoChecksumFormat(param.format, STR(info)); - // Encrypt the way the format stores the file. A file that carries a header gets it in place of the magic the cipher writes, + // Encrypt the way the format stores the file. A file that contains a header gets it in place of the magic the cipher writes, // and from format 6 the pass derives with SHA-256 rather than SHA-1. if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) { diff --git a/test/src/harness/info.h b/test/src/harness/info.h index 541732676c..c1470c5722 100644 --- a/test/src/harness/info.h +++ b/test/src/harness/info.h @@ -20,7 +20,7 @@ typedef struct HrnInfoPutParam { VAR_PARAM_HEADER; unsigned int format; // Repository format, default format when zero - bool header; // Does the file carry a header, i.e. is it an info file? + bool header; // Does the file contain a header, i.e. is it an info file? const CipherSpec *cipherSpec; // Cipher spec when the file is encrypted, digest set by format const char *comment; // Comment } HrnInfoPutParam; @@ -42,7 +42,7 @@ Buffer *harnessInfoChecksum(const String *info); Buffer *harnessInfoChecksumFormat(unsigned int format, const String *info); Buffer *harnessInfoChecksumZ(const char *info); -// Encrypt content the way a file that carries no header is stored, e.g. a manifest +// Encrypt content the way a file that contains no header is stored, e.g. a manifest Buffer *harnessInfoEncrypt(const Buffer *content, const CipherSpec *cipherSpec); void harnessInfoLoadNewCallback(void *callbackData, const String *section, const String *key, JsonRead *json); diff --git a/test/src/harness/storage.c b/test/src/harness/storage.c index e096315207..f276cbc4ad 100644 --- a/test/src/harness/storage.c +++ b/test/src/harness/storage.c @@ -70,7 +70,7 @@ testStorageGet(const Storage *const storage, const char *const file, const char if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) { // Derive with SHA-1 since the harness reads and writes files the way a repository at the format these tests build stores - // them, which is the format that had no header to say anything else + // them, which is the format that had no header to define anything else ioFilterGroupAdd( filterGroup, cipherBlockNewP( diff --git a/test/src/module/command/infoTest.c b/test/src/module/command/infoTest.c index da25dcef42..369c50be6f 100644 --- a/test/src/module/command/infoTest.c +++ b/test/src/module/command/infoTest.c @@ -502,7 +502,7 @@ testRun(void) STORAGE_REPO_ARCHIVE "/9.5-2/0000000100000000/000000010000000000000001-ac61b8f1ec7b1e6c3eaee9345214595eb7daa9a1.gz", .comment = "write WAL db2 timeline 1 repo1"); - // The repository was migrated to format 6 after the first backup, so each backup carries the format it was written + // The repository was migrated to format 6 after the first backup, so each backup contains the format it was written // with rather than the format of the info file const Buffer *const backupInfoContent = harnessInfoChecksumFormat( REPOSITORY_FORMAT_6, diff --git a/test/src/module/command/stanzaTest.c b/test/src/module/command/stanzaTest.c index ba7bc1a6c7..132c2b24a5 100644 --- a/test/src/module/command/stanzaTest.c +++ b/test/src/module/command/stanzaTest.c @@ -1194,7 +1194,7 @@ testRun(void) TEST_TITLE("stanza-upgrade - every format that can be read can be requested"); // The allow list for repo-format in build/config.yaml and REPOSITORY_FORMAT_MIN/MAX in version.h are declared separately, - // so make sure they say the same thing + // so make sure they match for (unsigned int format = REPOSITORY_FORMAT_MIN; format <= REPOSITORY_FORMAT_MAX; format++) { argList = strLstDup(argListBase); diff --git a/test/src/module/common/cryptoTest.c b/test/src/module/common/cryptoTest.c index 44141da655..a27711f08a 100644 --- a/test/src/module/common/cryptoTest.c +++ b/test/src/module/common/cryptoTest.c @@ -85,7 +85,7 @@ testRun(void) // Initialization of object // ------------------------------------------------------------------------------------------------------------------------- - // Build from a duplicate to show the copy carries the type, digest, and pass of the original + // Build from a duplicate to show the copy contains the type, digest, and pass of the original TEST_RESULT_UINT(cipherSpecType(cipherSpecDup(cipherSpecNewNone())), cipherTypeNone, "dup of none"); const CipherSpec *const cipherSpec = cipherSpecDup(cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS))); @@ -95,7 +95,7 @@ testRun(void) cipherSpecDigest(cipherSpecDup(cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS), .digest = hashTypeSha1))), hashTypeSha1, "dup digest"); - // A pack carries nothing but the type when there is no cipher + // A pack contains nothing but the type when there is no cipher PackWrite *packWrite = pckWriteNewP(); cipherSpecPack(packWrite, cipherSpecNewNone()); @@ -104,7 +104,7 @@ testRun(void) TEST_RESULT_UINT( cipherSpecType(cipherSpecNewPack(pckReadNew(pckWriteResult(packWrite)))), cipherTypeNone, "unpack none"); - // Else it carries the type, digest, and pass. Pack a digest that is not the default so that a pack which loses the digest + // Else it contains the type, digest, and pass. Pack a digest that is not the default so that a pack which loses the digest // cannot pass by falling back to the default. packWrite = pckWriteNewP(); diff --git a/test/src/module/info/infoTest.c b/test/src/module/info/infoTest.c index d7f8ea33e3..25fc7418f8 100644 --- a/test/src/module/info/infoTest.c +++ b/test/src/module/info/infoTest.c @@ -273,8 +273,8 @@ testRun(void) const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); - // A file with no header, e.g. a manifest, is decrypted with the spec as it was given since there is nothing to say the - // digest should be anything else + // A file with no header, e.g. a manifest, is decrypted with the spec as it was given since nothing defines the digest + // as anything else TEST_ASSIGN( info, infoNewLoadP( From 2a2288457161751ef3df7968df0d823d5882ff6c Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 11:38:05 +0700 Subject: [PATCH 07/15] Awkward phrasing. --- src/command/stanza/upgrade.c | 4 ++-- src/common/crypto/spec.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/command/stanza/upgrade.c b/src/command/stanza/upgrade.c index 0db843e7e7..21d7a81d38 100644 --- a/src/command/stanza/upgrade.c +++ b/src/command/stanza/upgrade.c @@ -104,13 +104,13 @@ cmdStanzaUpgrade(void) if (format != formatArchive || format != formatBackup) { // Report a repository that was found at two formats. It is repaired here but a prior upgrade did not finish, which - // the user has not been told about since the run it happened on did not get far enough to say so. + // the user has not been told about since the run it happened on did not get far enough to report it. if (formatArchive != formatBackup) LOG_WARN("repository format mismatch from an interrupted " CFGCMD_STANZA_UPGRADE " will be repaired"); // Log the format the repository is migrating from, which is the lower of the two when an interrupted upgrade left // them at different formats. This cannot be undone and a version that does not support the new format will no - // longer be able to read the stanza, so say so rather than migrating silently. + // longer be able to read the stanza, so report it rather than migrating silently. LOG_INFO_FMT( "upgrade repository format from %u to %u", formatArchive < formatBackup ? formatArchive : formatBackup, format); diff --git a/src/common/crypto/spec.h b/src/common/crypto/spec.h index 003c9ca2ba..00a61a8080 100644 --- a/src/common/crypto/spec.h +++ b/src/common/crypto/spec.h @@ -2,7 +2,7 @@ Cipher Spec Everything needed to encrypt or decrypt, kept together so that adding to it does not mean changing every function and protocol -message that carries it. +message that contains it. The passphrase (pass) contains the bytes the key is derived from. There is no pass when the type is none, and the pass is never logged. From 88b0fd14c57c9b9f8def8608c9ff69471f869c96 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 11:45:35 +0700 Subject: [PATCH 08/15] Comment. --- src/common/format.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/format.h b/src/common/format.h index 385c15bbff..23a1647da6 100644 --- a/src/common/format.h +++ b/src/common/format.h @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Repository Format -Defines format for info and manifest files as well as on-disk structure. Each info file and manifest stores the format it was +Defines the format for info and manifest files as well as on-disk structure. Each info file and manifest stores the format it was written with, so a repository may contain more than one format while older backups and archives expire. A constant is defined for each format so that code which varies by format can be explicit about the format it applies to. From cadcc85f88f066893aca600de24a7576c93fbef4 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 11:48:27 +0700 Subject: [PATCH 09/15] Fix. --- src/config/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config.h b/src/config/config.h index 7d65723aa2..65c9e5f570 100644 --- a/src/config/config.h +++ b/src/config/config.h @@ -243,7 +243,7 @@ FN_EXTERN bool cfgOptionIdxTest(ConfigOption optionId, unsigned int optionIdx); Cipher Functions Main cipher spec for a repository, built from the cipher options and cached per repo since the options cannot change while a command -runs. This is the top of the chain. The info files are read with it and each of those carries the cipher spec for the files that +runs. This is the top of the chain. The info files are read with it and each of those contains the cipher spec for the files that depend on it, so nothing further down needs to consult the options again. ***********************************************************************************************************************************/ FN_EXTERN const CipherSpec *cfgCipherSpecMainIdx(unsigned int repoIdx); From a1a7bb7a955f2d73070ddcbd23035ecd7a450ac6 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 12:00:17 +0700 Subject: [PATCH 10/15] Move format to cipher filter. --- src/command/backup/complete.c.inc | 10 ++++----- src/command/restore/timeline.c | 2 +- src/command/verify/verify.c | 2 +- src/common/crypto/cipherBlock.c | 20 +++++++++-------- src/common/crypto/cipherBlock.h | 19 ++++++++++++---- src/info/info.c | 31 -------------------------- src/info/info.h | 5 ----- src/info/infoArchive.c | 4 +++- src/info/infoBackup.c | 4 +++- test/src/harness/backup.c | 2 +- test/src/module/command/restoreTest.c | 2 +- test/src/module/common/cryptoTest.c | 12 ++++------ test/src/module/info/infoArchiveTest.c | 7 ++++-- test/src/module/info/infoBackupTest.c | 6 +++-- test/src/module/info/infoTest.c | 12 +++++----- 15 files changed, 61 insertions(+), 77 deletions(-) diff --git a/src/command/backup/complete.c.inc b/src/command/backup/complete.c.inc index 777f7e26ca..be1617a7f6 100644 --- a/src/command/backup/complete.c.inc +++ b/src/command/backup/complete.c.inc @@ -25,7 +25,7 @@ backupManifestSaveCopy(Manifest *const manifest, const CipherSpec *const cipherS STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(manifestData(manifest)->backupLabel)))); // Add encryption filter if required - cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpecManifest); + cipherBlockFilterGroupAddP(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpecManifest); // Save file manifestSave(manifest, write); @@ -101,7 +101,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif IoFilterGroup *const filterGroup = ioReadFilterGroup(storageReadIo(read)); // Decrypt with archive key if encrypted - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( filterGroup, cipherModeDecrypt, infoArchiveCipherSpec(backupData->archiveInfo)); // Compress/decompress if archive and backup do not have the same compression settings @@ -118,7 +118,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif } // Encrypt with backup key if encrypted - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( filterGroup, cipherModeEncrypt, manifestCipherSpec(manifest)); // Add size filter last to calculate repo size @@ -201,7 +201,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest) StorageRead *const manifestRead = storageNewReadP( storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupLabel))); - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( ioReadFilterGroup(storageReadIo(manifestRead)), cipherModeDecrypt, infoBackupCipherSpec(infoBackup)); StorageWrite *const manifestWrite = storageNewWriteP( @@ -212,7 +212,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest) ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), compressFilterP(compressTypeGz, 9)); - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( ioWriteFilterGroup(storageWriteIo(manifestWrite)), cipherModeEncrypt, infoBackupCipherSpec(infoBackup)); storageCopyP(manifestRead, manifestWrite); diff --git a/src/command/restore/timeline.c b/src/command/restore/timeline.c index dd7f71289a..584c0604e4 100644 --- a/src/command/restore/timeline.c +++ b/src/command/restore/timeline.c @@ -94,7 +94,7 @@ historyLoad( { const String *const historyFile = strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%08X.history", strZ(archiveId), timeline); StorageRead *const storageRead = storageNewReadP(storageRepo, historyFile); - cipherBlockFilterGroupAdd(ioReadFilterGroup(storageReadIo(storageRead)), cipherModeDecrypt, cipherSpecArchive); + cipherBlockFilterGroupAddP(ioReadFilterGroup(storageReadIo(storageRead)), cipherModeDecrypt, cipherSpecArchive); const Buffer *const history = storageGetP(storageRead); TRY_BEGIN() diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c index e158d7b47f..a6d1472425 100644 --- a/src/command/verify/verify.c +++ b/src/command/verify/verify.c @@ -187,7 +187,7 @@ verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherS // checksum is over the file as it is stored. The file and its copy are written from the same bytes, which is all the checksum // is used to compare. if (decrypt) - cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec); + cipherBlockFilterGroupAddP(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec); ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1)); diff --git a/src/common/crypto/cipherBlock.c b/src/common/crypto/cipherBlock.c index 8d0c8ef84d..feea2d8a67 100644 --- a/src/common/crypto/cipherBlock.c +++ b/src/common/crypto/cipherBlock.c @@ -550,11 +550,10 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const ASSERT(cipherSpecPass(cipherSpec) != NULL && !bufEmpty(cipherSpecPass(cipherSpec))); // The header takes the place of the magic, so a file that contains one is never also raw - ASSERT(!param.header || !param.raw); + ASSERT(!param.raw || (!param.header && param.format == 0)); - // The format must be known to write a header. On decrypt it is optional since the header defines the format, but when it is - // given the header must agree with it. - ASSERT(mode != cipherModeEncrypt || !param.header || param.format != 0); + // On encrypt the format defines whether a header is written, so a header is only ever requested on decrypt + ASSERT(mode == cipherModeDecrypt || !param.header); // Init crypto subsystem cryptoInit(); @@ -573,7 +572,7 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const // in that case the lookup waits until the header has been read. const EVP_MD *digest = NULL; - if (!param.header || mode == cipherModeEncrypt) + if (!param.header) { // A format defines the digest, otherwise it comes from the spec digest = cipherBlockDigest(param.format != 0 ? repoFormatDigest(param.format) : cipherSpecDigest(cipherSpec)); @@ -585,7 +584,7 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const { .mode = mode, .raw = param.raw, - .headerFormat = param.header, + .headerFormat = mode == cipherModeEncrypt ? param.format >= REPOSITORY_FORMAT_6 : param.header, .format = param.format, .cipher = cipher, .digest = digest, @@ -619,7 +618,7 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const .inputSame = cipherBlockInputSame, // Only a filter that reads a header has a format to report - .result = param.header && mode == cipherModeDecrypt ? cipherBlockResult : NULL)); + .result = param.header ? cipherBlockResult : NULL)); } FN_EXTERN IoFilter * @@ -646,19 +645,22 @@ cipherBlockNewPack(const Pack *const paramList) /**********************************************************************************************************************************/ FN_EXTERN IoFilterGroup * -cipherBlockFilterGroupAdd(IoFilterGroup *const filterGroup, const CipherMode mode, const CipherSpec *const cipherSpec) +cipherBlockFilterGroupAdd( + IoFilterGroup *const filterGroup, const CipherMode mode, const CipherSpec *const cipherSpec, + const CipherBlockFilterGroupAddParam param) { FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(IO_FILTER_GROUP, filterGroup); FUNCTION_LOG_PARAM(STRING_ID, mode); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_LOG_PARAM(UINT, param.format); FUNCTION_LOG_END(); ASSERT(filterGroup != NULL); ASSERT(cipherSpec != NULL); if (cipherSpecType(cipherSpec) != cipherTypeNone) - ioFilterGroupAdd(filterGroup, cipherBlockNewP(mode, cipherSpec)); + ioFilterGroupAdd(filterGroup, cipherBlockNewP(mode, cipherSpec, .format = param.format)); FUNCTION_LOG_RETURN(IO_FILTER_GROUP, filterGroup); } diff --git a/src/common/crypto/cipherBlock.h b/src/common/crypto/cipherBlock.h index 360b5de796..f98b1984f9 100644 --- a/src/common/crypto/cipherBlock.h +++ b/src/common/crypto/cipherBlock.h @@ -19,8 +19,8 @@ typedef struct CipherBlockNewParam { VAR_PARAM_HEADER; bool raw; // Omit header magic to save space - bool header; // Read/write the format header - unsigned int format; // Repository format, required to write a header + bool header; // Read the format header, decrypt only + unsigned int format; // Repository format, which on encrypt defines the header } CipherBlockNewParam; #define cipherBlockNewP(mode, cipherSpec, ...) \ @@ -39,7 +39,18 @@ FN_EXTERN unsigned int cipherBlockFormat(PackRead *cipherBlockResult); /*********************************************************************************************************************************** Helper functions ***********************************************************************************************************************************/ -// Add a block cipher to an io object. Nothing is added when the repository is not encrypted. -FN_EXTERN IoFilterGroup *cipherBlockFilterGroupAdd(IoFilterGroup *filterGroup, CipherMode mode, const CipherSpec *cipherSpec); +// Add a block cipher to an io object. Nothing is added when the repository is not encrypted. The format is required when a file is +// written at one, since the format defines the digest and whether a header is written. +typedef struct CipherBlockFilterGroupAddParam +{ + VAR_PARAM_HEADER; + unsigned int format; // Repository format the file is written at +} CipherBlockFilterGroupAddParam; + +#define cipherBlockFilterGroupAddP(filterGroup, mode, cipherSpec, ...) \ + cipherBlockFilterGroupAdd(filterGroup, mode, cipherSpec, (CipherBlockFilterGroupAddParam){VAR_PARAM_INIT, __VA_ARGS__}) + +FN_EXTERN IoFilterGroup *cipherBlockFilterGroupAdd( + IoFilterGroup *filterGroup, CipherMode mode, const CipherSpec *cipherSpec, CipherBlockFilterGroupAddParam param); #endif diff --git a/src/info/info.c b/src/info/info.c index c5fa15aedf..69c443f3e3 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -308,37 +308,6 @@ infoNewLoad( FUNCTION_LOG_RETURN(INFO, this); } -/**********************************************************************************************************************************/ -FN_EXTERN IoWrite * -infoWriteNew(Buffer *const buffer, const unsigned int format, const CipherSpec *const cipherSpec) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(BUFFER, buffer); - FUNCTION_LOG_PARAM(UINT, format); - FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); - FUNCTION_LOG_END(); - - FUNCTION_AUDIT_HELPER(); - - ASSERT(buffer != NULL); - ASSERT(format >= REPOSITORY_FORMAT_MIN && format <= REPOSITORY_FORMAT_MAX); - ASSERT(cipherSpec != NULL); - - IoWrite *const result = ioBufferWriteNew(buffer); - - // The cipher writes the header and derives the pass with the digest the format calls for. Format 5 gets no header since it is - // the format a reader assumes when no header defines one. - if (cipherSpecType(cipherSpec) != cipherTypeNone) - { - ioFilterGroupAdd( - ioWriteFilterGroup(result), - cipherBlockNewP( - cipherModeEncrypt, cipherSpec, .header = format >= REPOSITORY_FORMAT_6, .format = format)); - } - - FUNCTION_LOG_RETURN(IO_WRITE, result); -} - /**********************************************************************************************************************************/ FN_EXTERN bool infoSaveSection(InfoSave *const infoSaveData, const char *const section, const String *const sectionNext) diff --git a/src/info/info.h b/src/info/info.h index 91104735e5..c2e512d715 100644 --- a/src/info/info.h +++ b/src/info/info.h @@ -98,11 +98,6 @@ infoBackrestVersion(const Info *const this) /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ -// Create a write to save an info file into a buffer. The header is written and the encryption filter added according to the -// format, so the caller has only to save into the write it gets back. The write side is the one that knows the format, so unlike -// the load there is nothing to work out first. -FN_EXTERN IoWrite *infoWriteNew(Buffer *buffer, unsigned int format, const CipherSpec *cipherSpec); - // Save to file FN_EXTERN void infoSave(Info *this, IoWrite *write, InfoSaveCallback *callbackFunction, void *callbackData); diff --git a/src/info/infoArchive.c b/src/info/infoArchive.c index d6839154c3..0da2a23d21 100644 --- a/src/info/infoArchive.c +++ b/src/info/infoArchive.c @@ -337,7 +337,9 @@ infoArchiveSaveFile( { // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); - IoWrite *const write = infoWriteNew(buffer, infoArchiveFormat(infoArchive), cipherSpec); + IoWrite *const write = ioBufferWriteNew(buffer); + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = infoArchiveFormat(infoArchive)); infoArchiveSave(infoArchive, write); // Save the file and make a copy diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index c10ce3f843..1ac4c135cf 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -936,7 +936,9 @@ infoBackupSaveFile( { // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); - IoWrite *const write = infoWriteNew(buffer, infoBackupFormat(infoBackup), cipherSpec); + IoWrite *const write = ioBufferWriteNew(buffer); + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = infoBackupFormat(infoBackup)); infoBackupSave(infoBackup, write); // Save the file and make a copy diff --git a/test/src/harness/backup.c b/test/src/harness/backup.c index 2c442d795e..d83c6a3908 100644 --- a/test/src/harness/backup.c +++ b/test/src/harness/backup.c @@ -243,7 +243,7 @@ hrnBackupPqScript(const unsigned int pgVersion, const time_t backupTimeStart, Hr ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), compressFilterP(param.walCompressType, 1)); // Encrypt with the archive passphrase, which is what archive-push writes WAL with - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( ioWriteFilterGroup(storageWriteIo(write)), cipherModeEncrypt, infoArchiveCipherSpec(infoArchive)); storagePutP(write, walBuffer); diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index 8487a0a7ad..0a1b4a1812 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -2320,7 +2320,7 @@ testRun(void) // Add encryption filter and save the encrypted manifest #define TEST_CIPHER_PASS_MANIFEST "backpass" - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpecNewP( cfgOptionIdxStrId(cfgOptRepoCipherType, 1), BUFSTRDEF(TEST_CIPHER_PASS_MANIFEST), .digest = hashTypeSha1)); diff --git a/test/src/module/common/cryptoTest.c b/test/src/module/common/cryptoTest.c index a27711f08a..1706699431 100644 --- a/test/src/module/common/cryptoTest.c +++ b/test/src/module/common/cryptoTest.c @@ -126,9 +126,7 @@ testRun(void) ioFilterGroupAdd( ioWriteFilterGroup(headerWrite), - cipherBlockNewP( - cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true, - .format = REPOSITORY_FORMAT_6)); + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .format = REPOSITORY_FORMAT_6)); ioWriteOpen(headerWrite); ioWrite(headerWrite, testPlainText); ioWriteClose(headerWrite); @@ -189,9 +187,7 @@ testRun(void) ioFilterGroupAdd( ioWriteFilterGroup(headerWrite), - cipherBlockNewP( - cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true, - .format = REPOSITORY_FORMAT_6)); + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .format = REPOSITORY_FORMAT_6)); ioWriteOpen(headerWrite); ioWrite(headerWrite, testPlainText); ioWriteClose(headerWrite); @@ -480,12 +476,12 @@ testRun(void) IoFilterGroup *filterGroup = ioFilterGroupNew(); TEST_RESULT_PTR( - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( filterGroup, cipherModeEncrypt, cipherSpecNewNone()), filterGroup, " no filter add"); TEST_RESULT_UINT(ioFilterGroupSize(filterGroup), 0, " check no filter add"); TEST_RESULT_VOID( - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( filterGroup, cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("X"))), " filter add"); TEST_RESULT_UINT(ioFilterGroupSize(filterGroup), 1, " check filter add"); } diff --git a/test/src/module/info/infoArchiveTest.c b/test/src/module/info/infoArchiveTest.c index 6f826221c3..6a8d8f879a 100644 --- a/test/src/module/info/infoArchiveTest.c +++ b/test/src/module/info/infoArchiveTest.c @@ -94,8 +94,11 @@ testRun(void) contentSave = bufNew(0); - TEST_RESULT_VOID( - infoArchiveSave(info, infoWriteNew(contentSave, REPOSITORY_FORMAT_DEFAULT, cipherSpec)), "save new with cipher"); + IoWrite *write = ioBufferWriteNew(contentSave); + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = REPOSITORY_FORMAT_DEFAULT); + + TEST_RESULT_VOID(infoArchiveSave(info, write), "save new with cipher"); TEST_RESULT_BOOL( strBeginsWithZ(strNewBuf(contentSave), "PGBR"), false, "no header before the format that added it"); diff --git a/test/src/module/info/infoBackupTest.c b/test/src/module/info/infoBackupTest.c index 83886ebb17..908164c942 100644 --- a/test/src/module/info/infoBackupTest.c +++ b/test/src/module/info/infoBackupTest.c @@ -95,8 +95,10 @@ testRun(void) contentSave = bufNew(0); - TEST_RESULT_VOID( - infoBackupSave(infoBackup, infoWriteNew(contentSave, REPOSITORY_FORMAT_6, cipherSpec)), "save new with cipher sub"); + IoWrite *write = ioBufferWriteNew(contentSave); + cipherBlockFilterGroupAddP(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = REPOSITORY_FORMAT_6); + + TEST_RESULT_VOID(infoBackupSave(infoBackup, write), "save new with cipher sub"); TEST_RESULT_STR_Z( strNewZN((const char *)bufPtrConst(contentSave), 8), "PGBR006_", "header names the format"); diff --git a/test/src/module/info/infoTest.c b/test/src/module/info/infoTest.c index 25fc7418f8..66cba15ff4 100644 --- a/test/src/module/info/infoTest.c +++ b/test/src/module/info/infoTest.c @@ -74,7 +74,8 @@ static Buffer * testInfoEncrypt(const Buffer *const content, const unsigned int format, const CipherSpec *const cipherSpec) { Buffer *const result = bufNew(0); - IoWrite *const write = infoWriteNew(result, format, cipherSpec); + IoWrite *const write = ioBufferWriteNew(result); + cipherBlockFilterGroupAddP(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = format); ioWriteOpen(write); ioWrite(write, content); @@ -296,10 +297,11 @@ testRun(void) // An unencrypted file has no header no matter the format, since the format is read from the content contentSave = bufNew(0); - TEST_RESULT_VOID( - infoSave( - info, infoWriteNew(contentSave, REPOSITORY_FORMAT_6, cipherSpecNewNone()), testInfoSaveCallback, strNewZ("1")), - "info save"); + IoWrite *const writeNone = ioBufferWriteNew(contentSave); + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(writeNone), cipherModeEncrypt, cipherSpecNewNone(), .format = REPOSITORY_FORMAT_6); + + TEST_RESULT_VOID(infoSave(info, writeNone, testInfoSaveCallback, strNewZ("1")), "info save"); TEST_RESULT_BOOL(strBeginsWithZ(strNewBuf(contentSave), "PGBR"), false, " check no header"); contentLoad = harnessInfoChecksumFormat( From 87bf33b9c0a389690b5b2928720deb9034c6f505 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 12:11:11 +0700 Subject: [PATCH 11/15] Updates. --- test/src/module/command/infoTest.c | 2 +- test/src/module/command/stanzaTest.c | 2 +- test/src/module/common/cryptoTest.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/src/module/command/infoTest.c b/test/src/module/command/infoTest.c index da25dcef42..369c50be6f 100644 --- a/test/src/module/command/infoTest.c +++ b/test/src/module/command/infoTest.c @@ -502,7 +502,7 @@ testRun(void) STORAGE_REPO_ARCHIVE "/9.5-2/0000000100000000/000000010000000000000001-ac61b8f1ec7b1e6c3eaee9345214595eb7daa9a1.gz", .comment = "write WAL db2 timeline 1 repo1"); - // The repository was migrated to format 6 after the first backup, so each backup carries the format it was written + // The repository was migrated to format 6 after the first backup, so each backup contains the format it was written // with rather than the format of the info file const Buffer *const backupInfoContent = harnessInfoChecksumFormat( REPOSITORY_FORMAT_6, diff --git a/test/src/module/command/stanzaTest.c b/test/src/module/command/stanzaTest.c index 1519a0603e..b29ffa9bf6 100644 --- a/test/src/module/command/stanzaTest.c +++ b/test/src/module/command/stanzaTest.c @@ -1194,7 +1194,7 @@ testRun(void) TEST_TITLE("stanza-upgrade - every format that can be read can be requested"); // The allow list for repo-format in build/config.yaml and REPOSITORY_FORMAT_MIN/MAX in version.h are declared separately, - // so make sure they say the same thing + // so make sure they match for (unsigned int format = REPOSITORY_FORMAT_MIN; format <= REPOSITORY_FORMAT_MAX; format++) { argList = strLstDup(argListBase); diff --git a/test/src/module/common/cryptoTest.c b/test/src/module/common/cryptoTest.c index d3d42e1429..93ac692cdd 100644 --- a/test/src/module/common/cryptoTest.c +++ b/test/src/module/common/cryptoTest.c @@ -80,7 +80,7 @@ testRun(void) // Initialization of object // ------------------------------------------------------------------------------------------------------------------------- - // Build from a duplicate to show the copy carries the type and pass of the original + // Build from a duplicate to show the copy contains the type and pass of the original TEST_RESULT_UINT(cipherSpecType(cipherSpecDup(cipherSpecNewNone())), cipherTypeNone, "dup of none"); const CipherSpec *const cipherSpec = cipherSpecDup(cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS))); @@ -88,7 +88,7 @@ testRun(void) TEST_RESULT_UINT(cipherSpecType(cipherSpec), cipherTypeAes256Cbc, "dup type"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(cipherSpec)), TEST_PASS, "dup pass"); - // A pack carries nothing but the type when there is no cipher + // A pack contains nothing but the type when there is no cipher PackWrite *packWrite = pckWriteNewP(); cipherSpecPack(packWrite, cipherSpecNewNone()); @@ -97,7 +97,7 @@ testRun(void) TEST_RESULT_UINT( cipherSpecType(cipherSpecNewPack(pckReadNew(pckWriteResult(packWrite)))), cipherTypeNone, "unpack none"); - // Else it carries the type and pass + // Else it contains the type and pass packWrite = pckWriteNewP(); cipherSpecPack(packWrite, cipherSpecNew(cipherTypeAes256Cbc, testPass)); From e6381abe519e71007eac9518001bc2cffd42effe Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 12:29:58 +0700 Subject: [PATCH 12/15] Fix formatting. --- src/info/infoBackup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index 1ac4c135cf..b329837de2 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -937,8 +937,8 @@ infoBackupSaveFile( // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); IoWrite *const write = ioBufferWriteNew(buffer); - cipherBlockFilterGroupAddP( - ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = infoBackupFormat(infoBackup)); + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = infoBackupFormat(infoBackup)); infoBackupSave(infoBackup, write); // Save the file and make a copy From 785330748b76c0fd39fe73e56bfc56acbd309313 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 12:54:35 +0700 Subject: [PATCH 13/15] Store cipher-digest to preserve migrated files. --- src/info/info.c | 27 +++++++++++++++++---- test/src/module/info/infoTest.c | 37 +++++++++++++++++++++++++++++ test/src/module/info/manifestTest.c | 1 + 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/info/info.c b/src/info/info.c index 69c443f3e3..9cee21cabd 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -127,6 +127,7 @@ infoNew(const unsigned int format, const CipherSpec *const cipherSpecSub) #define INFO_SECTION_BACKREST "backrest" #define INFO_KEY_CHECKSUM "backrest-checksum" #define INFO_SECTION_CIPHER "cipher" +#define INFO_KEY_CIPHER_DIGEST "cipher-digest" #define INFO_KEY_CIPHER_PASS "cipher-pass" FN_EXTERN Info * @@ -158,6 +159,7 @@ infoNewLoad( String *const sectionLast = strNew(); // The last section seen during load IoFilter *const checksumActualFilter = cryptoHashNew(hashTypeSha1); // Checksum calculated from the file const String *checksumExpected = NULL; // Checksum found in ini file + HashType cipherDigest = hashTypeSha1; // Digest the stored pass derives with INFO_CHECKSUM_BEGIN(checksumActualFilter); @@ -229,17 +231,23 @@ infoNewLoad( // Process cipher section else if (strEqZ(value->section, INFO_SECTION_CIPHER)) { + // Store the digest the pass derives with. A file written before the digest was stored has none, so the + // default is what every repository derived with then. + if (strEqZ(value->key, INFO_KEY_CIPHER_DIGEST)) + { + cipherDigest = jsonReadStrId(jsonReadNew(value->value)); + } // No validation needed for cipher-pass, just store it - if (strEqZ(value->key, INFO_KEY_CIPHER_PASS)) + else if (strEqZ(value->key, INFO_KEY_CIPHER_PASS)) { MEM_CONTEXT_OBJ_BEGIN(this) { // The dependent files are encrypted with the same cipher type as this one and derive with the - // digest that goes with the format this file was written at. The format is read before this - // since the sections come out in order and backrest sorts before cipher. + // digest stored with the pass. The digest is read before this since the keys come out in order + // and digest sorts before pass. this->pub.cipherSpec = cipherSpecNewP( cipherSpecType(cipherSpec), BUFSTR(varStr(jsonToVar(value->value))), - .digest = repoFormatDigest(this->pub.format)); + .digest = cipherDigest); } MEM_CONTEXT_OBJ_END(); } @@ -421,6 +429,17 @@ infoSave(Info *const this, IoWrite *const write, InfoSaveCallback *const callbac if (cipherSpecType(infoCipherSpec(this)) != cipherTypeNone) { callbackFunction(callbackData, STRDEF(INFO_SECTION_CIPHER), &data); + + // Store the digest the pass derives with so that a pass outlives the format of the file it is stored in. A pass in a + // file written before this could be stored derives with SHA-1, which is what a reader assumes when it finds no digest. + if (infoFormat(this) >= REPOSITORY_FORMAT_6) + { + char digestZ[STRID_MAX + 1]; + strIdToZ(cipherSpecDigest(infoCipherSpec(this)), digestZ); + + infoSaveValue(&data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_DIGEST, jsonFromVar(VARSTRZ(digestZ))); + } + infoSaveValue( &data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_PASS, jsonFromVar(VARSTR(strNewBuf(cipherSpecPass(infoCipherSpec(this)))))); diff --git a/test/src/module/info/infoTest.c b/test/src/module/info/infoTest.c index 66cba15ff4..e96ef871bd 100644 --- a/test/src/module/info/infoTest.c +++ b/test/src/module/info/infoTest.c @@ -292,6 +292,25 @@ testRun(void) TEST_RESULT_VOID(infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "info save"); TEST_RESULT_STR(strNewBuf(contentSave), strNewBuf(contentLoad), " check save"); + // Migrating to a format that stores the digest does not change the digest the pass derives with, since the files the pass + // encrypted before the migration are not rewritten + contentSave = bufNew(0); + + TEST_RESULT_VOID(infoFormatSet(info, REPOSITORY_FORMAT_6), "migrate to format 6"); + TEST_RESULT_VOID( + infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "save migrated info"); + TEST_RESULT_BOOL( + strstr(strZ(strNewBuf(contentSave)), "cipher-digest=\"sha1\"") != NULL, true, " check digest stored with pass"); + + TEST_ASSIGN( + info, + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentSave, REPOSITORY_FORMAT_6, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, strNew(), .header = true), + "load migrated info"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest unchanged"); + // Header // ------------------------------------------------------------------------------------------------------------------------- // An unencrypted file has no header no matter the format, since the format is read from the content @@ -308,6 +327,7 @@ testRun(void) REPOSITORY_FORMAT_6, STRDEF( "[cipher]\n" + "cipher-digest=\"sha256\"\n" "cipher-pass=\"somepass\"\n")); callbackContent = strNew(); @@ -321,6 +341,23 @@ testRun(void) TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha256, " check cipher sub digest"); + // A pass migrated from a format that could not store the digest keeps deriving with SHA-1, so the files it encrypted + // before the migration are still readable + const Buffer *const contentMigrated = harnessInfoChecksumFormat( + REPOSITORY_FORMAT_6, + STRDEF( + "[cipher]\n" + "cipher-digest=\"sha1\"\n" + "cipher-pass=\"somepass\"\n")); + + TEST_ASSIGN( + info, + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentMigrated, REPOSITORY_FORMAT_6, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, callbackContent, .header = true), + "info migrated to the format that stores the digest"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); + // The content on its own, which is how a caller that wants the file rather than the values in it reads an info file IoRead *const infoRead = ioBufferReadNew(testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec)); diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index af48de8f8f..ddf657ee7d 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -1453,6 +1453,7 @@ testRun(void) "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" "\n" "[cipher]\n" + "cipher-digest=\"sha256\"\n" "cipher-pass=\"somepass\"\n" "\n" "[target:file]\n" From d2e14761d26ee5ef83a11acb6acc4e2f478d91a5 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 17:07:53 +0700 Subject: [PATCH 14/15] Add assert. --- test/src/harness/storage.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/src/harness/storage.c b/test/src/harness/storage.c index f276cbc4ad..8b8901f214 100644 --- a/test/src/harness/storage.c +++ b/test/src/harness/storage.c @@ -70,7 +70,10 @@ testStorageGet(const Storage *const storage, const char *const file, const char if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) { // Derive with SHA-1 since the harness reads and writes files the way a repository at the format these tests build stores - // them, which is the format that had no header to define anything else + // them, which is the format that had no header to define anything else. A caller that asked for a digest would not get + // it, so only a spec at the default is accepted. + ASSERT(cipherSpecDigest(param.cipherSpec) == hashTypeSha256); + ioFilterGroupAdd( filterGroup, cipherBlockNewP( @@ -401,7 +404,10 @@ hrnStoragePut( // Add encrypted filter if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) { - // Derive with SHA-1 to match how the harness reads these files back + // Derive with SHA-1 to match how the harness reads these files back. A caller that asked for a digest would not get it, + // so only a spec at the default is accepted. + ASSERT(cipherSpecDigest(param.cipherSpec) == hashTypeSha256); + ioFilterGroupAdd( filterGroup, cipherBlockNewP( From 4db9fa6709b3268c01717d61e1f8ccd37af27551 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 21 Aug 2026 17:17:04 +0700 Subject: [PATCH 15/15] Stanza-upgrade test. --- test/src/module/command/stanzaTest.c | 65 ++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/test/src/module/command/stanzaTest.c b/test/src/module/command/stanzaTest.c index 132c2b24a5..14408fd141 100644 --- a/test/src/module/command/stanzaTest.c +++ b/test/src/module/command/stanzaTest.c @@ -1,6 +1,8 @@ /*********************************************************************************************************************************** Test Stanza Commands ***********************************************************************************************************************************/ +#include "common/crypto/cipherBlock.h" +#include "common/io/bufferWrite.h" #include "postgres/interface.h" #include "postgres/version.h" #include "storage/posix/storage.h" @@ -23,6 +25,7 @@ testRun(void) Storage *storageHrn = storagePosixNewP(HRN_PATH_STR, .write = true); #define TEST_STANZA "db" + #define TEST_WAL_MIGRATE "archive/db/15-1/000000010000000000000001" #define TEST_STANZA_OTHER "otherstanza" StringList *argListBase = strLstNew(); @@ -1190,6 +1193,68 @@ testRun(void) hrnCfgEnvKeyRemoveRaw(cfgOptRepoFormat, 1); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("stanza-upgrade - a file encrypted before a format migration is readable after it"); + + // A stanza on an encrypted repository at the format before the digest could be stored + argList = strLstNew(); + hrnCfgArgRawBool(argList, cfgOptOnline, false); + hrnCfgArgRawZ(argList, cfgOptStanza, TEST_STANZA); + hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, TEST_PATH "/repo-migrate"); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + hrnCfgArgKeyRawStrId(argList, cfgOptRepoCipherType, 1, cipherTypeAes256Cbc); + hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 1, "12345678"); + HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); + + TEST_RESULT_VOID(cmdStanzaCreate(), "stanza create on encrypted repo"); + TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'db' on repo1"); + + const CipherSpec *const cipherSpecMain = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("12345678")); + const InfoArchive *infoArchiveMigrate = NULL; + + TEST_ASSIGN( + infoArchiveMigrate, infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecMain), + "load archive info before migration"); + + // Write a file with the archive sub-pass, which is how everything below the info files is encrypted + Buffer *const walBuffer = bufNew(0); + IoWrite *const walWrite = ioBufferWriteNew(walBuffer); + + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(walWrite), cipherModeEncrypt, infoArchiveCipherSpec(infoArchiveMigrate)); + ioWriteOpen(walWrite); + ioWrite(walWrite, BUFSTRDEF("WAL BEFORE MIGRATION")); + ioWriteClose(walWrite); + + HRN_STORAGE_PUT(storageRepoIdxWrite(0), TEST_WAL_MIGRATE, walBuffer, .comment = "wal written at format 5"); + + // Migrate the repository + hrnCfgArgKeyRawZ(argList, cfgOptRepoFormat, 1, "6"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format 6 on encrypted repo"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 INFO: upgrade repository format from 5 to 6"); + + // The pass is unchanged by the migration, so a file it encrypted before the migration must still be readable. The digest + // the pass derives with cannot follow the format of the info file storing it or this read would fail. + TEST_ASSIGN( + infoArchiveMigrate, infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecMain), + "load archive info after migration"); + TEST_RESULT_UINT(infoArchiveFormat(infoArchiveMigrate), REPOSITORY_FORMAT_6, "archive info at format 6"); + + StorageRead *const walRead = storageNewReadP(storageRepoIdx(0), STRDEF(TEST_WAL_MIGRATE)); + + cipherBlockFilterGroupAddP( + ioReadFilterGroup(storageReadIo(walRead)), cipherModeDecrypt, infoArchiveCipherSpec(infoArchiveMigrate)); + + TEST_RESULT_STR_Z( + strNewBuf(storageGetP(walRead)), "WAL BEFORE MIGRATION", "wal from before the migration is still readable"); + + hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 1); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("stanza-upgrade - every format that can be read can be requested");