diff --git a/build/config.yaml b/build/config.yaml index 6b500af5a5..30f635d07e 100644 --- a/build/config.yaml +++ b/build/config.yaml @@ -1690,8 +1690,8 @@ option: # Command-line only so that a format left in a configuration file cannot upgrade a repository as a side effect of an unrelated # stanza-upgrade. This does not cover the environment, which can set any option that is valid for the command, so a format left - # there does migrate on the next stanza-upgrade and the option reference says so. The allow list must be kept in sync with - # REPOSITORY_FORMAT_MIN/MAX in version.h + # there does migrate on the next stanza-upgrade and the option reference documents it. The allow list must be kept in sync with + # REPOSITORY_FORMAT_MIN/MAX in common/format.h repo-format: group: repo type: integer diff --git a/doc/xml/reference.xml b/doc/xml/reference.xml index e6b8228f40..4bfcc3521c 100644 --- a/doc/xml/reference.xml +++ b/doc/xml/reference.xml @@ -2929,7 +2929,7 @@ Format supported by all versions since 1.00 - Adds no features yet and is reserved for features in development + Derives repository encryption keys with SHA-256 rather than SHA-1

An existing stanza is migrated to a newer format with the stanza-upgrade command rather than with this command, which sets the format only for a stanza it creates.

diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c index a4e564be37..42b2f9f144 100644 --- a/src/command/backup/backup.c +++ b/src/command/backup/backup.c @@ -241,7 +241,7 @@ cmdBackup(void) // Build an incremental backup if type is not full (manifestPrior will be freed in this call) if (!backupBuildIncr(manifest, manifestPrior, backupStartResult.walSegmentName)) - manifestCipherSpecSet(manifest, cipherSpecGen(cfgOptionStrId(cfgOptRepoCipherType))); + manifestCipherSpecSet(manifest, cipherSpecGen(cfgOptionStrId(cfgOptRepoCipherType), manifestFormat(manifest))); // Set delta if it is not already set and the manifest requires it if (!cfgOptionBool(cfgOptDelta) && varBool(manifestData(manifest)->backupOptionDelta)) diff --git a/src/command/backup/complete.c.inc b/src/command/backup/complete.c.inc index 777f7e26ca..be1617a7f6 100644 --- a/src/command/backup/complete.c.inc +++ b/src/command/backup/complete.c.inc @@ -25,7 +25,7 @@ backupManifestSaveCopy(Manifest *const manifest, const CipherSpec *const cipherS STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(manifestData(manifest)->backupLabel)))); // Add encryption filter if required - cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpecManifest); + cipherBlockFilterGroupAddP(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpecManifest); // Save file manifestSave(manifest, write); @@ -101,7 +101,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif IoFilterGroup *const filterGroup = ioReadFilterGroup(storageReadIo(read)); // Decrypt with archive key if encrypted - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( filterGroup, cipherModeDecrypt, infoArchiveCipherSpec(backupData->archiveInfo)); // Compress/decompress if archive and backup do not have the same compression settings @@ -118,7 +118,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif } // Encrypt with backup key if encrypted - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( filterGroup, cipherModeEncrypt, manifestCipherSpec(manifest)); // Add size filter last to calculate repo size @@ -201,7 +201,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest) StorageRead *const manifestRead = storageNewReadP( storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupLabel))); - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( ioReadFilterGroup(storageReadIo(manifestRead)), cipherModeDecrypt, infoBackupCipherSpec(infoBackup)); StorageWrite *const manifestWrite = storageNewWriteP( @@ -212,7 +212,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest) ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), compressFilterP(compressTypeGz, 9)); - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( ioWriteFilterGroup(storageWriteIo(manifestWrite)), cipherModeEncrypt, infoBackupCipherSpec(infoBackup)); storageCopyP(manifestRead, manifestWrite); diff --git a/src/command/repo/get.c b/src/command/repo/get.c index 81a42e28a6..789aa39c05 100644 --- a/src/command/repo/get.c +++ b/src/command/repo/get.c @@ -16,6 +16,7 @@ Repository Get Command #include "config/config.h" #include "storage/helper.h" +#include "info/info.h" #include "info/infoArchive.h" #include "info/infoBackup.h" @@ -47,6 +48,9 @@ storageGetProcess(IoWrite *const destination) IoRead *const source = storageReadIo( storageNewReadP(storageRepo(), file, .ignoreMissing = cfgOptionBool(cfgOptIgnoreMissing))); + // Is the file an info file, i.e. one that has a header in front of its content? + bool fileIsInfo = false; + // Add decryption if needed if (!cfgOptionBool(cfgOptRaw)) { @@ -96,6 +100,9 @@ storageGetProcess(IoWrite *const destination) cfgCipherSpecMain()); cipherSpec = infoArchiveCipherSpec(info); } + // Else the file is the archive info, which the repo passphrase opens + else + fileIsInfo = true; } // Backup path @@ -128,6 +135,9 @@ storageGetProcess(IoWrite *const destination) else cipherSpec = cipherSpecManifest; } + // Else the file is the backup info, which the repo passphrase opens + else + fileIsInfo = true; } } @@ -137,8 +147,9 @@ storageGetProcess(IoWrite *const destination) ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); - // Add encryption filter - cipherBlockFilterGroupAdd(ioReadFilterGroup(source), cipherModeDecrypt, cipherSpec); + // Add the decryption filter. An info file is read with a header, which the cipher consumes. + ioFilterGroupAdd( + ioReadFilterGroup(source), cipherBlockNewP(cipherModeDecrypt, cipherSpec, .header = fileIsInfo)); } } diff --git a/src/command/restore/timeline.c b/src/command/restore/timeline.c index dd7f71289a..584c0604e4 100644 --- a/src/command/restore/timeline.c +++ b/src/command/restore/timeline.c @@ -94,7 +94,7 @@ historyLoad( { const String *const historyFile = strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%08X.history", strZ(archiveId), timeline); StorageRead *const storageRead = storageNewReadP(storageRepo, historyFile); - cipherBlockFilterGroupAdd(ioReadFilterGroup(storageReadIo(storageRead)), cipherModeDecrypt, cipherSpecArchive); + cipherBlockFilterGroupAddP(ioReadFilterGroup(storageReadIo(storageRead)), cipherModeDecrypt, cipherSpecArchive); const Buffer *const history = storageGetP(storageRead); TRY_BEGIN() diff --git a/src/command/stanza/common.c b/src/command/stanza/common.c index cdb7feb380..05c2063b41 100644 --- a/src/command/stanza/common.c +++ b/src/command/stanza/common.c @@ -6,6 +6,7 @@ Stanza Commands Handler #include "command/check/common.h" #include "command/stanza/common.h" #include "common/debug.h" +#include "common/format.h" #include "common/log.h" #include "config/config.h" #include "db/helper.h" @@ -16,10 +17,11 @@ Stanza Commands Handler /**********************************************************************************************************************************/ FN_EXTERN CipherSpec * -cipherSpecGen(const CipherType cipherType) +cipherSpecGen(const CipherType cipherType, const unsigned int format) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING_ID, cipherType); + FUNCTION_TEST_PARAM(UINT, format); FUNCTION_TEST_END(); CipherSpec *result; @@ -33,8 +35,11 @@ cipherSpecGen(const CipherType cipherType) uint8_t buffer[48]; // 48 is the amount of entropy needed to get a 64 base key cryptoRandomBytes(buffer, sizeof(buffer)); - // The pass is the encoded text rather than the bytes it encodes, so it is stored and derived from as that text - result = cipherSpecNew(cipherType, BUFSTR(strNewEncode(encodingBase64, BUF(buffer, sizeof(buffer))))); + // The pass is the encoded text rather than the bytes it encodes, so it is stored and derived from as that text. The + // digest is the one the file it will be stored in is read with, since that is what will derive it later. + result = cipherSpecNewP( + cipherType, BUFSTR(strNewEncode(encodingBase64, BUF(buffer, sizeof(buffer)))), + .digest = repoFormatDigest(format)); cipherSpecMove(result, memContextPrior()); } MEM_CONTEXT_TEMP_END(); diff --git a/src/command/stanza/common.h b/src/command/stanza/common.h index 225ca5ee34..e763789d6e 100644 --- a/src/command/stanza/common.h +++ b/src/command/stanza/common.h @@ -11,7 +11,8 @@ Stanza Commands Handler Functions ***********************************************************************************************************************************/ // Generate cipher spec with a new pass, none when the type is none -FN_EXTERN CipherSpec *cipherSpecGen(CipherType cipherType); +// Generate a sub pass for a file at this format, which sets the digest since that is what will derive it when it is read back +FN_EXTERN CipherSpec *cipherSpecGen(CipherType cipherType, unsigned int format); // Validate and return database information FN_EXTERN PgControl pgValidate(void); diff --git a/src/command/stanza/create.c b/src/command/stanza/create.c index 856ffe5f6e..61d1585711 100644 --- a/src/command/stanza/create.c +++ b/src/command/stanza/create.c @@ -78,7 +78,7 @@ cmdStanzaCreate(void) const unsigned int format = cfgOptionIdxUInt(cfgOptRepoFormat, repoIdx); // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent archive files - const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx)); + const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), format); // Create and save archive info infoArchive = infoArchiveNew(pgControl.version, pgControl.systemId, format, cipherSpecArchive); @@ -86,7 +86,8 @@ cmdStanzaCreate(void) infoArchiveSaveFile(infoArchive, storageRepoWriteStanza, INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx)); // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent manifests - const CipherSpec *const cipherSpecManifest = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx)); + const CipherSpec *const cipherSpecManifest = cipherSpecGen( + cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), format); // Create and save backup info infoBackup = infoBackupNew( diff --git a/src/command/verify/verify.c b/src/command/verify/verify.c index f30d40fe3e..a6d1472425 100644 --- a/src/command/verify/verify.c +++ b/src/command/verify/verify.c @@ -167,11 +167,12 @@ verifyInvalidFileAdd(List *const invalidFileList, const VerifyResult reason, con Load a file into memory ***********************************************************************************************************************************/ static StorageRead * -verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherSpec) +verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherSpec, const bool decrypt) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING, pathFileName); // Fully qualified path/file name FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); // Cipher spec to open file if encrypted + FUNCTION_TEST_PARAM(BOOL, decrypt); // Decrypt here rather than leaving it to the load? FUNCTION_TEST_END(); ASSERT(pathFileName != NULL); @@ -182,7 +183,12 @@ verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherS // *read points to a location within result so update result with contents based on necessary filters IoRead *const read = storageReadIo(result); - cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec); + // An info file or manifest is decrypted by the load, which has to read the header before it knows what to decrypt with, so the + // checksum is over the file as it is stored. The file and its copy are written from the same bytes, which is all the checksum + // is used to compare. + if (decrypt) + cipherBlockFilterGroupAddP(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec); + ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1)); // If the file is compressed, add a decompression filter @@ -214,7 +220,7 @@ verifyInfoFile(const String *const pathFileName, const bool keepFile, const Ciph { TRY_BEGIN() { - IoRead *const infoRead = storageReadIo(verifyFileLoad(pathFileName, cipherSpec)); + IoRead *const infoRead = storageReadIo(verifyFileLoad(pathFileName, cipherSpec, false)); // If directed to keep the loaded file in memory, then move the file into the result, else drain the io and close it if (keepFile) @@ -827,7 +833,7 @@ verifyArchive(VerifyJobData *const jobData) strNewFmt( STORAGE_REPO_ARCHIVE "/%s/%s/%s", strZ(archiveResult->archiveId), strZ(walPath), strZ(strLstGet(jobData->walFileList, 0))), - jobData->cipherSpecArchive); + jobData->cipherSpecArchive, true); const PgWal walInfo = pgWalFromBuffer( storageGetP(walRead, .exactSize = PG_WAL_HEADER_SIZE), cfgOptionStrNull(cfgOptPgVersionForce)); diff --git a/src/common/crypto/cipherBlock.c b/src/common/crypto/cipherBlock.c index bd3ff35577..feea2d8a67 100644 --- a/src/common/crypto/cipherBlock.c +++ b/src/common/crypto/cipherBlock.c @@ -3,6 +3,7 @@ Block Cipher ***********************************************************************************************************************************/ #include +#include #include #include @@ -11,21 +12,71 @@ Block Cipher #include "common/crypto/cipherBlock.h" #include "common/crypto/common.h" #include "common/debug.h" +#include "common/format.h" #include "common/io/filter/filter.h" #include "common/log.h" +#include "common/type/convert.h" #include "common/type/object.h" /*********************************************************************************************************************************** -Header constants and sizes +Magic constant for salted encrypt, written before the salt unless the cipher is raw. Only salted encrypt is done here, but this +constant is required for compatibility with the openssl command-line tool. ***********************************************************************************************************************************/ -// Magic constant for salted encrypt. Only salted encrypt is done here, but this constant is required for compatibility with the -// openssl command-line tool. #define CIPHER_BLOCK_MAGIC "Salted__" #define CIPHER_BLOCK_MAGIC_SIZE (sizeof(CIPHER_BLOCK_MAGIC) - 1) +/*********************************************************************************************************************************** +Format header + +A file written with a header begins with fixed-size plaintext naming the repository format it was written with. The header exists +because the digest the pass derives with follows the format, and the format is recorded inside the file that the pass encrypts. A +reader that could not see the format in advance would have to decrypt to learn what it should have decrypted with. + +The header takes the place of the salted magic, which is why a file that contains one is written raw. Both are eight bytes followed +by the salt, so the eight bytes are consumed either way and what follows begins with the salt no matter which was there. It also +means a file of either kind is opened with the openssl command-line tool the same way: replace the first eight bytes with the magic +that tool expects. + +A file that begins with the magic rather than the header was written at format 5, the only format there was before the header, so +that is not an error when a header was expected. + +For the four bytes after the header magic, the first three are the format and the last is reserved for future use, e.g. naming which +key the file was encrypted with once a repository can hold more than one. The format comes first so that it is always at the same +place, which is what lets a version work out whether it can read the file at all. Once the format is identified as compatible with +this version, the spare byte is examined, and it must only be the underscore this version writes. + +The header is not part of the file content. This filter adds it on encrypt and consumes it on decrypt, so nothing on either side +sees anything but the content. +***********************************************************************************************************************************/ +#define CIPHER_BLOCK_HEADER_MAGIC "PGBR" +#define CIPHER_BLOCK_HEADER_MAGIC_SIZE (sizeof(CIPHER_BLOCK_HEADER_MAGIC) - 1) +#define CIPHER_BLOCK_HEADER_RESERVED '_' +#define CIPHER_BLOCK_HEADER_FORMAT_SIZE 3 + // Total length of cipher header #define CIPHER_BLOCK_HEADER_SIZE (CIPHER_BLOCK_MAGIC_SIZE + PKCS5_SALT_LEN) +/*********************************************************************************************************************************** +Digest the pass derives the key with. The lookup is by name, so a digest must be one openssl knows. +***********************************************************************************************************************************/ +static const EVP_MD * +cipherBlockDigest(const HashType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING_ID, type); + FUNCTION_TEST_END(); + + char typeZ[STRID_MAX + 1]; + strIdToZ(type, typeZ); + + const EVP_MD *const result = EVP_get_digestbyname(typeZ); + + if (result == NULL) + THROW_FMT(AssertError, "unable to load digest '%s'", typeZ); + + FUNCTION_TEST_RETURN_TYPE_CONST_P(EVP_MD, result); +} + /*********************************************************************************************************************************** Object type ***********************************************************************************************************************************/ @@ -33,6 +84,8 @@ typedef struct CipherBlock { CipherMode mode; // Mode encrypt/decrypt bool raw; // Omit header magic to save space + bool headerFormat; // Read/write the format header + unsigned int format; // Repository format, zero until a read header gives one bool saltDone; // Has the salt been read/generated? bool processDone; // Has any data been processed? const Buffer *pass; // Passphrase used to generate encryption key @@ -131,8 +184,25 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s // On encrypt the salt is generated if (this->mode == cipherModeEncrypt) { - // Add magic to the destination buffer so openssl knows the file is salted - if (!this->raw) + // Add the header to the destination buffer in place of the magic. Both are the same size so what follows begins with + // the salt either way. + if (this->headerFormat) + { + memcpy(destination, CIPHER_BLOCK_HEADER_MAGIC, CIPHER_BLOCK_HEADER_MAGIC_SIZE); + + // Write the format zero-padded so it is always the same size. The terminator lands on the reserved byte, which is + // written next. + snprintf( + (char *)destination + CIPHER_BLOCK_HEADER_MAGIC_SIZE, CIPHER_BLOCK_HEADER_FORMAT_SIZE + 1, "%0*u", + CIPHER_BLOCK_HEADER_FORMAT_SIZE, this->format); + + destination[CIPHER_BLOCK_MAGIC_SIZE - 1] = CIPHER_BLOCK_HEADER_RESERVED; + + destination += CIPHER_BLOCK_MAGIC_SIZE; + destinationSize += CIPHER_BLOCK_MAGIC_SIZE; + } + // Else add magic to the destination buffer so openssl knows the file is salted + else if (!this->raw) { memcpy(destination, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE); destination += CIPHER_BLOCK_MAGIC_SIZE; @@ -161,9 +231,43 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s source += headerExpected - this->headerSize; sourceSize -= headerExpected - this->headerSize; - // The first bytes of the file to decrypt should be equal to the magic. If not then this is not an encrypted file, - // or at least not in a format we recognize. - if (!this->raw && memcmp(this->header, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) + // Read the format from the header. A file that begins with the magic instead was written at format 5, the only + // format there was before the header, so that start is not an error here. + if (this->headerFormat) + { + const char *const headerZ = (const char *)this->header; + unsigned int format = REPOSITORY_FORMAT_5; + + if (memcmp(headerZ, CIPHER_BLOCK_HEADER_MAGIC, CIPHER_BLOCK_HEADER_MAGIC_SIZE) == 0) + { + for (unsigned int digitIdx = 0; digitIdx < CIPHER_BLOCK_HEADER_FORMAT_SIZE; digitIdx++) + { + if (!isdigit((unsigned char)headerZ[CIPHER_BLOCK_HEADER_MAGIC_SIZE + digitIdx])) + THROW(FormatError, "invalid cipher header"); + } + + format = cvtZSubNToUInt(headerZ, CIPHER_BLOCK_HEADER_MAGIC_SIZE, CIPHER_BLOCK_HEADER_FORMAT_SIZE); + + // Error on a format this version cannot read before anything is decrypted + repoFormatValidate(format); + + // The format is one this version knows, so the byte held back for later must be the one this version writes + if (headerZ[CIPHER_BLOCK_MAGIC_SIZE - 1] != CIPHER_BLOCK_HEADER_RESERVED) + THROW(FormatError, "invalid cipher header"); + } + // Else the bytes must be the magic, since that is all a file with no header can begin with + else if (memcmp(headerZ, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) + THROW(CryptoError, "cipher header invalid"); + + // Error when the format the caller expected is not the one the file was written with + if (this->format != 0 && this->format != format) + THROW_FMT(FormatError, "expected repository format %u but found %u", this->format, format); + + this->format = format; + } + // Else the first bytes of the file to decrypt should be equal to the magic. If not then this is not an encrypted + // file, or at least not in a format we recognize. + else if (!this->raw && memcmp(this->header, CIPHER_BLOCK_MAGIC, CIPHER_BLOCK_MAGIC_SIZE) != 0) THROW(CryptoError, "cipher header invalid"); } // Else copy what was provided into the header buffer and return 0 @@ -180,6 +284,10 @@ cipherBlockProcessBlock(CipherBlock *const this, const uint8_t *source, size_t s // If salt generation/read is done if (salt) { + // Resolve the digest now that the format is known, which for a header that was read is only true here + if (this->digest == NULL) + this->digest = cipherBlockDigest(repoFormatDigest(this->format)); + // Generate key and initialization vector uint8_t key[EVP_MAX_KEY_LENGTH]; uint8_t initVector[EVP_MAX_IV_LENGTH]; @@ -382,6 +490,49 @@ cipherBlockInputSame(const THIS_VOID) FUNCTION_TEST_RETURN(BOOL, this->inputSame); } +/*********************************************************************************************************************************** +Report the format the header contained +***********************************************************************************************************************************/ +static Pack * +cipherBlockResult(THIS_VOID) +{ + THIS(CipherBlock); + + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CIPHER_BLOCK, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + Pack *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + PackWrite *const packWrite = pckWriteNewP(); + + pckWriteU32P(packWrite, this->format); + pckWriteEndP(packWrite); + + result = pckMove(pckWriteResult(packWrite), memContextPrior()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_TEST_RETURN(PACK, result); +} + +/**********************************************************************************************************************************/ +FN_EXTERN unsigned int +cipherBlockFormat(PackRead *const cipherBlockResult) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, cipherBlockResult); + FUNCTION_TEST_END(); + + ASSERT(cipherBlockResult != NULL); + + FUNCTION_TEST_RETURN(UINT, pckReadU32P(cipherBlockResult)); +} + /**********************************************************************************************************************************/ FN_EXTERN IoFilter * cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const CipherBlockNewParam param) @@ -390,12 +541,20 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const FUNCTION_LOG_PARAM(STRING_ID, mode); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); FUNCTION_LOG_PARAM(BOOL, param.raw); + FUNCTION_LOG_PARAM(BOOL, param.header); + FUNCTION_LOG_PARAM(UINT, param.format); FUNCTION_LOG_END(); ASSERT(cipherSpec != NULL); ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone); ASSERT(cipherSpecPass(cipherSpec) != NULL && !bufEmpty(cipherSpecPass(cipherSpec))); + // The header takes the place of the magic, so a file that contains one is never also raw + ASSERT(!param.raw || (!param.header && param.format == 0)); + + // On encrypt the format defines whether a header is written, so a header is only ever requested on decrypt + ASSERT(mode == cipherModeDecrypt || !param.header); + // Init crypto subsystem cryptoInit(); @@ -409,16 +568,26 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const zFree(cipherTypeZ); + // Lookup digest. A header that has yet to be read is what defines the format of the file, and the format defines the digest, so + // in that case the lookup waits until the header has been read. + const EVP_MD *digest = NULL; + + if (!param.header) + { + // A format defines the digest, otherwise it comes from the spec + digest = cipherBlockDigest(param.format != 0 ? repoFormatDigest(param.format) : cipherSpecDigest(cipherSpec)); + } + OBJ_NEW_BEGIN(CipherBlock, .childQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1) { *this = (CipherBlock) { .mode = mode, .raw = param.raw, + .headerFormat = mode == cipherModeEncrypt ? param.format >= REPOSITORY_FORMAT_6 : param.header, + .format = param.format, .cipher = cipher, - - // The key is always derived with SHA-1. Deriving it with anything else is not supported yet. - .digest = EVP_sha1(), + .digest = digest, .pass = bufDup(cipherSpecPass(cipherSpec)), }; } @@ -434,6 +603,8 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const pckWriteU64P(packWrite, mode); cipherSpecPack(packWrite, cipherSpec); pckWriteBoolP(packWrite, param.raw); + pckWriteBoolP(packWrite, param.header); + pckWriteU32P(packWrite, param.format); pckWriteEndP(packWrite); paramList = pckMove(pckWriteResult(packWrite), memContextPrior()); @@ -444,7 +615,10 @@ cipherBlockNew(const CipherMode mode, const CipherSpec *const cipherSpec, const IO_FILTER, ioFilterNewP( CIPHER_BLOCK_FILTER_TYPE, this, paramList, .done = cipherBlockDone, .inOut = cipherBlockProcess, - .inputSame = cipherBlockInputSame)); + .inputSame = cipherBlockInputSame, + + // Only a filter that reads a header has a format to report + .result = param.header ? cipherBlockResult : NULL)); } FN_EXTERN IoFilter * @@ -458,8 +632,11 @@ cipherBlockNewPack(const Pack *const paramList) const CipherMode cipherMode = (CipherMode)pckReadU64P(paramListPack); const CipherSpec *const cipherSpec = cipherSpecNewPack(paramListPack); const bool raw = pckReadBoolP(paramListPack); + const bool header = pckReadBoolP(paramListPack); + const unsigned int format = pckReadU32P(paramListPack); - result = ioFilterMove(cipherBlockNewP(cipherMode, cipherSpec, .raw = raw), memContextPrior()); + result = ioFilterMove( + cipherBlockNewP(cipherMode, cipherSpec, .raw = raw, .header = header, .format = format), memContextPrior()); } MEM_CONTEXT_TEMP_END(); @@ -468,19 +645,22 @@ cipherBlockNewPack(const Pack *const paramList) /**********************************************************************************************************************************/ FN_EXTERN IoFilterGroup * -cipherBlockFilterGroupAdd(IoFilterGroup *const filterGroup, const CipherMode mode, const CipherSpec *const cipherSpec) +cipherBlockFilterGroupAdd( + IoFilterGroup *const filterGroup, const CipherMode mode, const CipherSpec *const cipherSpec, + const CipherBlockFilterGroupAddParam param) { FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(IO_FILTER_GROUP, filterGroup); FUNCTION_LOG_PARAM(STRING_ID, mode); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_LOG_PARAM(UINT, param.format); FUNCTION_LOG_END(); ASSERT(filterGroup != NULL); ASSERT(cipherSpec != NULL); if (cipherSpecType(cipherSpec) != cipherTypeNone) - ioFilterGroupAdd(filterGroup, cipherBlockNewP(mode, cipherSpec)); + ioFilterGroupAdd(filterGroup, cipherBlockNewP(mode, cipherSpec, .format = param.format)); FUNCTION_LOG_RETURN(IO_FILTER_GROUP, filterGroup); } diff --git a/src/common/crypto/cipherBlock.h b/src/common/crypto/cipherBlock.h index ad28a26149..f98b1984f9 100644 --- a/src/common/crypto/cipherBlock.h +++ b/src/common/crypto/cipherBlock.h @@ -19,6 +19,8 @@ typedef struct CipherBlockNewParam { VAR_PARAM_HEADER; bool raw; // Omit header magic to save space + bool header; // Read the format header, decrypt only + unsigned int format; // Repository format, which on encrypt defines the header } CipherBlockNewParam; #define cipherBlockNewP(mode, cipherSpec, ...) \ @@ -27,10 +29,28 @@ typedef struct CipherBlockNewParam FN_EXTERN IoFilter *cipherBlockNew(CipherMode mode, const CipherSpec *cipherSpec, CipherBlockNewParam param); FN_EXTERN IoFilter *cipherBlockNewPack(const Pack *paramList); +/*********************************************************************************************************************************** +Filter result + +The format the file was written with, which is what the header was read for. Only a filter that read a header has one to report. +***********************************************************************************************************************************/ +FN_EXTERN unsigned int cipherBlockFormat(PackRead *cipherBlockResult); + /*********************************************************************************************************************************** Helper functions ***********************************************************************************************************************************/ -// Add a block cipher to an io object. Nothing is added when the repository is not encrypted. -FN_EXTERN IoFilterGroup *cipherBlockFilterGroupAdd(IoFilterGroup *filterGroup, CipherMode mode, const CipherSpec *cipherSpec); +// Add a block cipher to an io object. Nothing is added when the repository is not encrypted. The format is required when a file is +// written at one, since the format defines the digest and whether a header is written. +typedef struct CipherBlockFilterGroupAddParam +{ + VAR_PARAM_HEADER; + unsigned int format; // Repository format the file is written at +} CipherBlockFilterGroupAddParam; + +#define cipherBlockFilterGroupAddP(filterGroup, mode, cipherSpec, ...) \ + cipherBlockFilterGroupAdd(filterGroup, mode, cipherSpec, (CipherBlockFilterGroupAddParam){VAR_PARAM_INIT, __VA_ARGS__}) + +FN_EXTERN IoFilterGroup *cipherBlockFilterGroupAdd( + IoFilterGroup *filterGroup, CipherMode mode, const CipherSpec *cipherSpec, CipherBlockFilterGroupAddParam param); #endif diff --git a/src/common/crypto/spec.c b/src/common/crypto/spec.c index 76627b8af9..6e32ffb5a9 100644 --- a/src/common/crypto/spec.c +++ b/src/common/crypto/spec.c @@ -17,11 +17,12 @@ struct CipherSpec /**********************************************************************************************************************************/ FN_EXTERN CipherSpec * -cipherSpecNew(const CipherType type, const Buffer *const pass) +cipherSpecNew(const CipherType type, const Buffer *const pass, const CipherSpecNewParam param) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING_ID, type); FUNCTION_TEST_PARAM(BUFFER, pass); + FUNCTION_TEST_PARAM(STRING_ID, param.digest); FUNCTION_TEST_END(); ASSERT((type == cipherTypeNone) == (pass == NULL)); @@ -32,7 +33,14 @@ cipherSpecNew(const CipherType type, const Buffer *const pass) *this = (CipherSpec){.pub = {.type = type}}; if (this->pub.type != cipherTypeNone) + { + if (param.digest == 0) + this->pub.digest = hashTypeSha256; + else + this->pub.digest = param.digest; + this->pub.pass = bufDup(pass); + } } OBJ_NEW_END(); @@ -56,7 +64,10 @@ cipherSpecNewPack(PackRead *const packRead) // Nothing else was written when there is no cipher. The pass is read here rather than copied in since this context is the // one it belongs in. if (this->pub.type != cipherTypeNone) + { + this->pub.digest = (HashType)pckReadStrIdP(packRead); this->pub.pass = pckReadBinP(packRead); + } } OBJ_NEW_END(); @@ -79,7 +90,10 @@ cipherSpecPack(PackWrite *const packWrite, const CipherSpec *const this) // Nothing else is needed when there is no cipher if (cipherSpecType(this) != cipherTypeNone) + { + pckWriteStrIdP(packWrite, cipherSpecDigest(this)); pckWriteBinP(packWrite, cipherSpecPass(this)); + } FUNCTION_TEST_RETURN_VOID(); } @@ -91,6 +105,14 @@ cipherSpecToLog(const CipherSpec *const this, StringStatic *const debugLog) char typeZ[STRID_MAX + 1]; strIdToZ(cipherSpecType(this), typeZ); - // The pass is never logged - strStcFmt(debugLog, "{type: %s}", typeZ); + // There is no digest when there is no cipher. The pass is never logged. + if (cipherSpecType(this) == cipherTypeNone) + strStcFmt(debugLog, "{type: %s}", typeZ); + else + { + char digestZ[STRID_MAX + 1]; + strIdToZ(cipherSpecDigest(this), digestZ); + + strStcFmt(debugLog, "{type: %s, digest: %s}", typeZ, digestZ); + } } diff --git a/src/common/crypto/spec.h b/src/common/crypto/spec.h index 00a61a8080..919503b8da 100644 --- a/src/common/crypto/spec.h +++ b/src/common/crypto/spec.h @@ -4,8 +4,16 @@ Cipher Spec Everything needed to encrypt or decrypt, kept together so that adding to it does not mean changing every function and protocol message that contains it. -The passphrase (pass) contains the bytes the key is derived from. There is no pass when the type is none, and the pass is never -logged. +The pass holds the bytes used to derive the key. The digest travels with the pass because the two are chosen together and deriving +with the wrong digest gives a wrong key instead of an error. + +The pass is a buffer rather than a string because it may be binary or it may be text and nothing here needs to know which. It is +copied into the object, so the caller can release whatever it read the pass from. + +The digest defaults to SHA-256, so a caller with no preference gets the digest new work should use. Repositories before format 6 +derived with SHA-1, so a caller that works with that format must pass SHA-1 rather than taking the default. + +There is no digest or pass when the type is none, and the pass is never logged. ***********************************************************************************************************************************/ #ifndef COMMON_CRYPTO_SPEC_H #define COMMON_CRYPTO_SPEC_H @@ -23,14 +31,23 @@ typedef struct CipherSpec CipherSpec; /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -// Create from a pass -FN_EXTERN CipherSpec *cipherSpecNew(CipherType type, const Buffer *pass); +// Create from a pass, which is the key bytes or the passphrase text +typedef struct CipherSpecNewParam +{ + VAR_PARAM_HEADER; + HashType digest; // Digest to derive the key with instead of SHA-256 +} CipherSpecNewParam; + +#define cipherSpecNewP(type, pass, ...) \ + cipherSpecNew(type, pass, (CipherSpecNewParam){VAR_PARAM_INIT, __VA_ARGS__}) -// Create for no encryption +FN_EXTERN CipherSpec *cipherSpecNew(CipherType type, const Buffer *pass, CipherSpecNewParam param); + +// Create for data that is not encrypted FN_INLINE_ALWAYS CipherSpec * cipherSpecNewNone(void) { - return cipherSpecNew(cipherTypeNone, NULL); + return cipherSpecNewP(cipherTypeNone, NULL); } // Create from a pack written by cipherSpecPack() @@ -42,6 +59,7 @@ Getters/Setters typedef struct CipherSpecPub { CipherType type; // Cipher type, none when not encrypted + HashType digest; // Digest the pass derives the key with const Buffer *pass; // Passphrase text or key bytes } CipherSpecPub; @@ -52,6 +70,13 @@ cipherSpecType(const CipherSpec *const this) return THIS_PUB(CipherSpec)->type; } +// Digest the pass derives the key with +FN_INLINE_ALWAYS HashType +cipherSpecDigest(const CipherSpec *const this) +{ + return THIS_PUB(CipherSpec)->digest; +} + // Passphrase text or key bytes FN_INLINE_ALWAYS const Buffer * cipherSpecPass(const CipherSpec *const this) @@ -66,7 +91,7 @@ Functions FN_INLINE_ALWAYS CipherSpec * cipherSpecDup(const CipherSpec *const this) { - return cipherSpecNew(cipherSpecType(this), cipherSpecPass(this)); + return cipherSpecNewP(cipherSpecType(this), cipherSpecPass(this), .digest = cipherSpecDigest(this)); } // Write to a pack so it can be passed over a protocol diff --git a/src/common/format.c b/src/common/format.c new file mode 100644 index 0000000000..aba51f5710 --- /dev/null +++ b/src/common/format.c @@ -0,0 +1,51 @@ +/*********************************************************************************************************************************** +Repository Format +***********************************************************************************************************************************/ +#include + +#include "common/debug.h" +#include "common/format.h" +#include "version.h" + +/**********************************************************************************************************************************/ +FN_EXTERN void +repoFormatValidate(const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + // A format newer than this version can read requires an upgrade. Do not suggest a version since this version cannot know which + // version added the format. + if (format > REPOSITORY_FORMAT_MAX) + { + THROW_FMT( + FormatError, + "repository format %u requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + // A format older than this version can read requires an older version to migrate the repository + if (format < REPOSITORY_FORMAT_MIN) + { + THROW_FMT( + FormatError, + "repository format %u is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", + format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); + } + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +FN_EXTERN HashType +repoFormatDigest(const unsigned int format) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, format); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RETURN(STRING_ID, format >= REPOSITORY_FORMAT_6 ? hashTypeSha256 : hashTypeSha1); +} diff --git a/src/common/format.h b/src/common/format.h new file mode 100644 index 0000000000..23a1647da6 --- /dev/null +++ b/src/common/format.h @@ -0,0 +1,37 @@ +/*********************************************************************************************************************************** +Repository Format + +Defines the format for info and manifest files as well as on-disk structure. Each info file and manifest stores the format it was +written with, so a repository may contain more than one format while older backups and archives expire. + +A constant is defined for each format so that code which varies by format can be explicit about the format it applies to. +REPOSITORY_FORMAT_MIN/MAX are the range that can be read. The allow list for repo-format in build/config.yaml must be kept in sync +with MIN/MAX and its default is the format used for new repositories. +***********************************************************************************************************************************/ +#ifndef COMMON_FORMAT_H +#define COMMON_FORMAT_H + +#include "common/crypto/common.h" + +/*********************************************************************************************************************************** +Format numbers +***********************************************************************************************************************************/ +#define REPOSITORY_FORMAT_5 5 +#define REPOSITORY_FORMAT_6 6 + +#define REPOSITORY_FORMAT_MIN REPOSITORY_FORMAT_5 +#define REPOSITORY_FORMAT_MAX REPOSITORY_FORMAT_6 + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Error when a format cannot be read by this version. The format in a cipher header is checked before anything is decrypted, since +// decrypting requires knowing what the format expects and this version does not know what a newer format expects. +FN_EXTERN void repoFormatValidate(unsigned int format); + +// Digest a pass stored in a file at this format derives the key with. SHA-1 is what every repository used before format 6 and is +// kept for those, so a repository that has not been migrated is read and written exactly as it was. A pass is generated with the +// digest of the file it will be stored in, since that is what a reader will derive it with. +FN_EXTERN HashType repoFormatDigest(unsigned int format); + +#endif diff --git a/src/config/config.c b/src/config/config.c index de49e46c3b..7b49f979d2 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -1117,7 +1117,7 @@ cfgCipherSpecMainIdx(const unsigned int repoIdx) { const CipherType cipherType = cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx); - configLocal->cipherSpecMain[repoIdx] = cipherSpecNew( + configLocal->cipherSpecMain[repoIdx] = cipherSpecNewP( cipherType, cipherType == cipherTypeNone ? NULL : BUFSTR(cfgOptionIdxStr(cfgOptRepoCipherPass, repoIdx))); } MEM_CONTEXT_END(); diff --git a/src/info/info.c b/src/info/info.c index d31941f1b1..9cee21cabd 100644 --- a/src/info/info.c +++ b/src/info/info.c @@ -3,14 +3,20 @@ Info Handler ***********************************************************************************************************************************/ #include +#include #include #include #include +#include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" #include "common/debug.h" +#include "common/format.h" #include "common/ini.h" +#include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" #include "common/io/filter/filter.h" +#include "common/io/io.h" #include "common/log.h" #include "common/type/convert.h" #include "common/type/json.h" @@ -121,17 +127,20 @@ infoNew(const unsigned int format, const CipherSpec *const cipherSpecSub) #define INFO_SECTION_BACKREST "backrest" #define INFO_KEY_CHECKSUM "backrest-checksum" #define INFO_SECTION_CIPHER "cipher" +#define INFO_KEY_CIPHER_DIGEST "cipher-digest" #define INFO_KEY_CIPHER_PASS "cipher-pass" FN_EXTERN Info * infoNewLoad( - IoRead *const read, const CipherSpec *const cipherSpec, InfoLoadNewCallback *const callbackFunction, void *const callbackData) + IoRead *const read, const CipherSpec *const cipherSpec, InfoLoadNewCallback *const callbackFunction, + void *const callbackData, const InfoNewLoadParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(IO_READ, read); FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec); FUNCTION_LOG_PARAM(FUNCTIONP, callbackFunction); FUNCTION_LOG_PARAM_P(VOID, callbackData); + FUNCTION_LOG_PARAM(BOOL, param.header); FUNCTION_LOG_END(); FUNCTION_AUDIT_CALLBACK(); @@ -150,11 +159,20 @@ infoNewLoad( String *const sectionLast = strNew(); // The last section seen during load IoFilter *const checksumActualFilter = cryptoHashNew(hashTypeSha1); // Checksum calculated from the file const String *checksumExpected = NULL; // Checksum found in ini file + HashType cipherDigest = hashTypeSha1; // Digest the stored pass derives with INFO_CHECKSUM_BEGIN(checksumActualFilter); TRY_BEGIN() { + // The content is decrypted as it is parsed. A file that may contain a header is read with one, which the cipher + // consumes and reports the format of once the read is done. + if (cipherSpecType(cipherSpec) != cipherTypeNone) + { + ioFilterGroupAdd( + ioReadFilterGroup(read), cipherBlockNewP(cipherModeDecrypt, cipherSpec, .header = param.header)); + } + Ini *const ini = iniNewP(read, .strict = true); MEM_CONTEXT_TEMP_RESET_BEGIN() @@ -186,30 +204,10 @@ infoNewLoad( // Validate and store format if (strEqZ(value->key, INFO_KEY_FORMAT)) { - const uint64_t format = varUInt64(jsonToVar(value->value)); - - // A format newer than this version can read requires an upgrade. Do not suggest a version since - // this version cannot know which version added the format. - if (format > REPOSITORY_FORMAT_MAX) - { - THROW_FMT( - FormatError, - "repository format %" PRIu64 " requires a newer version of " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } + const unsigned int format = jsonReadUInt(jsonReadNew(value->value)); + repoFormatValidate(format); - // A format older than this version can read requires an older version to migrate the repository - if (format < REPOSITORY_FORMAT_MIN) - { - THROW_FMT( - FormatError, - "repository format %" PRIu64 " is no longer supported by " PROJECT_NAME "\n" - "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format %d to %d.", - format, REPOSITORY_FORMAT_MIN, REPOSITORY_FORMAT_MAX); - } - - this->pub.format = (unsigned int)format; + this->pub.format = format; } // Store pgBackRest version else if (strEqZ(value->key, INFO_KEY_VERSION)) @@ -233,14 +231,23 @@ infoNewLoad( // Process cipher section else if (strEqZ(value->section, INFO_SECTION_CIPHER)) { + // Store the digest the pass derives with. A file written before the digest was stored has none, so the + // default is what every repository derived with then. + if (strEqZ(value->key, INFO_KEY_CIPHER_DIGEST)) + { + cipherDigest = jsonReadStrId(jsonReadNew(value->value)); + } // No validation needed for cipher-pass, just store it - if (strEqZ(value->key, INFO_KEY_CIPHER_PASS)) + else if (strEqZ(value->key, INFO_KEY_CIPHER_PASS)) { MEM_CONTEXT_OBJ_BEGIN(this) { - // The dependent files are encrypted with the same cipher type as this one - this->pub.cipherSpec = cipherSpecNew( - cipherSpecType(cipherSpec), BUFSTR(varStr(jsonToVar(value->value)))); + // The dependent files are encrypted with the same cipher type as this one and derive with the + // digest stored with the pass. The digest is read before this since the keys come out in order + // and digest sorts before pass. + this->pub.cipherSpec = cipherSpecNewP( + cipherSpecType(cipherSpec), BUFSTR(varStr(jsonToVar(value->value))), + .digest = cipherDigest); } MEM_CONTEXT_OBJ_END(); } @@ -281,6 +288,22 @@ infoNewLoad( // format is zero until the key is found and the value stored, so if we got here then the key was not found. if (infoFormat(this) == 0) THROW(FormatError, "repository format not found\nHINT: is this a valid " PROJECT_NAME " info file?"); + + // Only a cipher that read a header reports a format, so a result here means the file had one. The header is written + // from the same format as the content, so a file where they disagree has been damaged or put together from parts of two + // files. + PackRead *const cipherResult = ioFilterGroupResultP(ioReadFilterGroup(read), CIPHER_BLOCK_FILTER_TYPE); + + if (cipherResult != NULL) + { + const unsigned int formatHeader = cipherBlockFormat(cipherResult); + + if (this->pub.format != formatHeader) + { + THROW_FMT( + FormatError, "repository format %u does not match header format %u", this->pub.format, formatHeader); + } + } } MEM_CONTEXT_TEMP_END(); @@ -406,6 +429,17 @@ infoSave(Info *const this, IoWrite *const write, InfoSaveCallback *const callbac if (cipherSpecType(infoCipherSpec(this)) != cipherTypeNone) { callbackFunction(callbackData, STRDEF(INFO_SECTION_CIPHER), &data); + + // Store the digest the pass derives with so that a pass outlives the format of the file it is stored in. A pass in a + // file written before this could be stored derives with SHA-1, which is what a reader assumes when it finds no digest. + if (infoFormat(this) >= REPOSITORY_FORMAT_6) + { + char digestZ[STRID_MAX + 1]; + strIdToZ(cipherSpecDigest(infoCipherSpec(this)), digestZ); + + infoSaveValue(&data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_DIGEST, jsonFromVar(VARSTRZ(digestZ))); + } + infoSaveValue( &data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_PASS, jsonFromVar(VARSTR(strNewBuf(cipherSpecPass(infoCipherSpec(this)))))); diff --git a/src/info/info.h b/src/info/info.h index 3af5818afc..c2e512d715 100644 --- a/src/info/info.h +++ b/src/info/info.h @@ -41,10 +41,22 @@ Constructors ***********************************************************************************************************************************/ FN_EXTERN Info *infoNew(unsigned int format, const CipherSpec *cipherSpecSub); -// Create new object and load contents from a file. The cipher spec the file is read with supplies the type for the cipher spec -// built from the pass stored in it. +// Create new object and load contents from a file. Decryption is added here rather than by the caller because a file that contains +// a header cannot be decrypted until the header has been read. The cipher spec supplies the type and pass; the digest comes from +// the format the file turns out to be at. The same spec supplies the type for the cipher spec built from the pass stored in the +// file. +typedef struct InfoNewLoadParam +{ + VAR_PARAM_HEADER; + bool header; // Does the file begin with a header? +} InfoNewLoadParam; + +#define infoNewLoadP(read, cipherSpec, callbackFunction, callbackData, ...) \ + infoNewLoad(read, cipherSpec, callbackFunction, callbackData, (InfoNewLoadParam){VAR_PARAM_INIT, __VA_ARGS__}) + FN_EXTERN Info *infoNewLoad( - IoRead *read, const CipherSpec *cipherSpec, InfoLoadNewCallback *callbackFunction, void *callbackData); + IoRead *read, const CipherSpec *cipherSpec, InfoLoadNewCallback *callbackFunction, void *callbackData, + InfoNewLoadParam param); /*********************************************************************************************************************************** Getters/Setters diff --git a/src/info/infoArchive.c b/src/info/infoArchive.c index 4e38a60038..0da2a23d21 100644 --- a/src/info/infoArchive.c +++ b/src/info/infoArchive.c @@ -250,10 +250,8 @@ infoArchiveLoadFileCallback(void *const data, const unsigned int try) // Construct filename based on try const String *const fileName = try == 0 ? loadData->fileName : strNewFmt("%s" INFO_COPY_EXT, strZ(loadData->fileName)); - // Attempt to load the file + // Attempt to load the file. Decryption is added during load since the header has to be read before the digest is known. IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); - cipherBlockFilterGroupAdd( - ioReadFilterGroup(read), cipherModeDecrypt, loadData->cipherSpec); MEM_CONTEXT_BEGIN(loadData->memContext) { @@ -340,7 +338,8 @@ infoArchiveSaveFile( // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); IoWrite *const write = ioBufferWriteNew(buffer); - cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec); + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = infoArchiveFormat(infoArchive)); infoArchiveSave(infoArchive, write); // Save the file and make a copy diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index 5385097a2f..b329837de2 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -714,10 +714,8 @@ infoBackupLoadFileCallback(void *const data, const unsigned int try) // Construct filename based on try const String *const fileName = try == 0 ? loadData->fileName : strNewFmt("%s" INFO_COPY_EXT, strZ(loadData->fileName)); - // Attempt to load the file + // Attempt to load the file. Decryption is added during load since the header has to be read before the digest is known. IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); - cipherBlockFilterGroupAdd( - ioReadFilterGroup(read), cipherModeDecrypt, loadData->cipherSpec); MEM_CONTEXT_BEGIN(loadData->memContext) { @@ -939,7 +937,8 @@ infoBackupSaveFile( // Write output into a buffer since it needs to be saved to storage twice Buffer *const buffer = bufNew(ioBufferSize()); IoWrite *const write = ioBufferWriteNew(buffer); - cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec); + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = infoBackupFormat(infoBackup)); infoBackupSave(infoBackup, write); // Save the file and make a copy diff --git a/src/info/infoPg.c b/src/info/infoPg.c index ebced1fc02..ea1124c173 100644 --- a/src/info/infoPg.c +++ b/src/info/infoPg.c @@ -189,7 +189,8 @@ infoPgNewLoad( .infoPg = this, }; - this->pub.info = infoNewLoad(read, cipherSpec, infoPgLoadCallback, &loadData); + // The archive.info and backup.info files are the only users of this object and both contain a header + this->pub.info = infoNewLoadP(read, cipherSpec, infoPgLoadCallback, &loadData, .header = true); CHECK(FormatError, !lstEmpty(this->pub.history), "history is missing"); CHECK(FormatError, loadData.currentId > 0, "current id is missing"); diff --git a/src/info/manifest/manifest.c b/src/info/manifest/manifest.c index ebd3ea1678..b46ab149f3 100644 --- a/src/info/manifest/manifest.c +++ b/src/info/manifest/manifest.c @@ -705,7 +705,7 @@ manifestNewLoad(IoRead *const read, const CipherSpec *const cipherSpec) } MEM_CONTEXT_END(); - this->pub.info = infoNewLoad(read, cipherSpec, manifestLoadCallback, &loadData); + this->pub.info = infoNewLoadP(read, cipherSpec, manifestLoadCallback, &loadData); this->pub.data.backrestVersion = infoBackrestVersion(this->pub.info); // Add the label to the reference list in case the manifest was created before 2.42 when the explicit reference list was diff --git a/src/info/manifest/serialize.c.inc b/src/info/manifest/serialize.c.inc index 64bf889b79..02c0a58f31 100644 --- a/src/info/manifest/serialize.c.inc +++ b/src/info/manifest/serialize.c.inc @@ -1039,7 +1039,6 @@ manifestLoadFileCallback(void *const data, const unsigned int try) // Attempt to load the file IoRead *const read = storageReadIo(storageNewReadP(loadData->storage, fileName)); - cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, loadData->cipherSpec); MEM_CONTEXT_BEGIN(loadData->memContext) { diff --git a/src/meson.build b/src/meson.build index 67793b0d86..ff92b09944 100644 --- a/src/meson.build +++ b/src/meson.build @@ -126,6 +126,7 @@ src_pgbackrest = [ 'common/error/retry.c', 'common/exec.c', 'common/fork.c', + 'common/format.c', 'common/ini.c', 'common/io/bufferRead.c', 'common/io/bufferWrite.c', diff --git a/src/version.h b/src/version.h index d1e078206e..eaa579b5ff 100644 --- a/src/version.h +++ b/src/version.h @@ -24,20 +24,6 @@ Config include path name. The parent path will vary based on configuration. ***********************************************************************************************************************************/ #define PROJECT_CONFIG_INCLUDE_PATH "conf.d" -/*********************************************************************************************************************************** -Format Number -- defines format for info and manifest files as well as on-disk structure. Each info file and manifest stores the -format it was written with, so a repository may contain more than one format while older backups and archives expire. - -A constant is defined for each format so that code which varies by format can be explicit about the format it applies to. -REPOSITORY_FORMAT_MIN/MAX are the range that can be read. The allow list for repo-format in build/config.yaml must be kept in sync -with MIN/MAX and its default is the format used for new repositories. -***********************************************************************************************************************************/ -#define REPOSITORY_FORMAT_5 5 -#define REPOSITORY_FORMAT_6 6 - -#define REPOSITORY_FORMAT_MIN REPOSITORY_FORMAT_5 -#define REPOSITORY_FORMAT_MAX REPOSITORY_FORMAT_6 - /*********************************************************************************************************************************** Project version components. PROJECT_VERSION and PROJECT_VERSION_NUM are automatically generated from the component parts. ***********************************************************************************************************************************/ diff --git a/test/define.yaml b/test/define.yaml index 36ea93f94a..1b0777d710 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -358,6 +358,13 @@ unit: - storage/storage - storage/write + # -------------------------------------------------------------------------------------------------------------------------------- + - name: common/format + total: 2 + + coverage: + - common/format + # -------------------------------------------------------------------------------------------------------------------------------- - name: common/crypto total: 4 diff --git a/test/src/harness/backup.c b/test/src/harness/backup.c index 2c442d795e..d83c6a3908 100644 --- a/test/src/harness/backup.c +++ b/test/src/harness/backup.c @@ -243,7 +243,7 @@ hrnBackupPqScript(const unsigned int pgVersion, const time_t backupTimeStart, Hr ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), compressFilterP(param.walCompressType, 1)); // Encrypt with the archive passphrase, which is what archive-push writes WAL with - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( ioWriteFilterGroup(storageWriteIo(write)), cipherModeEncrypt, infoArchiveCipherSpec(infoArchive)); storagePutP(write, walBuffer); diff --git a/test/src/harness/config.h b/test/src/harness/config.h index c129316457..ea25041232 100644 --- a/test/src/harness/config.h +++ b/test/src/harness/config.h @@ -18,7 +18,7 @@ Config option constants #define TEST_CIPHER_PASS_ARCHIVE "xarchivex" // Cipher spec for a test pass, defaulting to the main pass, since most tests encrypt with that -#define TEST_CIPHER_SPEC_PASS(pass) cipherSpecNew(cipherTypeAes256Cbc, BUFSTRZ(pass)) +#define TEST_CIPHER_SPEC_PASS(pass) cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRZ(pass)) #define TEST_CIPHER_SPEC TEST_CIPHER_SPEC_PASS(TEST_CIPHER_PASS) /*********************************************************************************************************************************** diff --git a/test/src/harness/host.c b/test/src/harness/host.c index d82554a0d5..6492c946d8 100644 --- a/test/src/harness/host.c +++ b/test/src/harness/host.c @@ -1232,7 +1232,7 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si hrnHostLocal.storage = strIdFromZ(testDef->stg); hrnHostLocal.compressType = compressTypeEnum(strIdFromZ(testDef->cmp)); hrnHostLocal.cipherSpecMain = - testDef->enc ? cipherSpecNew(cipherTypeAes256Cbc, BUFSTRZ(HRN_CIPHER_PASSPHRASE)) : cipherSpecNewNone(); + testDef->enc ? cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRZ(HRN_CIPHER_PASSPHRASE)) : cipherSpecNewNone(); hrnHostLocal.repoTotal = testDef->rt; hrnHostLocal.tls = testDef->tls; hrnHostLocal.bundle = testDef->bnd; diff --git a/test/src/harness/info.c b/test/src/harness/info.c index d99f178e4c..dd44041e75 100644 --- a/test/src/harness/info.c +++ b/test/src/harness/info.c @@ -6,13 +6,17 @@ Harness for Loading Test Configurations #include #include "common/assert.h" +#include "common/crypto/cipherBlock.h" #include "common/crypto/hash.h" +#include "common/format.h" #include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" #include "common/io/filter/filter.h" #include "common/type/json.h" #include "info/info.h" #include "version.h" +#include "harness/config.h" #include "harness/debug.h" #include "harness/info.h" @@ -116,6 +120,83 @@ harnessInfoChecksum(const String *const info) FUNCTION_HARNESS_RETURN(BUFFER, harnessInfoChecksumFormat(REPOSITORY_FORMAT_DEFAULT, info)); } +/**********************************************************************************************************************************/ +void +hrnInfoPut(const Storage *const storage, const char *const file, const char *const info, HrnInfoPutParam param) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STORAGE, storage); + FUNCTION_HARNESS_PARAM(STRINGZ, file); + FUNCTION_HARNESS_PARAM(STRINGZ, info); + FUNCTION_HARNESS_PARAM(UINT, param.format); + FUNCTION_HARNESS_PARAM(BOOL, param.header); + FUNCTION_HARNESS_PARAM(CIPHER_SPEC, param.cipherSpec); + FUNCTION_HARNESS_PARAM(STRINGZ, param.comment); + FUNCTION_HARNESS_END(); + + ASSERT(info != NULL); + + // Default to the format a new repository is created at + if (param.format == 0) + param.format = REPOSITORY_FORMAT_DEFAULT; + + const Buffer *content = harnessInfoChecksumFormat(param.format, STR(info)); + + // Encrypt the way the format stores the file. A file that contains a header gets it in place of the magic the cipher writes, + // and from format 6 the pass derives with SHA-256 rather than SHA-1. + if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) + { + const bool header = param.header && param.format >= REPOSITORY_FORMAT_6; + Buffer *const encrypted = bufNew(0); + + if (header) + bufCat(encrypted, BUFSTR(strNewFmt("PGBR%03u_", param.format))); + + IoWrite *const write = ioBufferWriteNew(encrypted); + ioFilterGroupAdd( + ioWriteFilterGroup(write), + cipherBlockNewP( + cipherModeEncrypt, + cipherSpecNewP( + cipherSpecType(param.cipherSpec), cipherSpecPass(param.cipherSpec), + .digest = param.format >= REPOSITORY_FORMAT_6 ? hashTypeSha256 : hashTypeSha1), + .raw = header)); + + ioWriteOpen(write); + ioWrite(write, content); + ioWriteClose(write); + + content = encrypted; + } + + hrnStoragePut(storage, file, content, "put info", (HrnStoragePutParam){VAR_PARAM_INIT, .comment = param.comment}); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +Buffer * +harnessInfoEncrypt(const Buffer *const content, const CipherSpec *const cipherSpec) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(BUFFER, content); + FUNCTION_HARNESS_PARAM(CIPHER_SPEC, cipherSpec); + FUNCTION_HARNESS_END(); + + ASSERT(content != NULL); + ASSERT(cipherSpec != NULL); + + Buffer *const result = bufNew(0); + IoWrite *const write = ioBufferWriteNew(result); + ioFilterGroupAdd(ioWriteFilterGroup(write), cipherBlockNewP(cipherModeEncrypt, cipherSpec)); + + ioWriteOpen(write); + ioWrite(write, content); + ioWriteClose(write); + + FUNCTION_HARNESS_RETURN(BUFFER, result); +} + /*********************************************************************************************************************************** Test callback that logs the results to a string ***********************************************************************************************************************************/ diff --git a/test/src/harness/info.h b/test/src/harness/info.h index e1457ee7df..c1470c5722 100644 --- a/test/src/harness/info.h +++ b/test/src/harness/info.h @@ -1,6 +1,7 @@ /*********************************************************************************************************************************** Harness for Generating Test Info Files ***********************************************************************************************************************************/ +#include "common/format.h" #include "common/type/buffer.h" #include "info/info.h" @@ -13,17 +14,27 @@ know this since the option supplies it everywhere else. #define REPOSITORY_FORMAT_DEFAULT REPOSITORY_FORMAT_5 /*********************************************************************************************************************************** -Write info to a file and add the checksum +Write info to a file and add the checksum, storing it the way the format stores it when it is encrypted ***********************************************************************************************************************************/ +typedef struct HrnInfoPutParam +{ + VAR_PARAM_HEADER; + unsigned int format; // Repository format, default format when zero + bool header; // Does the file contain a header, i.e. is it an info file? + const CipherSpec *cipherSpec; // Cipher spec when the file is encrypted, digest set by format + const char *comment; // Comment +} HrnInfoPutParam; + #define HRN_INFO_PUT(storage, file, info, ...) \ do \ { \ hrnTestLogPrefix(__LINE__); \ - hrnStoragePut( \ - storage, file, harnessInfoChecksumZ(info), "put info", (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__}); \ + hrnInfoPut(storage, file, info, (HrnInfoPutParam){VAR_PARAM_INIT, __VA_ARGS__}); \ } \ while (0) +void hrnInfoPut(const Storage *storage, const char *file, const char *info, HrnInfoPutParam param); + /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ @@ -31,4 +42,7 @@ Buffer *harnessInfoChecksum(const String *info); Buffer *harnessInfoChecksumFormat(unsigned int format, const String *info); Buffer *harnessInfoChecksumZ(const char *info); +// Encrypt content the way a file that contains no header is stored, e.g. a manifest +Buffer *harnessInfoEncrypt(const Buffer *content, const CipherSpec *cipherSpec); + void harnessInfoLoadNewCallback(void *callbackData, const String *section, const String *key, JsonRead *json); diff --git a/test/src/harness/storage.c b/test/src/harness/storage.c index 89a5dbdd7f..8b8901f214 100644 --- a/test/src/harness/storage.c +++ b/test/src/harness/storage.c @@ -69,7 +69,17 @@ testStorageGet(const Storage *const storage, const char *const file, const char // Add decrypt filter if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) { - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, param.cipherSpec)); + // Derive with SHA-1 since the harness reads and writes files the way a repository at the format these tests build stores + // them, which is the format that had no header to define anything else. A caller that asked for a digest would not get + // it, so only a spec at the default is accepted. + ASSERT(cipherSpecDigest(param.cipherSpec) == hashTypeSha256); + + ioFilterGroupAdd( + filterGroup, + cipherBlockNewP( + cipherModeDecrypt, + cipherSpecNewP( + cipherSpecType(param.cipherSpec), cipherSpecPass(param.cipherSpec), .digest = hashTypeSha1))); strCatFmt( filter, "enc[%s,%s] ", zNewStrId(cipherSpecType(param.cipherSpec)), @@ -393,7 +403,18 @@ hrnStoragePut( // Add encrypted filter if (param.cipherSpec != NULL && cipherSpecType(param.cipherSpec) != cipherTypeNone) - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, param.cipherSpec)); + { + // Derive with SHA-1 to match how the harness reads these files back. A caller that asked for a digest would not get it, + // so only a spec at the default is accepted. + ASSERT(cipherSpecDigest(param.cipherSpec) == hashTypeSha256); + + ioFilterGroupAdd( + filterGroup, + cipherBlockNewP( + cipherModeEncrypt, + cipherSpecNewP( + cipherSpecType(param.cipherSpec), cipherSpecPass(param.cipherSpec), .digest = hashTypeSha1))); + } // Add file name printf( diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index 853726305b..69afbb2bd4 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -1331,7 +1331,7 @@ testRun(void) blockIncrNew( 3, 3, 8, 2, 4, 5, NULL, compressFilterP(compressTypeGz, 1, .raw = true), cipherBlockNewP( - cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS)), .raw = true)))), + cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS)), .raw = true)))), "block incr pack"); } diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index 53d9fe0d3e..0a1b4a1812 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -311,7 +311,8 @@ testRun(void) TEST_ERROR( restoreFile( strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strZ(repoFileReferenceFull), strZ(repoFile1)), repoIdx, compressTypeGz, - 0, false, false, false, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("badpass")), NULL, fileList), + 0, false, false, false, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("badpass"), .digest = hashTypeSha1), NULL, + fileList), ChecksumError, "error restoring 'normal': actual checksum 'd1cd8a7d11daa26814b93eb604e1d49ab4b43770' does not match expected checksum" " 'ffffffffffffffffffffffffffffffffffffffff'"); @@ -2309,7 +2310,7 @@ testRun(void) Manifest *manifestEncrypted = manifestLoadFile( storageRepoIdxWrite(0), STRDEF(STORAGE_REPO_BACKUP "/" TEST_LABEL "/" BACKUP_MANIFEST_FILE), cipherSpecNewNone()); - manifestCipherSpecSet(manifestEncrypted, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); + manifestCipherSpecSet(manifestEncrypted, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); // Open file for write IoWrite *write = storageWriteIo( @@ -2319,9 +2320,10 @@ testRun(void) // Add encryption filter and save the encrypted manifest #define TEST_CIPHER_PASS_MANIFEST "backpass" - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( ioWriteFilterGroup(write), cipherModeEncrypt, - cipherSpecNew(cfgOptionIdxStrId(cfgOptRepoCipherType, 1), BUFSTRDEF(TEST_CIPHER_PASS_MANIFEST))); + cipherSpecNewP( + cfgOptionIdxStrId(cfgOptRepoCipherType, 1), BUFSTRDEF(TEST_CIPHER_PASS_MANIFEST), .digest = hashTypeSha1)); manifestSave(manifestEncrypted, write); // Write backup.info to the encrypted repo @@ -2332,10 +2334,10 @@ testRun(void) // Write archive.info to the encrypted repo InfoArchive *infoArchive = infoArchiveNew( PG_VERSION_11, 6569239123849665679, REPOSITORY_FORMAT_DEFAULT, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS_ARCHIVE))); infoArchiveSaveFile( infoArchive, storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE_STR, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS))); + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS))); TEST_RESULT_VOID(hrnCmdRestore(), "successful restore"); diff --git a/test/src/module/command/stanzaTest.c b/test/src/module/command/stanzaTest.c index b29ffa9bf6..14408fd141 100644 --- a/test/src/module/command/stanzaTest.c +++ b/test/src/module/command/stanzaTest.c @@ -1,6 +1,8 @@ /*********************************************************************************************************************************** Test Stanza Commands ***********************************************************************************************************************************/ +#include "common/crypto/cipherBlock.h" +#include "common/io/bufferWrite.h" #include "postgres/interface.h" #include "postgres/version.h" #include "storage/posix/storage.h" @@ -23,6 +25,7 @@ testRun(void) Storage *storageHrn = storagePosixNewP(HRN_PATH_STR, .write = true); #define TEST_STANZA "db" + #define TEST_WAL_MIGRATE "archive/db/15-1/000000010000000000000001" #define TEST_STANZA_OTHER "otherstanza" StringList *argListBase = strLstNew(); @@ -137,7 +140,7 @@ testRun(void) TEST_ASSIGN( infoArchive, infoArchiveLoadFile( - storageRepoIdx(1), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), + storageRepoIdx(1), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), "load archive info from encrypted repo2"); TEST_RESULT_UINT(cipherSpecType(infoArchiveCipherSpec(infoArchive)), cipherTypeAes256Cbc, "cipher sub set"); @@ -145,7 +148,7 @@ testRun(void) TEST_ASSIGN( infoBackup, infoBackupLoadFile( - storageRepoIdx(1), INFO_BACKUP_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), + storageRepoIdx(1), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("12345678"))), "load backup info from encrypted repo2"); TEST_RESULT_UINT(cipherSpecType(infoBackupCipherSpec(infoBackup)), cipherTypeAes256Cbc, "cipher sub set"); @@ -185,14 +188,14 @@ testRun(void) TEST_ASSIGN( infoArchive, infoArchiveLoadFile( - storageRepoIdx(3), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), + storageRepoIdx(3), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), "load archive info from encrypted repo4"); TEST_RESULT_UINT(cipherSpecType(infoArchiveCipherSpec(infoArchive)), cipherTypeAes256Cbc, "cipher sub set"); TEST_ASSIGN( infoBackup, infoBackupLoadFile( - storageRepoIdx(3), INFO_BACKUP_PATH_FILE_STR, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), + storageRepoIdx(3), INFO_BACKUP_PATH_FILE_STR, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("87654321"))), "load backup info from encrypted repo4"); TEST_RESULT_UINT(cipherSpecType(infoBackupCipherSpec(infoBackup)), cipherTypeAes256Cbc, "cipher sub set"); @@ -1190,6 +1193,68 @@ testRun(void) hrnCfgEnvKeyRemoveRaw(cfgOptRepoFormat, 1); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("stanza-upgrade - a file encrypted before a format migration is readable after it"); + + // A stanza on an encrypted repository at the format before the digest could be stored + argList = strLstNew(); + hrnCfgArgRawBool(argList, cfgOptOnline, false); + hrnCfgArgRawZ(argList, cfgOptStanza, TEST_STANZA); + hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, TEST_PATH "/repo-migrate"); + hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15"); + hrnCfgArgKeyRawStrId(argList, cfgOptRepoCipherType, 1, cipherTypeAes256Cbc); + hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 1, "12345678"); + HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); + + TEST_RESULT_VOID(cmdStanzaCreate(), "stanza create on encrypted repo"); + TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'db' on repo1"); + + const CipherSpec *const cipherSpecMain = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("12345678")); + const InfoArchive *infoArchiveMigrate = NULL; + + TEST_ASSIGN( + infoArchiveMigrate, infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecMain), + "load archive info before migration"); + + // Write a file with the archive sub-pass, which is how everything below the info files is encrypted + Buffer *const walBuffer = bufNew(0); + IoWrite *const walWrite = ioBufferWriteNew(walBuffer); + + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(walWrite), cipherModeEncrypt, infoArchiveCipherSpec(infoArchiveMigrate)); + ioWriteOpen(walWrite); + ioWrite(walWrite, BUFSTRDEF("WAL BEFORE MIGRATION")); + ioWriteClose(walWrite); + + HRN_STORAGE_PUT(storageRepoIdxWrite(0), TEST_WAL_MIGRATE, walBuffer, .comment = "wal written at format 5"); + + // Migrate the repository + hrnCfgArgKeyRawZ(argList, cfgOptRepoFormat, 1, "6"); + HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList); + + TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - format 6 on encrypted repo"); + TEST_RESULT_LOG( + "P00 INFO: stanza-upgrade for stanza 'db' on repo1\n" + "P00 INFO: upgrade repository format from 5 to 6"); + + // The pass is unchanged by the migration, so a file it encrypted before the migration must still be readable. The digest + // the pass derives with cannot follow the format of the info file storing it or this read would fail. + TEST_ASSIGN( + infoArchiveMigrate, infoArchiveLoadFile(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR, cipherSpecMain), + "load archive info after migration"); + TEST_RESULT_UINT(infoArchiveFormat(infoArchiveMigrate), REPOSITORY_FORMAT_6, "archive info at format 6"); + + StorageRead *const walRead = storageNewReadP(storageRepoIdx(0), STRDEF(TEST_WAL_MIGRATE)); + + cipherBlockFilterGroupAddP( + ioReadFilterGroup(storageReadIo(walRead)), cipherModeDecrypt, infoArchiveCipherSpec(infoArchiveMigrate)); + + TEST_RESULT_STR_Z( + strNewBuf(storageGetP(walRead)), "WAL BEFORE MIGRATION", "wal from before the migration is still readable"); + + hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 1); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("stanza-upgrade - every format that can be read can be requested"); diff --git a/test/src/module/command/verifyTest.c b/test/src/module/command/verifyTest.c index 92740965aa..e4a3abd123 100644 --- a/test/src/module/command/verifyTest.c +++ b/test/src/module/command/verifyTest.c @@ -876,22 +876,43 @@ testRun(void) hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 1, TEST_CIPHER_PASS); HRN_CFG_LOAD(cfgCmdVerify, argList); + // The copy is a copy of the file rather than a second encryption of the same content, which is how the info files are + // saved, i.e. the content is built once in a buffer and that buffer is written twice HRN_INFO_PUT( - storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .cipherSpec = TEST_CIPHER_SPEC, - .comment = "encrypted backup.info"); + storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .header = true, + .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted backup.info"); + HRN_STORAGE_COPY( + storageRepo(), INFO_BACKUP_PATH_FILE, storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted backup.info.copy"); HRN_INFO_PUT( - storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, TEST_NO_CURRENT_BACKUP, - .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted backup.info.copy"); - HRN_INFO_PUT( - storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, .header = true, .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted archive.info"); - HRN_INFO_PUT( - storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, - .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted archive.info.copy"); + HRN_STORAGE_COPY( + storageRepo(), INFO_ARCHIVE_PATH_FILE, storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted archive.info.copy"); TEST_RESULT_VOID(cmdVerify(), "usable encrypted backup and archive info files"); TEST_RESULT_LOG(""); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("encrypted info files at format 6"); + + HRN_INFO_PUT( + storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_NO_CURRENT_BACKUP, .format = REPOSITORY_FORMAT_6, .header = true, + .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted backup.info at format 6"); + HRN_STORAGE_COPY( + storageRepo(), INFO_BACKUP_PATH_FILE, storageRepoWrite(), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted backup.info.copy at format 6"); + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE, .format = REPOSITORY_FORMAT_6, + .header = true, .cipherSpec = TEST_CIPHER_SPEC, .comment = "encrypted archive.info at format 6"); + HRN_STORAGE_COPY( + storageRepo(), INFO_ARCHIVE_PATH_FILE, storageRepoWrite(), INFO_ARCHIVE_PATH_FILE INFO_COPY_EXT, + .comment = "encrypted archive.info.copy at format 6"); + + TEST_RESULT_VOID(cmdVerify(), "usable encrypted backup and archive info files at format 6"); + TEST_RESULT_LOG(""); + hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 1); harnessLogLevelReset(); } @@ -941,12 +962,12 @@ testRun(void) TEST_RESULT_UINT( verifyFile( filePathName, 0, NULL, compressTypeGz, fileChecksum, fileSize, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("pass"))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("pass"), .digest = hashTypeSha1)), verifyOk, "file encrypted compressed ok"); TEST_RESULT_UINT( verifyFile( filePathName, 0, NULL, compressTypeGz, bufNewDecode(encodingHex, STRDEF("aa")), fileSize, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("pass"))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("pass"), .digest = hashTypeSha1)), verifyChecksumMismatch, "file encrypted compressed checksum mismatch"); } diff --git a/test/src/module/common/cryptoTest.c b/test/src/module/common/cryptoTest.c index 93ac692cdd..1706699431 100644 --- a/test/src/module/common/cryptoTest.c +++ b/test/src/module/common/cryptoTest.c @@ -2,9 +2,11 @@ Test Block Cipher ***********************************************************************************************************************************/ #include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" #include "common/io/filter/filter.h" #include "common/io/io.h" #include "common/type/json.h" +#include "version.h" /*********************************************************************************************************************************** Data for testing @@ -72,21 +74,26 @@ testRun(void) // ***************************************************************************************************************************** if (testBegin("CipherBlock")) { - // Cipher error + // Cipher and digest errors // ------------------------------------------------------------------------------------------------------------------------- TEST_ERROR( - cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(strIdFromZ(BOGUS_STR), testPass)), AssertError, + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(strIdFromZ(BOGUS_STR), testPass)), AssertError, "unable to load cipher 'BOGUS'"); + TEST_ERROR( + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass, .digest = strIdFromZ(BOGUS_STR))), + AssertError, "unable to load digest 'BOGUS'"); // Initialization of object // ------------------------------------------------------------------------------------------------------------------------- - // Build from a duplicate to show the copy contains the type and pass of the original + // Build from a duplicate to show the copy contains the type, digest, and pass of the original TEST_RESULT_UINT(cipherSpecType(cipherSpecDup(cipherSpecNewNone())), cipherTypeNone, "dup of none"); - const CipherSpec *const cipherSpec = cipherSpecDup(cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS))); + const CipherSpec *const cipherSpec = cipherSpecDup(cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS))); - TEST_RESULT_UINT(cipherSpecType(cipherSpec), cipherTypeAes256Cbc, "dup type"); - TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(cipherSpec)), TEST_PASS, "dup pass"); + TEST_RESULT_UINT(cipherSpecDigest(cipherSpec), hashTypeSha256, "dup default digest"); + TEST_RESULT_UINT( + cipherSpecDigest(cipherSpecDup(cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF(TEST_PASS), .digest = hashTypeSha1))), + hashTypeSha1, "dup digest"); // A pack contains nothing but the type when there is no cipher PackWrite *packWrite = pckWriteNewP(); @@ -97,17 +104,167 @@ testRun(void) TEST_RESULT_UINT( cipherSpecType(cipherSpecNewPack(pckReadNew(pckWriteResult(packWrite)))), cipherTypeNone, "unpack none"); - // Else it contains the type and pass + // Else it contains the type, digest, and pass. Pack a digest that is not the default so that a pack which loses the digest + // cannot pass by falling back to the default. packWrite = pckWriteNewP(); - cipherSpecPack(packWrite, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + cipherSpecPack(packWrite, cipherSpecNewP(cipherTypeAes256Cbc, testPass, .digest = hashTypeSha1)); pckWriteEndP(packWrite); const CipherSpec *const cipherSpecUnpack = cipherSpecNewPack(pckReadNew(pckWriteResult(packWrite))); TEST_RESULT_UINT(cipherSpecType(cipherSpecUnpack), cipherTypeAes256Cbc, "unpack type"); + TEST_RESULT_UINT(cipherSpecDigest(cipherSpecUnpack), hashTypeSha1, "unpack digest"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(cipherSpecUnpack)), TEST_PASS, "unpack pass"); + // Format header + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("write a header and read the format back from it"); + + Buffer *headerBuffer = bufNew(TEST_BUFFER_SIZE); + IoWrite *headerWrite = ioBufferWriteNew(headerBuffer); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerWrite), + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .format = REPOSITORY_FORMAT_6)); + ioWriteOpen(headerWrite); + ioWrite(headerWrite, testPlainText); + ioWriteClose(headerWrite); + + TEST_RESULT_BOOL( + memcmp(bufPtrConst(headerBuffer), CIPHER_BLOCK_HEADER_MAGIC "006_", CIPHER_BLOCK_MAGIC_SIZE) == 0, true, + "header names the format"); + + // The format is not given on decrypt, so it comes from the header and is what the pass derives with + Buffer *headerResult = bufNew(TEST_BUFFER_SIZE); + IoWrite *headerRead = ioBufferWriteNew(headerResult); + IoFilterGroup *headerFilterGroup = ioWriteFilterGroup(headerRead); + + ioFilterGroupAdd( + headerFilterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true)); + ioWriteOpen(headerRead); + ioWrite(headerRead, headerBuffer); + ioWriteClose(headerRead); + + TEST_RESULT_STR_Z(strNewBuf(headerResult), TEST_PLAINTEXT, "content decrypted with the digest the header called for"); + TEST_RESULT_UINT( + cipherBlockFormat(ioFilterGroupResultP(headerFilterGroup, CIPHER_BLOCK_FILTER_TYPE)), REPOSITORY_FORMAT_6, + "filter reports the format"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a file that begins with the magic was written before there was a header"); + + headerBuffer = bufNew(TEST_BUFFER_SIZE); + headerWrite = ioBufferWriteNew(headerBuffer); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerWrite), + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass, .digest = hashTypeSha1))); + ioWriteOpen(headerWrite); + ioWrite(headerWrite, testPlainText); + ioWriteClose(headerWrite); + + headerResult = bufNew(TEST_BUFFER_SIZE); + headerRead = ioBufferWriteNew(headerResult); + headerFilterGroup = ioWriteFilterGroup(headerRead); + + ioFilterGroupAdd( + headerFilterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true)); + ioWriteOpen(headerRead); + ioWrite(headerRead, headerBuffer); + ioWriteClose(headerRead); + + TEST_RESULT_STR_Z(strNewBuf(headerResult), TEST_PLAINTEXT, "content decrypted"); + TEST_RESULT_UINT( + cipherBlockFormat(ioFilterGroupResultP(headerFilterGroup, CIPHER_BLOCK_FILTER_TYPE)), REPOSITORY_FORMAT_5, + "filter reports the format before the header"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a format given on decrypt must be the one the header names"); + + headerBuffer = bufNew(TEST_BUFFER_SIZE); + headerWrite = ioBufferWriteNew(headerBuffer); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerWrite), + cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .format = REPOSITORY_FORMAT_6)); + ioWriteOpen(headerWrite); + ioWrite(headerWrite, testPlainText); + ioWriteClose(headerWrite); + + IoWrite *const headerMismatch = ioBufferWriteNew(bufNew(TEST_BUFFER_SIZE)); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerMismatch), + cipherBlockNewP( + cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true, + .format = REPOSITORY_FORMAT_5)); + ioWriteOpen(headerMismatch); + + TEST_ERROR( + ioWrite(headerMismatch, headerBuffer), FormatError, "expected repository format 5 but found 6"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("a format given on decrypt that the header agrees with is accepted"); + + headerResult = bufNew(TEST_BUFFER_SIZE); + headerRead = ioBufferWriteNew(headerResult); + + ioFilterGroupAdd( + ioWriteFilterGroup(headerRead), + cipherBlockNewP( + cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true, + .format = REPOSITORY_FORMAT_6)); + ioWriteOpen(headerRead); + ioWrite(headerRead, headerBuffer); + ioWriteClose(headerRead); + + TEST_RESULT_STR_Z(strNewBuf(headerResult), TEST_PLAINTEXT, "content decrypted"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("damaged headers"); + + // Decrypt the buffer above after damaging a byte of the header, which is the same buffer for each case + #define TEST_HEADER_DAMAGE(damageIdx, damageChar, errorType, errorMessage) \ + do \ + { \ + Buffer *const damaged = bufDup(headerBuffer); \ + bufPtr(damaged)[damageIdx] = damageChar; \ + \ + IoWrite *const write = ioBufferWriteNew(bufNew(TEST_BUFFER_SIZE)); \ + ioFilterGroupAdd( \ + ioWriteFilterGroup(write), \ + cipherBlockNewP( \ + cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .header = true)); \ + ioWriteOpen(write); \ + \ + TEST_ERROR(ioWrite(write, damaged), errorType, errorMessage); \ + } \ + while (0) + + // Where the format should be, so this version cannot tell what it is looking at + TEST_HEADER_DAMAGE(CIPHER_BLOCK_HEADER_MAGIC_SIZE, 'X', FormatError, "invalid cipher header"); + + // The byte held back for later, which must be the one this version writes + TEST_HEADER_DAMAGE(CIPHER_BLOCK_MAGIC_SIZE - 1, 'X', FormatError, "invalid cipher header"); + + // A format newer than this version can read, reported before anything is decrypted + TEST_HEADER_DAMAGE( + CIPHER_BLOCK_MAGIC_SIZE - 2, '7', FormatError, + "repository format 7 requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + + // A format older than this version can read + TEST_HEADER_DAMAGE( + CIPHER_BLOCK_MAGIC_SIZE - 2, '4', FormatError, + "repository format 4 is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + + // Neither a header nor the magic, which is what a file that was never encrypted looks like from here + TEST_HEADER_DAMAGE(0, 'X', CryptoError, "cipher header invalid"); + + #undef TEST_HEADER_DAMAGE + CipherBlock *cipherBlock = (CipherBlock *)ioFilterDriver(cipherBlockNewP(cipherModeEncrypt, cipherSpec)); TEST_RESULT_UINT(cipherBlock->mode, cipherModeEncrypt, "mode is valid"); TEST_RESULT_UINT(bufSize(cipherBlock->pass), strlen(TEST_PASS), "passphrase size is valid"); @@ -123,7 +280,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- Buffer *encryptBuffer = bufNew(TEST_BUFFER_SIZE); - IoFilter *blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + IoFilter *blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockEncryptFilter = cipherBlockNewPack(ioFilterParamList(blockEncryptFilter)); CipherBlock *blockEncrypt = (CipherBlock *)ioFilterDriver(blockEncryptFilter); @@ -174,7 +331,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- Buffer *decryptBuffer = bufNew(TEST_BUFFER_SIZE); - IoFilter *blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + IoFilter *blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecryptFilter = cipherBlockNewPack(ioFilterParamList(blockDecryptFilter)); CipherBlock *blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); @@ -194,7 +351,7 @@ testRun(void) // Decrypt in small chunks to test buffering // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -235,7 +392,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("encrypt zero byte file with no magic"); - blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass), .raw = true); + blockEncryptFilter = cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .raw = true); blockEncrypt = (CipherBlock *)ioFilterDriver(blockEncryptFilter); bufUsedZero(encryptBuffer); @@ -248,13 +405,13 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("error on decrypt expecting magic"); - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); TEST_ERROR(ioFilterProcessInOut(blockDecryptFilter, encryptBuffer, decryptBuffer), CryptoError, "cipher header invalid"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("decrypt zero byte file with no magic"); - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass), .raw = true); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass), .raw = true); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -268,7 +425,7 @@ testRun(void) // Invalid cipher header // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); TEST_ERROR( @@ -279,7 +436,7 @@ testRun(void) // Invalid encrypted data cannot be flushed // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -293,7 +450,7 @@ testRun(void) // File with no header should not flush // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -304,7 +461,7 @@ testRun(void) // File with header only should error // ------------------------------------------------------------------------------------------------------------------------- - blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, testPass)); + blockDecryptFilter = cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, testPass)); blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter); bufUsedZero(decryptBuffer); @@ -319,13 +476,13 @@ testRun(void) IoFilterGroup *filterGroup = ioFilterGroupNew(); TEST_RESULT_PTR( - cipherBlockFilterGroupAdd( + cipherBlockFilterGroupAddP( filterGroup, cipherModeEncrypt, cipherSpecNewNone()), filterGroup, " no filter add"); TEST_RESULT_UINT(ioFilterGroupSize(filterGroup), 0, " check no filter add"); TEST_RESULT_VOID( - cipherBlockFilterGroupAdd( - filterGroup, cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("X"))), " filter add"); + cipherBlockFilterGroupAddP( + filterGroup, cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("X"))), " filter add"); TEST_RESULT_UINT(ioFilterGroupSize(filterGroup), 1, " check filter add"); } diff --git a/test/src/module/common/formatTest.c b/test/src/module/common/formatTest.c new file mode 100644 index 0000000000..059dadba71 --- /dev/null +++ b/test/src/module/common/formatTest.c @@ -0,0 +1,38 @@ +/*********************************************************************************************************************************** +Test Repository Format +***********************************************************************************************************************************/ +#include "version.h" + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +static void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + // ***************************************************************************************************************************** + if (testBegin("repoFormatValidate()")) + { + TEST_ERROR( + repoFormatValidate(REPOSITORY_FORMAT_MIN - 1), FormatError, + "repository format 4 is no longer supported by " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + TEST_ERROR( + repoFormatValidate(REPOSITORY_FORMAT_MAX + 1), FormatError, + "repository format 7 requires a newer version of " PROJECT_NAME "\n" + "HINT: " PROJECT_NAME " " PROJECT_VERSION " supports repository format 5 to 6."); + + TEST_RESULT_VOID(repoFormatValidate(REPOSITORY_FORMAT_5), "format 5 is readable"); + TEST_RESULT_VOID(repoFormatValidate(REPOSITORY_FORMAT_6), "format 6 is readable"); + } + + // ***************************************************************************************************************************** + if (testBegin("repoFormatDigest()")) + { + TEST_RESULT_UINT(repoFormatDigest(REPOSITORY_FORMAT_5), hashTypeSha1, "format 5 derives with sha1"); + TEST_RESULT_UINT(repoFormatDigest(REPOSITORY_FORMAT_6), hashTypeSha256, "format 6 derives with sha256"); + } + + FUNCTION_HARNESS_RETURN_VOID(); +} diff --git a/test/src/module/info/infoArchiveTest.c b/test/src/module/info/infoArchiveTest.c index 5a33e9a516..6a8d8f879a 100644 --- a/test/src/module/info/infoArchiveTest.c +++ b/test/src/module/info/infoArchiveTest.c @@ -87,16 +87,23 @@ testRun(void) info, infoArchiveNew( PG_VERSION_10, 6569239123849665999, REPOSITORY_FORMAT_DEFAULT, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), "infoArchiveNew() - cipher sub"); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + contentSave = bufNew(0); - TEST_RESULT_VOID(infoArchiveSave(info, ioBufferWriteNew(contentSave)), "save new with cipher"); + IoWrite *write = ioBufferWriteNew(contentSave); + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = REPOSITORY_FORMAT_DEFAULT); + + TEST_RESULT_VOID(infoArchiveSave(info, write), "save new with cipher"); + TEST_RESULT_BOOL( + strBeginsWithZ(strNewBuf(contentSave), "PGBR"), false, "no header before the format that added it"); TEST_ASSIGN( - info, infoArchiveNewLoad(ioBufferReadNew(contentSave), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))), - "load encrypted archive info"); + info, infoArchiveNewLoad(ioBufferReadNew(contentSave), cipherSpec), "load encrypted archive info"); TEST_RESULT_STR_Z(infoArchiveId(info), "10-1", "archiveId set"); TEST_RESULT_PTR(infoArchivePg(info), infoArchivePg(info), "infoPg set"); TEST_RESULT_STR_Z( diff --git a/test/src/module/info/infoBackupTest.c b/test/src/module/info/infoBackupTest.c index ab1e8a5cee..908164c942 100644 --- a/test/src/module/info/infoBackupTest.c +++ b/test/src/module/info/infoBackupTest.c @@ -87,18 +87,24 @@ testRun(void) TEST_ASSIGN( infoBackup, infoBackupNew( - PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), REPOSITORY_FORMAT_DEFAULT, - cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), + PG_VERSION_10, 6569239123849665999, hrnPgCatalogVersion(PG_VERSION_10), REPOSITORY_FORMAT_6, + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("zWa/6Xtp-IVZC5444yXB+cgFDFl7MxGlgkZSaoPvTGirhPygu4jOKOXf9LO4vjfO"))), "infoBackupNew() - cipher sub"); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + contentSave = bufNew(0); - TEST_RESULT_VOID(infoBackupSave(infoBackup, ioBufferWriteNew(contentSave)), "save new with cipher sub"); + IoWrite *write = ioBufferWriteNew(contentSave); + cipherBlockFilterGroupAddP(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = REPOSITORY_FORMAT_6); + + TEST_RESULT_VOID(infoBackupSave(infoBackup, write), "save new with cipher sub"); + TEST_RESULT_STR_Z( + strNewZN((const char *)bufPtrConst(contentSave), 8), "PGBR006_", "header names the format"); infoBackup = NULL; TEST_ASSIGN( - infoBackup, infoBackupNewLoad(ioBufferReadNew(contentSave), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))), - "load backup info with cipher sub"); + infoBackup, infoBackupNewLoad(ioBufferReadNew(contentSave), cipherSpec), "load backup info with cipher sub"); TEST_RESULT_PTR(infoBackupPg(infoBackup), infoBackupPg(infoBackup), "infoPg set"); TEST_RESULT_STR_Z( strNewBuf(cipherSpecPass(infoBackupCipherSpec(infoBackup))), diff --git a/test/src/module/info/infoPgTest.c b/test/src/module/info/infoPgTest.c index 13a04301b9..18b7578ada 100644 --- a/test/src/module/info/infoPgTest.c +++ b/test/src/module/info/infoPgTest.c @@ -43,7 +43,7 @@ testRun(void) TEST_ASSIGN( infoPg, - infoPgNew(infoPgArchive, REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + infoPgNew(infoPgArchive, REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), "infoPgNew(cipherTypeAes256Cbc, 123xyz)"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 0, " 0 history"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(infoPgInfo(infoPg)))), "123xyz", " cipherPass set"); @@ -80,7 +80,7 @@ testRun(void) TEST_ASSIGN( infoPg, infoPgSet( - infoPgNew(infoPgBackup, REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + infoPgNew(infoPgBackup, REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), PG_VERSION_18, 6569239123849665679, hrnPgCatalogVersion(PG_VERSION_18)), "infoPgSet - infoPgBackup"); TEST_RESULT_INT(infoPgDataTotal(infoPg), 1, " 1 history"); diff --git a/test/src/module/info/infoTest.c b/test/src/module/info/infoTest.c index 2bd306e5d3..e96ef871bd 100644 --- a/test/src/module/info/infoTest.c +++ b/test/src/module/info/infoTest.c @@ -67,6 +67,23 @@ testInfoSaveCallback(void *data, const String *sectionNext, InfoSave *infoSaveDa infoSaveValue(infoSaveData, "d", "key", (String *)data); } +/*********************************************************************************************************************************** +Store info file content the way it would be stored at a format, i.e. with the header and digest that go with the format +***********************************************************************************************************************************/ +static Buffer * +testInfoEncrypt(const Buffer *const content, const unsigned int format, const CipherSpec *const cipherSpec) +{ + Buffer *const result = bufNew(0); + IoWrite *const write = ioBufferWriteNew(result); + cipherBlockFilterGroupAddP(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpec, .format = format); + + ioWriteOpen(write); + ioWrite(write, content); + ioWriteClose(write); + + return result; +} + /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -79,7 +96,7 @@ testRun(void) Info *info = NULL; TEST_ASSIGN( - info, infoNew(REPOSITORY_FORMAT_DEFAULT, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), + info, infoNew(REPOSITORY_FORMAT_DEFAULT, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("123xyz"))), "infoNew(cipher)"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(info))), "123xyz", " cipherPass is set"); @@ -99,7 +116,7 @@ testRun(void) String *callbackContent = strNew(); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), FormatError, "repository format 4 is no longer supported by pgBackRest\n" "HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); @@ -112,7 +129,7 @@ testRun(void) "backrest-format=7\n"); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), FormatError, "repository format 7 requires a newer version of pgBackRest\n" "HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); @@ -124,7 +141,7 @@ testRun(void) "backrest-format=5\n"); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), ChecksumError, "invalid checksum, actual 'a3765a8c2c1e5d35274a0b0ce118f4031faff0bd' but no checksum found"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); @@ -138,7 +155,7 @@ testRun(void) "bogus=\"BOGUS\"\n"); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), ChecksumError, "invalid checksum, actual 'fe989a75dcf7a0261e57d210707c0db741462763' but expected 'BOGUS'"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); @@ -150,7 +167,7 @@ testRun(void) "backrest-version=\"2.17\"\n"); TEST_ERROR( - infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), FormatError, "repository format not found\n" "HINT: is this a valid pgBackRest info file?"); @@ -167,10 +184,10 @@ testRun(void) IoRead *read = ioBufferReadNew(contentLoad); ioFilterGroupAdd( ioReadFilterGroup(read), - cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("X")))); + cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("X")))); TEST_ERROR( - infoNewLoad(read, cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), CryptoError, + infoNewLoadP(read, cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), CryptoError, "cipher header invalid\n" "HINT: is or was the repo encrypted?"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); @@ -185,7 +202,7 @@ testRun(void) callbackContent = strNew(); TEST_ASSIGN( - info, infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + info, infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), "info with other cipher"); TEST_RESULT_STR_Z(callbackContent, "", " check callback content"); TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(info)), cipherTypeNone, " check cipher pass not set"); @@ -202,7 +219,7 @@ testRun(void) callbackContent = strNew(); TEST_ASSIGN( - info, infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + info, infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), "info with content"); TEST_RESULT_STR_Z(callbackContent, "[c] key=1\n[d] key=1\n", " check callback content"); TEST_RESULT_UINT(cipherSpecType(infoCipherSpec(info)), cipherTypeNone, " check cipher pass not set"); @@ -227,7 +244,7 @@ testRun(void) callbackContent = strNew(); TEST_ASSIGN( - info, infoNewLoad(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), + info, infoNewLoadP(ioBufferReadNew(contentLoad), cipherSpecNewNone(), harnessInfoLoadNewCallback, callbackContent), "info format 6"); TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); @@ -255,20 +272,163 @@ testRun(void) callbackContent = strNew(); + const CipherSpec *const cipherSpec = cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")); + + // A file with no header, e.g. a manifest, is decrypted with the spec as it was given since nothing defines the digest + // as anything else TEST_ASSIGN( info, - infoNewLoad( - ioBufferReadNew(contentLoad), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")), harnessInfoLoadNewCallback, + infoNewLoadP( + ioBufferReadNew(harnessInfoEncrypt(contentLoad, cipherSpec)), cipherSpec, harnessInfoLoadNewCallback, callbackContent), "info with content and cipher"); TEST_RESULT_STR_Z(callbackContent, "[c] key=1\n[d] key=1\n", " check callback content"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(infoCipherSpec(info))), "somepass", " check cipher pass set"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); TEST_RESULT_STR_Z(infoBackrestVersion(info), PROJECT_VERSION, " check backrest version"); contentSave = bufNew(0); TEST_RESULT_VOID(infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "info save"); TEST_RESULT_STR(strNewBuf(contentSave), strNewBuf(contentLoad), " check save"); + + // Migrating to a format that stores the digest does not change the digest the pass derives with, since the files the pass + // encrypted before the migration are not rewritten + contentSave = bufNew(0); + + TEST_RESULT_VOID(infoFormatSet(info, REPOSITORY_FORMAT_6), "migrate to format 6"); + TEST_RESULT_VOID( + infoSave(info, ioBufferWriteNew(contentSave), testInfoSaveCallback, strNewZ("1")), "save migrated info"); + TEST_RESULT_BOOL( + strstr(strZ(strNewBuf(contentSave)), "cipher-digest=\"sha1\"") != NULL, true, " check digest stored with pass"); + + TEST_ASSIGN( + info, + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentSave, REPOSITORY_FORMAT_6, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, strNew(), .header = true), + "load migrated info"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest unchanged"); + + // Header + // ------------------------------------------------------------------------------------------------------------------------- + // An unencrypted file has no header no matter the format, since the format is read from the content + contentSave = bufNew(0); + + IoWrite *const writeNone = ioBufferWriteNew(contentSave); + cipherBlockFilterGroupAddP( + ioWriteFilterGroup(writeNone), cipherModeEncrypt, cipherSpecNewNone(), .format = REPOSITORY_FORMAT_6); + + TEST_RESULT_VOID(infoSave(info, writeNone, testInfoSaveCallback, strNewZ("1")), "info save"); + TEST_RESULT_BOOL(strBeginsWithZ(strNewBuf(contentSave), "PGBR"), false, " check no header"); + + contentLoad = harnessInfoChecksumFormat( + REPOSITORY_FORMAT_6, + STRDEF( + "[cipher]\n" + "cipher-digest=\"sha256\"\n" + "cipher-pass=\"somepass\"\n")); + + callbackContent = strNew(); + + TEST_ASSIGN( + info, + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, callbackContent, .header = true), + "info with header"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_6, " check format"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha256, " check cipher sub digest"); + + // A pass migrated from a format that could not store the digest keeps deriving with SHA-1, so the files it encrypted + // before the migration are still readable + const Buffer *const contentMigrated = harnessInfoChecksumFormat( + REPOSITORY_FORMAT_6, + STRDEF( + "[cipher]\n" + "cipher-digest=\"sha1\"\n" + "cipher-pass=\"somepass\"\n")); + + TEST_ASSIGN( + info, + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentMigrated, REPOSITORY_FORMAT_6, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, callbackContent, .header = true), + "info migrated to the format that stores the digest"); + TEST_RESULT_UINT(cipherSpecDigest(infoCipherSpec(info)), hashTypeSha1, " check cipher sub digest"); + + // The content on its own, which is how a caller that wants the file rather than the values in it reads an info file + IoRead *const infoRead = ioBufferReadNew(testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec)); + + ioFilterGroupAdd(ioReadFilterGroup(infoRead), cipherBlockNewP(cipherModeDecrypt, cipherSpec, .header = true)); + ioReadOpen(infoRead); + + TEST_RESULT_STR(strNewBuf(ioReadBuf(infoRead)), strNewBuf(contentLoad), "info content read"); + + // A file written before the header existed is read as the format that had none + contentLoad = harnessInfoChecksumZ("[c]\nkey=1\n"); + + TEST_ASSIGN( + info, + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_5, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, callbackContent, .header = true), + "info with no header"); + TEST_RESULT_UINT(infoFormat(info), REPOSITORY_FORMAT_5, " check format"); + + // A file too short to hold a header cannot have one + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(BUFSTRDEF("PGBR")), cipherSpec, harnessInfoLoadNewCallback, callbackContent, .header = true), + CryptoError, + "cipher header missing\n" + "HINT: is or was the repo encrypted?"); + + // A file with neither header was never encrypted, which is reported the way the cipher reports it + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(BUFSTRDEF("[backrest]\nbackrest-format=5\n")), cipherSpec, harnessInfoLoadNewCallback, + callbackContent, .header = true), + CryptoError, + "cipher header invalid\n" + "HINT: is or was the repo encrypted?"); + + // Header uses the byte held back for later, so this version does not know what it is looking at + Buffer *contentHeader = testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec); + bufPtr(contentHeader)[7] = 'X'; + + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(contentHeader), cipherSpec, harnessInfoLoadNewCallback, callbackContent, .header = true), + FormatError, "invalid cipher header"); + + // Header damaged where the format should be + contentHeader = testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec); + bufPtr(contentHeader)[5] = 'X'; + + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(contentHeader), cipherSpec, harnessInfoLoadNewCallback, callbackContent, .header = true), + FormatError, "invalid cipher header"); + + // Header names a format this version cannot read, which is reported before anything is decrypted + contentHeader = testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec); + bufPtr(contentHeader)[6] = '7'; + + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(contentHeader), cipherSpec, harnessInfoLoadNewCallback, callbackContent, .header = true), + FormatError, + "repository format 7 requires a newer version of pgBackRest\n" + "HINT: pgBackRest " PROJECT_VERSION " supports repository format 5 to 6."); + + // Header and content disagree about the format + TEST_ERROR( + infoNewLoadP( + ioBufferReadNew(testInfoEncrypt(contentLoad, REPOSITORY_FORMAT_6, cipherSpec)), cipherSpec, + harnessInfoLoadNewCallback, callbackContent, .header = true), + FormatError, "repository format 5 does not match header format 6"); } // ***************************************************************************************************************************** diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index 1d97cb2382..ddf657ee7d 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -1453,6 +1453,7 @@ testRun(void) "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" "\n" "[cipher]\n" + "cipher-digest=\"sha256\"\n" "cipher-pass=\"somepass\"\n" "\n" "[target:file]\n" @@ -1478,7 +1479,10 @@ testRun(void) MEM_CONTEXT_TEMP_BEGIN() { TEST_ASSIGN( - manifest, manifestNewLoad(ioBufferReadNew(contentLoad), cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x"))), + manifest, + manifestNewLoad( + ioBufferReadNew(harnessInfoEncrypt(contentLoad, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")))), + cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x"))), "load manifest"); TEST_RESULT_VOID(manifestMove(manifest, memContextPrior()), "move manifest"); } @@ -1781,7 +1785,7 @@ testRun(void) TEST_RESULT_UINT(cipherSpecType(manifestCipherSpec(manifest)), cipherTypeNone, "check cipher subpass"); TEST_RESULT_VOID( - manifestCipherSpecSet(manifest, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("supersecret"))), + manifestCipherSpecSet(manifest, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("supersecret"))), "cipher subpass set"); TEST_RESULT_STR_Z(strNewBuf(cipherSpecPass(manifestCipherSpec(manifest))), "supersecret", "check cipher subpass"); diff --git a/test/src/module/storage/remoteTest.c b/test/src/module/storage/remoteTest.c index b3047f305e..c22ff4d7de 100644 --- a/test/src/module/storage/remoteTest.c +++ b/test/src/module/storage/remoteTest.c @@ -381,8 +381,8 @@ testRun(void) IoFilterGroup *filterGroup = ioReadFilterGroup(storageReadIo(fileRead)); ioFilterGroupAdd(filterGroup, ioSizeNew()); ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1)); - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")))); - ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNew(cipherTypeAes256Cbc, BUFSTRDEF("x")))); + ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")))); + ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherSpecNewP(cipherTypeAes256Cbc, BUFSTRDEF("x")))); ioFilterGroupAdd(filterGroup, compressFilterP(compressTypeGz, 3)); ioFilterGroupAdd(filterGroup, decompressFilterP(compressTypeGz));