WIP: Derive the cipher key with SHA-256 at repository format 6.

build/config.yaml

@@ -1690,8 +1690,8 @@ option:
 
   # Command-line only so that a format left in a configuration file cannot upgrade a repository as a side effect of an unrelated
   # stanza-upgrade. This does not cover the environment, which can set any option that is valid for the command, so a format left
-  # there does migrate on the next stanza-upgrade and the option reference says so. The allow list must be kept in sync with
-  # REPOSITORY_FORMAT_MIN/MAX in version.h
+  # there does migrate on the next stanza-upgrade and the option reference documents it. The allow list must be kept in sync with
+  # REPOSITORY_FORMAT_MIN/MAX in common/format.h
   repo-format:
     group: repo
     type: integer

doc/xml/reference.xml

@@ -2929,7 +2929,7 @@
 
                             <allow-list caption="formats">
                                 <allow-item id="5">Format supported by all versions since <proper>1.00</proper></allow-item>
-                                <allow-item id="6">Adds no features yet and is reserved for features in development</allow-item>
+                                <allow-item id="6">Derives repository encryption keys with <id>SHA-256</id> rather than <id>SHA-1</id></allow-item>
                             </allow-list>
 
                             <p>An existing stanza is migrated to a newer format with the <cmd>stanza-upgrade</cmd> command rather than with this command, which sets the format only for a stanza it creates.</p>

src/command/backup/backup.c

@@ -241,7 +241,7 @@ cmdBackup(void)
 
         // Build an incremental backup if type is not full (manifestPrior will be freed in this call)
         if (!backupBuildIncr(manifest, manifestPrior, backupStartResult.walSegmentName))
-            manifestCipherSpecSet(manifest, cipherSpecGen(cfgOptionStrId(cfgOptRepoCipherType)));
+            manifestCipherSpecSet(manifest, cipherSpecGen(cfgOptionStrId(cfgOptRepoCipherType), manifestFormat(manifest)));
 
         // Set delta if it is not already set and the manifest requires it
         if (!cfgOptionBool(cfgOptDelta) && varBool(manifestData(manifest)->backupOptionDelta))

src/command/backup/complete.c.inc

@@ -25,7 +25,7 @@ backupManifestSaveCopy(Manifest *const manifest, const CipherSpec *const cipherS
                         STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(manifestData(manifest)->backupLabel))));
 
             // Add encryption filter if required
-            cipherBlockFilterGroupAdd(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpecManifest);
+            cipherBlockFilterGroupAddP(ioWriteFilterGroup(write), cipherModeEncrypt, cipherSpecManifest);
 
             // Save file
             manifestSave(manifest, write);
@@ -101,7 +101,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif
                         IoFilterGroup *const filterGroup = ioReadFilterGroup(storageReadIo(read));
 
                         // Decrypt with archive key if encrypted
-                        cipherBlockFilterGroupAdd(
+                        cipherBlockFilterGroupAddP(
                             filterGroup, cipherModeDecrypt, infoArchiveCipherSpec(backupData->archiveInfo));
 
                         // Compress/decompress if archive and backup do not have the same compression settings
@@ -118,7 +118,7 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif
                         }
 
                         // Encrypt with backup key if encrypted
-                        cipherBlockFilterGroupAdd(
+                        cipherBlockFilterGroupAddP(
                             filterGroup, cipherModeEncrypt, manifestCipherSpec(manifest));
 
                         // Add size filter last to calculate repo size
@@ -201,7 +201,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest)
         StorageRead *const manifestRead = storageNewReadP(
             storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupLabel)));
 
-        cipherBlockFilterGroupAdd(
+        cipherBlockFilterGroupAddP(
             ioReadFilterGroup(storageReadIo(manifestRead)), cipherModeDecrypt, infoBackupCipherSpec(infoBackup));
 
         StorageWrite *const manifestWrite = storageNewWriteP(
@@ -212,7 +212,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest)
 
         ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), compressFilterP(compressTypeGz, 9));
 
-        cipherBlockFilterGroupAdd(
+        cipherBlockFilterGroupAddP(
             ioWriteFilterGroup(storageWriteIo(manifestWrite)), cipherModeEncrypt, infoBackupCipherSpec(infoBackup));
 
         storageCopyP(manifestRead, manifestWrite);

src/command/repo/get.c

@@ -16,6 +16,7 @@ Repository Get Command
 #include "config/config.h"
 #include "storage/helper.h"
 
+#include "info/info.h"
 #include "info/infoArchive.h"
 #include "info/infoBackup.h"
 
@@ -47,6 +48,9 @@ storageGetProcess(IoWrite *const destination)
         IoRead *const source = storageReadIo(
             storageNewReadP(storageRepo(), file, .ignoreMissing = cfgOptionBool(cfgOptIgnoreMissing)));
 
+        // Is the file an info file, i.e. one that has a header in front of its content?
+        bool fileIsInfo = false;
+
         // Add decryption if needed
         if (!cfgOptionBool(cfgOptRaw))
         {
@@ -96,6 +100,9 @@ storageGetProcess(IoWrite *const destination)
                                 cfgCipherSpecMain());
                             cipherSpec = infoArchiveCipherSpec(info);
                         }
+                        // Else the file is the archive info, which the repo passphrase opens
+                        else
+                            fileIsInfo = true;
                     }
 
                     // Backup path
@@ -128,6 +135,9 @@ storageGetProcess(IoWrite *const destination)
                             else
                                 cipherSpec = cipherSpecManifest;
                         }
+                        // Else the file is the backup info, which the repo passphrase opens
+                        else
+                            fileIsInfo = true;
                     }
                 }
 
@@ -137,8 +147,9 @@ storageGetProcess(IoWrite *const destination)
 
                 ASSERT(cipherSpecType(cipherSpec) != cipherTypeNone);
 
-                // Add encryption filter
-                cipherBlockFilterGroupAdd(ioReadFilterGroup(source), cipherModeDecrypt, cipherSpec);
+                // Add the decryption filter. An info file is read with a header, which the cipher consumes.
+                ioFilterGroupAdd(
+                    ioReadFilterGroup(source), cipherBlockNewP(cipherModeDecrypt, cipherSpec, .header = fileIsInfo));
             }
         }
 

src/command/restore/timeline.c

@@ -94,7 +94,7 @@ historyLoad(
     {
         const String *const historyFile = strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%08X.history", strZ(archiveId), timeline);
         StorageRead *const storageRead = storageNewReadP(storageRepo, historyFile);
-        cipherBlockFilterGroupAdd(ioReadFilterGroup(storageReadIo(storageRead)), cipherModeDecrypt, cipherSpecArchive);
+        cipherBlockFilterGroupAddP(ioReadFilterGroup(storageReadIo(storageRead)), cipherModeDecrypt, cipherSpecArchive);
         const Buffer *const history = storageGetP(storageRead);
 
         TRY_BEGIN()

src/command/stanza/common.c

@@ -6,6 +6,7 @@ Stanza Commands Handler
 #include "command/check/common.h"
 #include "command/stanza/common.h"
 #include "common/debug.h"
+#include "common/format.h"
 #include "common/log.h"
 #include "config/config.h"
 #include "db/helper.h"
@@ -16,10 +17,11 @@ Stanza Commands Handler
 
 /**********************************************************************************************************************************/
 FN_EXTERN CipherSpec *
-cipherSpecGen(const CipherType cipherType)
+cipherSpecGen(const CipherType cipherType, const unsigned int format)
 {
     FUNCTION_TEST_BEGIN();
         FUNCTION_TEST_PARAM(STRING_ID, cipherType);
+        FUNCTION_TEST_PARAM(UINT, format);
     FUNCTION_TEST_END();
 
     CipherSpec *result;
@@ -33,8 +35,11 @@ cipherSpecGen(const CipherType cipherType)
             uint8_t buffer[48];                                     // 48 is the amount of entropy needed to get a 64 base key
             cryptoRandomBytes(buffer, sizeof(buffer));
 
-            // The pass is the encoded text rather than the bytes it encodes, so it is stored and derived from as that text
-            result = cipherSpecNew(cipherType, BUFSTR(strNewEncode(encodingBase64, BUF(buffer, sizeof(buffer)))));
+            // The pass is the encoded text rather than the bytes it encodes, so it is stored and derived from as that text. The
+            // digest is the one the file it will be stored in is read with, since that is what will derive it later.
+            result = cipherSpecNewP(
+                cipherType, BUFSTR(strNewEncode(encodingBase64, BUF(buffer, sizeof(buffer)))),
+                .digest = repoFormatDigest(format));
             cipherSpecMove(result, memContextPrior());
         }
         MEM_CONTEXT_TEMP_END();

src/command/stanza/common.h

@@ -11,7 +11,8 @@ Stanza Commands Handler
 Functions
 ***********************************************************************************************************************************/
 // Generate cipher spec with a new pass, none when the type is none
-FN_EXTERN CipherSpec *cipherSpecGen(CipherType cipherType);
+// Generate a sub pass for a file at this format, which sets the digest since that is what will derive it when it is read back
+FN_EXTERN CipherSpec *cipherSpecGen(CipherType cipherType, unsigned int format);
 
 // Validate and return database information
 FN_EXTERN PgControl pgValidate(void);

src/command/stanza/create.c

@@ -78,15 +78,16 @@ cmdStanzaCreate(void)
                 const unsigned int format = cfgOptionIdxUInt(cfgOptRepoFormat, repoIdx);
 
                 // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent archive files
-                const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx));
+                const CipherSpec *const cipherSpecArchive = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), format);
 
                 // Create and save archive info
                 infoArchive = infoArchiveNew(pgControl.version, pgControl.systemId, format, cipherSpecArchive);
 
                 infoArchiveSaveFile(infoArchive, storageRepoWriteStanza, INFO_ARCHIVE_PATH_FILE_STR, cfgCipherSpecMainIdx(repoIdx));
 
                 // If the repo is encrypted, generate a cipher passphrase for encrypting subsequent manifests
-                const CipherSpec *const cipherSpecManifest = cipherSpecGen(cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx));
+                const CipherSpec *const cipherSpecManifest = cipherSpecGen(
+                    cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), format);
 
                 // Create and save backup info
                 infoBackup = infoBackupNew(

src/command/verify/verify.c

@@ -167,11 +167,12 @@ verifyInvalidFileAdd(List *const invalidFileList, const VerifyResult reason, con
 Load a file into memory
 ***********************************************************************************************************************************/
 static StorageRead *
-verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherSpec)
+verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherSpec, const bool decrypt)
 {
     FUNCTION_TEST_BEGIN();
         FUNCTION_TEST_PARAM(STRING, pathFileName);                  // Fully qualified path/file name
         FUNCTION_LOG_PARAM(CIPHER_SPEC, cipherSpec);                // Cipher spec to open file if encrypted
+        FUNCTION_TEST_PARAM(BOOL, decrypt);                         // Decrypt here rather than leaving it to the load?
     FUNCTION_TEST_END();
 
     ASSERT(pathFileName != NULL);
@@ -182,7 +183,12 @@ verifyFileLoad(const String *const pathFileName, const CipherSpec *const cipherS
     // *read points to a location within result so update result with contents based on necessary filters
     IoRead *const read = storageReadIo(result);
 
-    cipherBlockFilterGroupAdd(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec);
+    // An info file or manifest is decrypted by the load, which has to read the header before it knows what to decrypt with, so the
+    // checksum is over the file as it is stored. The file and its copy are written from the same bytes, which is all the checksum
+    // is used to compare.
+    if (decrypt)
+        cipherBlockFilterGroupAddP(ioReadFilterGroup(read), cipherModeDecrypt, cipherSpec);
+
     ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
 
     // If the file is compressed, add a decompression filter
@@ -214,7 +220,7 @@ verifyInfoFile(const String *const pathFileName, const bool keepFile, const Ciph
     {
         TRY_BEGIN()
         {
-            IoRead *const infoRead = storageReadIo(verifyFileLoad(pathFileName, cipherSpec));
+            IoRead *const infoRead = storageReadIo(verifyFileLoad(pathFileName, cipherSpec, false));
 
             // If directed to keep the loaded file in memory, then move the file into the result, else drain the io and close it
             if (keepFile)
@@ -827,7 +833,7 @@ verifyArchive(VerifyJobData *const jobData)
                                     strNewFmt(
                                         STORAGE_REPO_ARCHIVE "/%s/%s/%s", strZ(archiveResult->archiveId), strZ(walPath),
                                         strZ(strLstGet(jobData->walFileList, 0))),
-                                    jobData->cipherSpecArchive);
+                                    jobData->cipherSpecArchive, true);
 
                                 const PgWal walInfo = pgWalFromBuffer(
                                     storageGetP(walRead, .exactSize = PG_WAL_HEADER_SIZE), cfgOptionStrNull(cfgOptPgVersionForce));