AES-GCM and ChaCha20-Poly1305: out-of-place encryption from pieces (Rust API)

bench/benches/primitives/aes_gcm.rs

@@ -91,6 +91,60 @@ pub fn bench(c: &mut Criterion) {
         });
         g.finish();
 
+        // Out of place, shaped as a TLS 1.3 record: the plaintext and the
+        // content-type byte after it encrypted into a separate buffer, as
+        // rustls's record layer asks of its provider (aws-lc-rs's
+        // `seal_out_of_place_scatter` with the byte as `extra_in`).
+        let typ = [0x17u8];
+        let mut out = vec![0u8; size + 1];
+        let mut extra = [0u8; 17];
+        let mut g = c.benchmark_group("aes-128-gcm-encrypt-out-of-place");
+        g.throughput(Throughput::Bytes(size as u64));
+        g.bench_function(BenchmarkId::new(VG, size), |b| {
+            b.iter(|| {
+                let k = AesGcm::new(black_box(&key)).unwrap();
+                k.encrypt(
+                    black_box(&nonce),
+                    black_box(&aad),
+                    &[black_box(&data[..]), &typ],
+                    black_box(&mut out),
+                )
+                .unwrap()
+            })
+        });
+        let mut ossl_out = vec![0u8; size + 1 + cipher.block_size()];
+        g.bench_function(BenchmarkId::new(OPENSSL, size), |b| {
+            b.iter(|| {
+                let mut s = Crypter::new(
+                    cipher,
+                    Mode::Encrypt,
+                    black_box(&key),
+                    Some(black_box(&nonce)),
+                )
+                .unwrap();
+                s.aad_update(black_box(&aad)).unwrap();
+                let n = s.update(black_box(&data), &mut ossl_out).unwrap();
+                let n = n + s.update(&typ, &mut ossl_out[n..]).unwrap();
+                s.finalize(&mut ossl_out[n..]).unwrap();
+                s.get_tag(&mut t).unwrap();
+            })
+        });
+        g.bench_function(BenchmarkId::new(AWS_LC, size), |b| {
+            b.iter(|| {
+                aws_lc_key(black_box(&key))
+                    .seal_out_of_place_scatter(
+                        Nonce::assume_unique_for_key(*black_box(&nonce)),
+                        Aad::from(black_box(&aad)),
+                        black_box(&data),
+                        black_box(&mut out[..size]),
+                        &typ,
+                        &mut extra,
+                    )
+                    .unwrap()
+            })
+        });
+        g.finish();
+
         let mut g = c.benchmark_group("aes-128-gcm-decrypt");
         g.throughput(Throughput::Bytes(size as u64));
         g.bench_function(BenchmarkId::new(VG, size), |b| {

bench/benches/primitives/chacha20poly1305.rs

@@ -18,7 +18,7 @@ pub fn bench(c: &mut Criterion) {
 
     use aws_lc_rs::aead::{Aad, CHACHA20_POLY1305, LessSafeKey, Nonce, UnboundKey};
     use criterion::{BenchmarkId, Throughput};
-    use openssl::symm::{Cipher, decrypt_aead, encrypt_aead};
+    use openssl::symm::{Cipher, Crypter, Mode, decrypt_aead, encrypt_aead};
     use verified_garbage::chacha20poly1305::ChaCha20Poly1305;
 
     use crate::{AWS_LC, OPENSSL, SIZES, VG};
@@ -66,6 +66,65 @@ pub fn bench(c: &mut Criterion) {
     }
     g.finish();
 
+    // Out of place, shaped as a TLS 1.3 record: the plaintext and the
+    // content-type byte after it encrypted into a separate buffer, as
+    // rustls's record layer asks of its provider (aws-lc-rs's
+    // `seal_out_of_place_scatter` with the byte as `extra_in`; OpenSSL's
+    // EVP interface always encrypts out of place).
+    let typ = [0x17u8];
+    let mut g = c.benchmark_group("chacha20poly1305-out-of-place");
+    for size in SIZES {
+        g.throughput(Throughput::Bytes(size as u64));
+        let data = vec![0u8; size];
+        let mut out = vec![0u8; size + 1];
+        g.bench_function(BenchmarkId::new(VG, size), |b| {
+            b.iter(|| {
+                ChaCha20Poly1305::new(black_box(&key))
+                    .encrypt(
+                        black_box(&nonce),
+                        black_box(&aad),
+                        &[black_box(&data[..]), &typ],
+                        black_box(&mut out),
+                    )
+                    .unwrap()
+            })
+        });
+        let mut ossl_out = vec![0u8; size + 1];
+        let mut tag = [0u8; 16];
+        g.bench_function(BenchmarkId::new(OPENSSL, size), |b| {
+            b.iter(|| {
+                let mut s = Crypter::new(
+                    Cipher::chacha20_poly1305(),
+                    Mode::Encrypt,
+                    black_box(&key),
+                    Some(black_box(&nonce)),
+                )
+                .unwrap();
+                s.aad_update(black_box(&aad)).unwrap();
+                let n = s.update(black_box(&data), &mut ossl_out).unwrap();
+                let n = n + s.update(&typ, &mut ossl_out[n..]).unwrap();
+                s.finalize(&mut ossl_out[n..]).unwrap();
+                s.get_tag(&mut tag).unwrap();
+            })
+        });
+        let mut extra = [0u8; 17];
+        g.bench_function(BenchmarkId::new(AWS_LC, size), |b| {
+            b.iter(|| {
+                aws_lc_key(black_box(&key))
+                    .seal_out_of_place_scatter(
+                        Nonce::assume_unique_for_key(*black_box(&nonce)),
+                        Aad::from(black_box(&aad)),
+                        black_box(&data),
+                        black_box(&mut out[..size]),
+                        &typ,
+                        &mut extra,
+                    )
+                    .unwrap()
+            })
+        });
+    }
+    g.finish();
+
     let mut g = c.benchmark_group("chacha20poly1305-decrypt");
     for size in SIZES {
         g.throughput(Throughput::Bytes(size as u64));

src/aes_gcm.rs

@@ -18,6 +18,12 @@
 //! fits in memory, [`AesGcm::decrypt_in_place`] checks the tag before it
 //! decrypts.
 //!
+//! [`AesGcm::encrypt`] and [`AesGcmEncryptor::update_into`] encrypt out of
+//! place, from plaintext in buffers of the caller's (in pieces, for
+//! `encrypt`) into an output buffer, as a TLS record layer needs. They gather
+//! the plaintext into the output and encrypt it there with the in-place
+//! functions.
+//!
 //! # Tags
 //!
 //! Encryption returns the full 16-byte tag; a protocol that sends a shorter
@@ -490,6 +496,27 @@ pub enum Error {
     /// was called after `update`: GCM authenticates all of the additional
     /// data before the text.
     AadAfterText,
+    /// The output buffer of [`AesGcm::encrypt`] or
+    /// [`AesGcmEncryptor::update_into`] is not exactly as long as the input.
+    InvalidOutputLength,
+}
+
+/// The total length of the pieces `chunks`, if it fits in a `usize`.
+fn total_len(chunks: &[&[u8]]) -> Option<usize> {
+    chunks
+        .iter()
+        .try_fold(0usize, |n, c| n.checked_add(c.len()))
+}
+
+/// Copies the pieces `chunks` one after the other into `out`, which is
+/// exactly as long as they are in all.
+fn gather(chunks: &[&[u8]], out: &mut [u8]) {
+    let mut rest = out;
+    for c in chunks {
+        let (head, tail) = rest.split_at_mut(c.len());
+        head.copy_from_slice(c);
+        rest = tail;
+    }
 }
 
 /// `len + n`, if it is at most `max`.
@@ -718,6 +745,35 @@ impl AesGcm {
         }
     }
 
+    /// GCM-AE (§7.1), out of place: encrypts the concatenation of the pieces
+    /// of `plaintext` under `nonce`, writes the ciphertext to `out`, and
+    /// returns the 16-byte tag authenticating it and `aad`. A caller holding
+    /// the plaintext in buffers it does not own, such as a TLS record layer
+    /// with a record in several pieces and its content type after them,
+    /// encrypts it without first gathering it into `out` itself.
+    ///
+    /// `out` must be exactly as long as the plaintext (the pieces' lengths
+    /// added up), or this returns [`Error::InvalidOutputLength`] and writes
+    /// nothing. The nonce is as for
+    /// [`encrypt_in_place`](Self::encrypt_in_place).
+    pub fn encrypt(
+        &self,
+        nonce: &[u8],
+        aad: &[u8],
+        plaintext: &[&[u8]],
+        out: &mut [u8],
+    ) -> Result<Block, Error> {
+        // Check every length before writing anything.
+        if total_len(plaintext) != Some(out.len()) {
+            return Err(Error::InvalidOutputLength);
+        }
+        check_nonce(nonce)?;
+        add_len(0, out.len(), MAX_TEXT).map_err(|()| Error::InvalidTextLength)?;
+        add_len(0, aad.len(), MAX_AAD).map_err(|()| Error::InvalidAadLength)?;
+        gather(plaintext, out);
+        self.encrypt_in_place(nonce, aad, out)
+    }
+
     /// Starts encrypting a message under this key and `nonce` (of any
     /// nonzero length; 12 bytes is the recommended one). A nonce must never
     /// be used twice with the same key.
@@ -1071,6 +1127,19 @@ impl AesGcmEncryptor<'_> {
         self.stream.update(data)
     }
 
+    /// Encrypts the next `input.len()` bytes of the plaintext, from `input`,
+    /// into `output`, which must be exactly as long (or this returns
+    /// [`Error::InvalidOutputLength`] and writes nothing).
+    pub fn update_into(&mut self, input: &[u8], output: &mut [u8]) -> Result<(), Error> {
+        if input.len() != output.len() {
+            return Err(Error::InvalidOutputLength);
+        }
+        add_len(self.stream.text_len, input.len(), MAX_TEXT)
+            .map_err(|()| Error::InvalidTextLength)?;
+        output.copy_from_slice(input);
+        self.stream.update(output)
+    }
+
     /// Finishes the message and returns its full 16-byte tag. A protocol
     /// that sends a shorter tag sends its first bytes. GCM leaves no
     /// buffered text to return.
@@ -1139,7 +1208,7 @@ impl AesGcmDecryptor<'_> {
 
 #[cfg(test)]
 mod tests {
-    use super::{AesGcm, Backend, Error, MAX_AAD, MAX_TEXT, add_len, select};
+    use super::{AesGcm, Backend, Error, MAX_AAD, MAX_TEXT, add_len, select, total_len};
     use crate::cpu::{Features, detected};
 
     /// The implementation chosen for each set of features: AES's and
@@ -1815,6 +1884,59 @@ mod tests {
         truncated::<16>();
     }
 
+    /// Out-of-place encryption matches in-place encryption for every split
+    /// of the plaintext into pieces, empty ones included, and its errors
+    /// write nothing.
+    #[test]
+    fn out_of_place() {
+        let k = AesGcm::new(&[3; 16]).unwrap();
+        let nonce = [5u8; 12];
+        let aad = [6u8; 20];
+        let msg: [u8; 70] = core::array::from_fn(|i| (i as u8).wrapping_mul(29));
+        let mut ct = msg;
+        let tag = k.encrypt_in_place(&nonce, &aad, &mut ct).unwrap();
+        for a in 0..=msg.len() {
+            for b in a..=msg.len() {
+                let mut out = [0u8; 70];
+                let pieces = [&msg[..a], &[][..], &msg[a..b], &msg[b..]];
+                assert_eq!(k.encrypt(&nonce, &aad, &pieces, &mut out), Ok(tag));
+                assert_eq!(out, ct);
+            }
+        }
+        // A short piece at a time, as `update_into`.
+        let mut e = k.encryptor(&nonce).unwrap();
+        e.update_aad(&aad).unwrap();
+        let mut out = [0u8; 70];
+        for (x, y) in msg.chunks(3).zip(out.chunks_mut(3)) {
+            e.update_into(x, y).unwrap();
+        }
+        assert_eq!(out, ct);
+        assert_eq!(e.finalize(), tag);
+
+        let mut out = [9u8; 4];
+        let three = [1u8; 3];
+        assert_eq!(
+            k.encrypt(&nonce, &aad, &[&three[..]], &mut out),
+            Err(Error::InvalidOutputLength)
+        );
+        assert_eq!(
+            k.encrypt(&[], &aad, &[&three[..], &[2][..]], &mut out),
+            Err(Error::InvalidNonceLength)
+        );
+        let mut e = k.encryptor(&nonce).unwrap();
+        assert_eq!(
+            e.update_into(&three, &mut out),
+            Err(Error::InvalidOutputLength)
+        );
+        e.stream.text_len = MAX_TEXT;
+        assert_eq!(
+            e.update_into(&three, &mut out[..3]),
+            Err(Error::InvalidTextLength)
+        );
+        assert_eq!(out, [9; 4]);
+        assert_eq!(total_len(&[&three[..], &three[..2]]), Some(5));
+    }
+
     #[test]
     fn errors() {
         assert_eq!(AesGcm::new(&[0; 15]).err(), Some(Error::InvalidKeyLength));

src/chacha20poly1305.rs

@@ -29,6 +29,11 @@
 //! the keystream of short data (up to 960 bytes with AVX-512, 192 with AVX2),
 //! which then takes no second call; every other variant computes it with the
 //! scalar `vg_chacha20_block`.
+//!
+//! [`ChaCha20Poly1305::encrypt`] encrypts out of place, from plaintext in
+//! buffers of the caller's (in pieces) into an output buffer, as a TLS record
+//! layer needs: it gathers the plaintext into the output and encrypts it
+//! there with `vg_chacha20_poly1305_seal`.
 
 #![cfg(any(
     target_arch = "x86_64",
@@ -95,6 +100,9 @@ pub enum Error {
     /// The tag does not match: the ciphertext, the additional data or the
     /// nonce is not what was authenticated under this key.
     TagMismatch,
+    /// The output buffer of [`ChaCha20Poly1305::encrypt`] is not exactly as
+    /// long as the plaintext.
+    InvalidOutputLength,
 }
 
 /// The AEAD with a key.
@@ -177,6 +185,40 @@ impl ChaCha20Poly1305 {
         Ok(tag)
     }
 
+    /// Encrypts the concatenation of the pieces of `plaintext`, with the
+    /// nonce `nonce` and the additional data `aad`, writes the ciphertext to
+    /// `out`, and returns the tag: encryption out of place, for a caller
+    /// holding the plaintext in buffers it does not own, such as a TLS record
+    /// layer with a record in several pieces and its content type after
+    /// them.
+    ///
+    /// `out` must be exactly as long as the plaintext (the pieces' lengths
+    /// added up), or this returns [`Error::InvalidOutputLength`] and writes
+    /// nothing.
+    pub fn encrypt(
+        &self,
+        nonce: &[u8; 12],
+        aad: &[u8],
+        plaintext: &[&[u8]],
+        out: &mut [u8],
+    ) -> Result<[u8; 16], Error> {
+        // Check the lengths before writing anything.
+        let total = plaintext
+            .iter()
+            .try_fold(0usize, |n, c| n.checked_add(c.len()));
+        if total != Some(out.len()) {
+            return Err(Error::InvalidOutputLength);
+        }
+        check_len(out.len())?;
+        let mut rest = &mut out[..];
+        for c in plaintext {
+            let (head, tail) = rest.split_at_mut(c.len());
+            head.copy_from_slice(c);
+            rest = tail;
+        }
+        self.encrypt_in_place(nonce, aad, out)
+    }
+
     /// Decrypts `data` in place, with the nonce `nonce` and the additional
     /// data `aad`, if `tag` authenticates it; otherwise returns
     /// [`Error::TagMismatch`] and zeroes `data`.
@@ -235,6 +277,33 @@ mod tests {
     use crate::cpu::Features;
     use crate::cpu::detected;
 
+    /// Out-of-place encryption matches in-place encryption for every split
+    /// of the plaintext into pieces, empty ones included, and its errors
+    /// write nothing.
+    #[test]
+    fn out_of_place() {
+        let aead = ChaCha20Poly1305::new(&[4; 32]);
+        let nonce = [8; 12];
+        let aad = [2; 9];
+        let msg: [u8; 80] = core::array::from_fn(|i| (i as u8).wrapping_mul(31));
+        let mut ct = msg;
+        let tag = aead.encrypt_in_place(&nonce, &aad, &mut ct).unwrap();
+        for a in 0..=msg.len() {
+            for b in (a..=msg.len()).step_by(7) {
+                let mut out = [0u8; 80];
+                let pieces = [&msg[..a], &[][..], &msg[a..b], &msg[b..]];
+                assert_eq!(aead.encrypt(&nonce, &aad, &pieces, &mut out), Ok(tag));
+                assert_eq!(out, ct);
+            }
+        }
+        let mut out = [9u8; 4];
+        assert_eq!(
+            aead.encrypt(&nonce, &aad, &[&[1, 2, 3][..]], &mut out),
+            Err(Error::InvalidOutputLength)
+        );
+        assert_eq!(out, [9; 4]);
+    }
+
     /// Decryption undoes encryption, and rejects any change to the data, the
     /// additional data, the nonce or the tag, zeroing the data.
     #[test]