diff --git a/src/uu/base32/src/base_common.rs b/src/uu/base32/src/base_common.rs index 79f6435eb46..5a5af236454 100644 --- a/src/uu/base32/src/base_common.rs +++ b/src/uu/base32/src/base_common.rs @@ -680,13 +680,16 @@ pub mod fast_decode { break; } - decode_in_chunks_to_buffer( + // write out whatever got decoded even on error, a partial/lenient + // decode can still be real output + let decode_result = decode_in_chunks_to_buffer( supports_fast_decode_and_encode, &buffer[..aligned_take], decoded_buffer, - )?; + ); write_to_output(decoded_buffer, output)?; + decode_result?; buffer.drain(..aligned_take); } @@ -838,15 +841,16 @@ pub mod fast_decode { output, )?; } else { - while buffer.len() >= decode_in_chunks_of_size { - decode_in_chunks_to_buffer( - supports_fast_decode_and_encode, - &buffer[..decode_in_chunks_of_size], - &mut decoded_buffer, - )?; - write_to_output(&mut decoded_buffer, output)?; - buffer.drain(..decode_in_chunks_of_size); - } + // Flush whatever complete groups are already buffered instead of only whole decode_in_chunks_of_size batches, so already-decodable data isn't silently dropped when we bail out below. + let buffered_len = buffer.len(); + flush_ready_chunks( + &mut buffer, + buffered_len, + valid_multiple, + supports_fast_decode_and_encode, + &mut decoded_buffer, + output, + )?; } return Err(USimpleError::new(1, "error: invalid input")); } @@ -874,16 +878,15 @@ pub mod fast_decode { input.consume(read_len); } - if supports_partial_decode { - flush_ready_chunks( - &mut buffer, - decode_in_chunks_of_size, - valid_multiple, - supports_fast_decode_and_encode, - &mut decoded_buffer, - output, - )?; - } + let buffered_len = buffer.len(); + flush_ready_chunks( + &mut buffer, + buffered_len, + valid_multiple, + supports_fast_decode_and_encode, + &mut decoded_buffer, + output, + )?; if !buffer.is_empty() { let mut owned_chunk: Option> = None; @@ -896,8 +899,10 @@ pub mod fast_decode { let final_chunk = owned_chunk.as_deref().unwrap_or(&buffer); - supports_fast_decode_and_encode.decode_into_vec(final_chunk, &mut decoded_buffer)?; + let decode_result = + supports_fast_decode_and_encode.decode_into_vec(final_chunk, &mut decoded_buffer); write_to_output(&mut decoded_buffer, output)?; + decode_result?; if had_invalid_tail { return Err(USimpleError::new(1, "error: invalid input")); diff --git a/src/uucore/src/lib/features/encoding.rs b/src/uucore/src/lib/features/encoding.rs index db6114d2783..f191ab88a4e 100644 --- a/src/uucore/src/lib/features/encoding.rs +++ b/src/uucore/src/lib/features/encoding.rs @@ -42,6 +42,18 @@ impl Base64SimdWrapper { } } + // fallback for when a quantum has non-zero padding bits: GNU still decodes + // it, just flags the input as invalid afterward + fn decode_with_forgiving(input: &[u8], output: &mut Vec) -> Result<(), ()> { + match base64_simd::forgiving_decode_to_vec(input) { + Ok(decoded_bytes) => { + output.extend_from_slice(&decoded_bytes); + Ok(()) + } + Err(_) => Err(()), + } + } + pub fn new( use_padding: bool, valid_decoding_multiple: usize, @@ -67,65 +79,64 @@ impl SupportsFastDecodeAndEncode for Base64SimdWrapper { } fn decode_into_vec(&self, input: &[u8], output: &mut Vec) -> UResult<()> { - let original_len = output.len(); - - let decode_result = if self.use_padding { - // GNU coreutils keeps decoding even when '=' appears before the true end - // of the stream (e.g. concatenated padded chunks). Mirror that logic - // by splitting at each '='-containing quantum, decoding those 4-byte - // groups with the padded variant, then letting the remainder fall back - // to whichever alphabet fits. - let mut start = 0usize; - while start < input.len() { - let remaining = &input[start..]; - - if remaining.is_empty() { - break; - } + if !self.use_padding { + return Self::decode_with_no_pad(input, output) + .map_err(|_| USimpleError::new(1, "error: invalid input")); + } - if let Some(eq_rel_idx) = remaining.iter().position(|&b| b == b'=') { - let blocks = (eq_rel_idx / 4) + 1; - let segment_len = blocks * 4; + // GNU keeps decoding past a '=' if more data follows (concatenated + // padded chunks), so split at each '=' and decode those 4-byte groups + // one at a time, falling back to whichever alphabet fits for the tail. + // if a quantum's padding bits turn out non-zero, decode it leniently + // anyway instead of dropping it, but stop right there either way - + // GNU never continues past a bad quantum. + let mut start = 0usize; + while start < input.len() { + let remaining = &input[start..]; + + if remaining.is_empty() { + break; + } - if segment_len > remaining.len() { - return Err(USimpleError::new(1, "error: invalid input")); - } + if let Some(eq_rel_idx) = remaining.iter().position(|&b| b == b'=') { + let blocks = (eq_rel_idx / 4) + 1; + let segment_len = blocks * 4; - if Self::decode_with_standard(&remaining[..segment_len], output).is_err() { - return Err(USimpleError::new(1, "error: invalid input")); - } + if segment_len > remaining.len() { + // not enough left to finish this padded quantum, e.g. "NA=" + // would need to be "NA==". GNU still recovers whatever came + // before the '=' by decoding it unpadded + let _ = Self::decode_with_forgiving(&remaining[..eq_rel_idx], output); + return Err(USimpleError::new(1, "error: invalid input")); + } - start += segment_len; - } else { - // If there are no more '=' bytes the tail might still be padded - // (len % 4 == 0) or purposely unpadded (GNU --ignore-garbage or - // concatenated streams), so select the matching alphabet. - let decoder = if remaining.len().is_multiple_of(4) { - Self::decode_with_standard - } else { - Self::decode_with_no_pad - }; - - if decoder(remaining, output).is_err() { - return Err(USimpleError::new(1, "error: invalid input")); - } - - break; + let segment = &remaining[..segment_len]; + if Self::decode_with_standard(segment, output).is_err() { + let _ = Self::decode_with_forgiving(segment, output); + return Err(USimpleError::new(1, "error: invalid input")); } - } - Ok(()) - } else { - Self::decode_with_no_pad(input, output) - .map_err(|_| USimpleError::new(1, "error: invalid input")) - }; + start += segment_len; + } else { + // no more '=' left, so this tail is either still padded (len % 4 + // == 0) or genuinely unpadded (--ignore-garbage, concatenated + // streams) - pick whichever decoder matches + let strict_decoder = if remaining.len().is_multiple_of(4) { + Self::decode_with_standard + } else { + Self::decode_with_no_pad + }; - if let Err(err) = decode_result { - output.truncate(original_len); - Err(err) - } else { - Ok(()) + if strict_decoder(remaining, output).is_err() { + let _ = Self::decode_with_forgiving(remaining, output); + return Err(USimpleError::new(1, "error: invalid input")); + } + + break; + } } + + Ok(()) } fn encode_to_vec_deque(&self, input: &[u8], output: &mut VecDeque) -> UResult<()> { diff --git a/tests/by-util/test_base64.rs b/tests/by-util/test_base64.rs index 8c9dac7ef27..dd3482ab419 100644 --- a/tests/by-util/test_base64.rs +++ b/tests/by-util/test_base64.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore unpadded, QUJD +// spell-checker:ignore unpadded, QUJD, baddecode #[cfg(target_os = "linux")] use uutests::at_and_ucmd; @@ -155,7 +155,8 @@ fn test_garbage() { .arg("-d") .pipe_in(input) .fails() - .stderr_only("base64: error: invalid input\n"); + .stdout_is("hello, world!") + .stderr_is("base64: error: invalid input\n"); } #[test] @@ -171,6 +172,78 @@ fn test_ignore_garbage() { } } +#[test] +fn test_ignore_garbage_decodes_prefix_before_final_error() { + // https://github.com/uutils/coreutils/issues/12923 + // With --ignore-garbage, GNU skips the '.' and keeps decoding the rest of + // the stream, only erroring once it hits the truncated trailing quantum. + let input = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiJjMjQ5MmF"; + new_ucmd!() + .arg("-di") + .pipe_in(input) + .fails() + .stdout_is("{\"alg\":\"RS256\",\"typ\":\"JWT\"}{\"jti\":\"c2492a") + .stderr_is("base64: error: invalid input\n"); +} + +#[test] +fn test_decode_trailing_remainder_canonical() { + // A trailing 2-character group whose unused padding bits are already + // zero decodes cleanly, with no error, even without explicit '=' padding. + new_ucmd!() + .arg("-d") + .pipe_in("aQ") + .succeeds() + .stdout_only("i"); +} + +#[test] +fn test_decode_trailing_remainder_non_canonical() { + // A trailing 2-character group whose unused padding bits are non-zero + // still decodes (GNU doesn't discard the byte it managed to produce), + // but is reported as invalid input. + new_ucmd!() + .arg("-d") + .pipe_in("XY") + .fails() + .stdout_is("]") + .stderr_is("base64: error: invalid input\n"); +} + +#[test] +fn test_decode_explicit_padding_non_canonical() { + // Same leniency as the trailing-remainder case, but for a quantum that's + // already a full 4 characters with an explicit '=' (from GNU's own + // basenc/base64.pl: baddecode6/baddecode7). + new_ucmd!() + .arg("-d") + .pipe_in("SB==") + .fails() + .stdout_is("H") + .stderr_is("base64: error: invalid input\n"); + + new_ucmd!() + .arg("-d") + .pipe_in("SGVsbG9=") + .fails() + .stdout_is("Hello") + .stderr_is("base64: error: invalid input\n"); +} + +#[test] +fn test_decode_insufficient_padding_recovers_preceding_data() { + // A trailing '=' without enough characters left to complete the padded + // quantum (e.g. "NA=" would need to be "NA==") isn't a valid quantum at + // all, but GNU still recovers whatever precedes it by decoding that much + // as an unpadded remainder (from GNU's own basenc/base64.pl: baddecode8). + new_ucmd!() + .arg("-d") + .pipe_in("MTIzNA=") + .fails() + .stdout_is("1234") + .stderr_is("base64: error: invalid input\n"); +} + #[test] fn test_wrap() { for wrap_param in ["-w", "--wrap", "--wr"] { diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 40218a8d63c..861747c1607 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -248,11 +248,6 @@ sed -i -e "s|invalid suffix in --pages argument|invalid --pages argument|" \ -e "s|--pages argument '\$too_big' too large|invalid --pages argument '\$too_big'|" \ -e "s|invalid page range|invalid --pages argument|" tests/misc/xstrtol.pl -# When decoding an invalid base32/64 string, gnu writes everything it was able to decode until -# it hit the decode error, while we don't write anything if the input is invalid. -sed -i "s/\(baddecode.*OUT=>\"\).*\"/\1\"/g" tests/basenc/base64.pl -sed -i "s/\(\(b2[ml]_[69]\|z85_8\|z85_35\).*OUT=>\)[^}]*\(.*\)/\1\"\"\3/g" tests/basenc/basenc.pl - # add "error: " to the expected error message sed -i "s/\$prog: invalid input/\$prog: error: invalid input/g" tests/basenc/basenc.pl