diff --git a/datafusion/functions/benches/encoding.rs b/datafusion/functions/benches/encoding.rs index 0b8f0c5c51a58..451baff5183fd 100644 --- a/datafusion/functions/benches/encoding.rs +++ b/datafusion/functions/benches/encoding.rs @@ -27,10 +27,35 @@ use std::sync::Arc; fn criterion_benchmark(c: &mut Criterion) { let decode = encoding::decode(); + let encode = encoding::encode(); let config_options = Arc::new(ConfigOptions::default()); for size in [1024, 4096, 8192] { let bin_array = Arc::new(create_binary_array::(size, 0.2)); + + c.bench_function(&format!("hex_encode/{size}"), |b| { + let method = ColumnarValue::Scalar("hex".into()); + let arg_fields = vec![ + Field::new("a", bin_array.data_type().to_owned(), true).into(), + Field::new("b", method.data_type().to_owned(), true).into(), + ]; + let args = vec![ColumnarValue::Array(bin_array.clone()), method]; + let return_field = Field::new("f", DataType::Utf8, true).into(); + + b.iter(|| { + black_box( + encode + .invoke_with_args(ScalarFunctionArgs { + args: args.clone(), + arg_fields: arg_fields.clone(), + number_rows: size, + return_field: Arc::clone(&return_field), + config_options: Arc::clone(&config_options), + }) + .unwrap(), + ) + }) + }); c.bench_function(&format!("base64_decode/{size}"), |b| { let method = ColumnarValue::Scalar("base64".into()); let encoded = encoding::encode() diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs index 877acbb529920..027ec8e5e59ab 100644 --- a/datafusion/functions/src/encoding/inner.rs +++ b/datafusion/functions/src/encoding/inner.rs @@ -410,11 +410,7 @@ impl Encoding { .collect(); Ok(Arc::new(array)) } - Self::Hex => { - let array: GenericStringArray = - array.iter().map(|x| x.map(hex::encode)).collect(); - Ok(Arc::new(array)) - } + Self::Hex => hex_encode_array::<_, OutputOffset>(array), } } @@ -459,6 +455,45 @@ impl Encoding { } } +/// Hex-encode a binary array into a string array, writing the lowercase hex +/// digits directly into a single pre-sized value buffer. Each input byte maps +/// to exactly two hex characters, so the output size is known up front and no +/// per-element `String` is allocated. +fn hex_encode_array<'a, InputBinaryArray, OutputOffset>( + array: &InputBinaryArray, +) -> Result +where + InputBinaryArray: BinaryArrayType<'a>, + OutputOffset: OffsetSizeTrait, +{ + let total_input_bytes: usize = array.iter().flatten().map(|v| v.len()).sum(); + + let mut values = vec![0u8; total_input_bytes * 2]; + let mut offsets = Vec::::with_capacity(array.len() + 1); + offsets.push(OutputOffset::zero()); + + let mut pos = 0usize; + for v in array.iter() { + if let Some(v) = v { + let out_len = v.len() * 2; + // The slice is sized to exactly `2 * v.len()`, which is the only + // condition under which `encode_to_slice` can fail, so this cannot + // error. + hex::encode_to_slice(v, &mut values[pos..pos + out_len]) + .map_err(|e| exec_datafusion_err!("Failed to encode to hex: {e}"))?; + pos += out_len; + } + offsets.push(OutputOffset::usize_as(pos)); + } + + let array = GenericStringArray::::try_new( + OffsetBuffer::new(offsets.into()), + Buffer::from_vec(values), + array.nulls().cloned(), + )?; + Ok(Arc::new(array)) +} + fn delegated_decode<'a, DecodeFunction, InputBinaryArray, OutputOffset>( decode: DecodeFunction, input: &InputBinaryArray,