Thread panics instead of errors

[blaginin]

Issue Description

use futures_util::TryStreamExt;
use itertools::Itertools;
use std::sync::Arc;
use vortex::VortexSessionDefault;
use vortex::dtype::datetime::TimeUnit;
use vortex::dtype::{DType, ExtDType, Nullability, PType};
use vortex::expr::{gt, root};
use vortex::scalar::Scalar;
use vortex::session::VortexSession;
use vortex_array::IntoArray;
use vortex_array::arrays::{PrimitiveArray, TemporalArray};
use vortex_array::expr::lit;
use vortex_file::{OpenOptionsSessionExt, WriteOptionsSessionExt};

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let session = VortexSession::default();

    // Write file with MILLISECONDS timestamps
    let ts_array = PrimitiveArray::from_iter(vec![1704067200000i64, 1704153600000, 1704240000000])
        .into_array();
    let temporal = TemporalArray::new_timestamp(ts_array, TimeUnit::Milliseconds, None);

    let mut buf = vortex::buffer::ByteBufferMut::empty();
    session
        .write_options()
        .write(&mut buf, temporal.into_array().to_array_stream())
        .await?;

    // Read with SECONDS filter scalar
    let seconds_ext_dtype = Arc::new(ExtDType::new(
        vortex::dtype::datetime::TIMESTAMP_ID.clone(),
        Arc::new(DType::Primitive(PType::I64, Nullability::Nullable)),
        Some(vortex::dtype::datetime::TemporalMetadata::Timestamp(TimeUnit::Seconds, None).into()),
    ));
    let filter_expr = gt(
        root(),
        lit(Scalar::extension(
            seconds_ext_dtype,
            Scalar::from(1704153600i64),
        )),
    );

    let stream = session
        .open_options()
        .open_buffer(buf)?
        .scan()?
        .with_filter(filter_expr)
        .into_array_stream()?;

    let results = stream.try_collect::<Vec<_>>().await;

    let err = results.err().unwrap();
    println!("Expected error: {}", err);

    Ok(())
}

This cases panic with:

thread 'tokio-runtime-worker' panicked at /Users/blaginin/Documents/vortex/vortex-error/src/lib.rs:334:33:
execute cannot fail:
  Cannot compare different DTypes ext(vortex.timestamp, i64, ExtMetadata([2, 0, 0]))? and ext(vortex.timestamp, i64, ExtMetadata([3, 0, 0]))?

Expected Behavior

No panic, get the result as err

Actual Behavior

panic

Reproduction Steps

Run the script above on the latest develop

OS Version Information

MacOS Tahoe

I acknowledge that:

• I have searched the Vortex repository (both open and closed Discussions and Issues) and confirm this is not a duplicate of an existing issue or discussion.
• I have checked the "Preview" tab on all text fields to ensure that everything looks right, and have wrapped all configuration and code in code blocks with a group of three backticks (```) on separate lines.

[robert3005]

What Dima is asking is for the operation to return error instead of panicking. Function signature suggests it can return errors

[connortsui20]

Oh right, I can't read

[connortsui20]

So the real issue is this:

vortex/vortex-array/src/arrays/scalar_fn/vtable/validity.rs

Line 40 in a30298b

.vortex_expect("execute cannot fail");

Also just for future reference can you paste in the imports and main function so reviewers don't have to do this manually 🙏?

Details

use std::sync::Arc;

use futures::TryStreamExt;
use vortex::VortexSessionDefault;
use vortex::array::arrays::PrimitiveArray;
use vortex::array::arrays::TemporalArray;
use vortex::buffer::ByteBufferMut;
use vortex::dtype::DType;
use vortex::dtype::ExtDType;
use vortex::dtype::Nullability;
use vortex::dtype::PType;
use vortex::dtype::datetime::TimeUnit;
use vortex::expr::gt;
use vortex::expr::lit;
use vortex::expr::root;
use vortex::scalar::Scalar;
use vortex::session::VortexSession;
use vortex_array::IntoArray;
use vortex_file::OpenOptionsSessionExt;
use vortex_file::WriteOptionsSessionExt;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
    let session = VortexSession::default();

    // Write file with MILLISECONDS timestamps
    let ts_array = PrimitiveArray::from_iter(vec![1704067200000i64, 1704153600000, 1704240000000])
        .into_array();
    let temporal = TemporalArray::new_timestamp(ts_array, TimeUnit::Milliseconds, None);

    let mut buf = ByteBufferMut::empty();
    session
        .write_options()
        .write(&mut buf, temporal.into_array().to_array_stream())
        .await?;

    // Read with SECONDS filter scalar
    let seconds_ext_dtype = Arc::new(ExtDType::new(
        vortex::dtype::datetime::TIMESTAMP_ID.clone(),
        Arc::new(DType::Primitive(PType::I64, Nullability::Nullable)),
        Some(vortex::dtype::datetime::TemporalMetadata::Timestamp(TimeUnit::Seconds, None).into()),
    ));
    let filter_expr = gt(
        root(),
        lit(Scalar::extension(
            seconds_ext_dtype,
            Scalar::from(1704153600i64),
        )),
    );

    let stream = session
        .open_options()
        .open_buffer(buf)?
        .scan()?
        .with_filter(filter_expr)
        .into_array_stream()?;

    let results = stream.try_collect::<Vec<_>>().await;

    #[expect(clippy::err_expect)]
    let err = results.err().expect("not error");
    println!("Expected error: {}", err);

    Ok(())
}

[connortsui20]

@joseph-isaacs Not really sure how to interpret this expect message... Should we just raise the error or should we be catching this dtype mismatch earlier before the execute?

[joseph-isaacs]

fixed in #6006

[blaginin]

on the latest develop, I am still getting:

thread 'tokio-runtime-worker' panicked at /Users/blaginin/Documents/vortex/vortex-error/src/lib.rs:334:33:
execute cannot fail:
  Cannot compare different DTypes ext(vortex.timestamp, i64, ExtMetadata([2, 0, 0]))? and ext(vortex.timestamp, i64, ExtMetadata([3, 0, 0]))?

[joseph-isaacs]

Working on it.

[joseph-isaacs]

We should validate the extDtype in the ctor

[robert3005]

We have fixed most of this