Skip to content

[refine](function) Unify nullable array access with ColumnArrayView #66234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Mryange wants to merge 3 commits into
base: master
Choose a base branch
from
Open

[refine](function) Unify nullable array access with ColumnArrayView #66234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
739a68f
upd
Mryange Jul 29, 2026
25e2326
[fix](be) Include decimal column for array apply
Mryange Jul 29, 2026
992265d
[fix](be) Preserve COW ownership in array enumerate uniq
Mryange Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 30 additions & 37 deletions be/src/exprs/function/array/function_array_apply.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@
#include "core/call_on_type_index.h"
#include "core/column/column.h"
#include "core/column/column_array.h"
#include "core/column/column_array_view.h"
#include "core/column/column_const.h"
#include "core/column/column_nullable.h"
#include "core/column/column_decimal.h"
#include "core/column/column_vector.h"
#include "core/data_type/data_type.h"
#include "core/data_type/data_type_array.h"
Expand Down Expand Up @@ -82,7 +83,6 @@ class FunctionArrayApply : public IFunction {
fmt::format("unsupported types for function {}({})", get_name(),
block.get_by_position(arguments[0]).type->get_name()));
}
const auto& src_offsets = src_column_array->get_offsets();
const auto* src_nested_column = &src_column_array->get_data();
DCHECK(src_nested_column != nullptr);

Expand All @@ -95,7 +95,7 @@ class FunctionArrayApply : public IFunction {
static_cast<const ColumnConst&>(*block.get_by_position(arguments[2]).column.get());
ColumnPtr result_ptr;
RETURN_IF_CATCH_EXCEPTION(
RETURN_IF_ERROR(_execute(*src_nested_column, nested_type, src_offsets, condition,
RETURN_IF_ERROR(_execute(src_column, *src_nested_column, nested_type, condition,
rhs_value_column, &result_ptr)));
block.replace_by_position(result, std::move(result_ptr));
return Status::OK();
Expand Down Expand Up @@ -135,37 +135,30 @@ class FunctionArrayApply : public IFunction {
}

// need exception safety
template <typename T, ApplyOp op>
ColumnPtr _apply_internal(const IColumn& src_column, const ColumnArray::Offsets64& src_offsets,
template <PrimitiveType PType, ApplyOp op>
ColumnPtr _apply_internal(const ColumnArrayView<PType>& array_view, const IColumn& src_column,
const ColumnConst& cmp) const {
using T = typename PrimitiveTypeTraits<PType>::CppType;
T rhs_val = *reinterpret_cast<const T*>(cmp.get_data_at(0).data);
auto column_filter = ColumnUInt8::create(src_column.size(), 0);
auto& column_filter_data = column_filter->get_data();
const char* src_column_data_ptr = nullptr;
const uint8_t* null_map_data = nullptr;
if (!is_column_nullable(src_column)) {
src_column_data_ptr = src_column.get_raw_data().data;
} else {
const auto* nullable_col = assert_cast<const ColumnNullable*>(&src_column);
src_column_data_ptr = nullable_col->get_nested_column().get_raw_data().data;
null_map_data = nullable_col->get_null_map_data().data();
}
const T* src_column_data_t_ptr = reinterpret_cast<const T*>(src_column_data_ptr);
const T* src_column_data_t_ptr = reinterpret_cast<const T*>(array_view.get_data());
const UInt8* null_map_data = array_view.get_null_map_data();
const size_t src_column_size = src_column.size();
for (size_t i = 0; i < src_column_size; ++i) {
if (null_map_data && null_map_data[i]) {
if (null_map_data[i]) {
continue; // null elements should not pass the filter
}
column_filter_data[i] = apply<T, op>(src_column_data_t_ptr[i], rhs_val);
}
const IColumn::Filter& filter = column_filter_data;
ColumnPtr filtered = src_column.filter(filter, src_column.size());
auto column_offsets = ColumnArray::ColumnOffsets::create(src_offsets.size());
auto column_offsets = ColumnArray::ColumnOffsets::create(array_view.offsets.size());
ColumnArray::Offsets64& dst_offsets = column_offsets->get_data();
size_t in_pos = 0;
size_t out_pos = 0;
for (size_t i = 0; i < src_offsets.size(); ++i) {
for (; in_pos < src_offsets[i]; ++in_pos) {
for (size_t i = 0; i < array_view.offsets.size(); ++i) {
for (; in_pos < array_view.offsets[i]; ++in_pos) {
if (filter[in_pos]) {
++out_pos;
}
Expand All @@ -176,14 +169,14 @@ class FunctionArrayApply : public IFunction {
}

template <ApplyOp OP>
void dispatch_array_scalar(DataTypePtr nested_type, const IColumn& src_column,
const ColumnArray::Offsets64& src_offsets, const ColumnConst& cmp,
void dispatch_array_scalar(const ColumnPtr& array_column, DataTypePtr nested_type,
const IColumn& src_column, const ColumnConst& cmp,
ColumnPtr* dst) const {
auto call = [&](const auto& type) -> bool {
using DispatchType = std::decay_t<decltype(type)>;
constexpr PrimitiveType PType = DispatchType::PType;
*dst = _apply_internal<typename PrimitiveTypeTraits<PType>::CppType, OP>(
src_column, src_offsets, cmp);
auto array_view = ColumnArrayView<PType>::create(array_column);

@github-actions github-actions Bot Jul 29, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This new view instantiation currently breaks the BE build for decimal arrays. dispatch_switch_scalar instantiates ColumnArrayView<TYPE_DECIMAL32/64/V2> here, but this translation unit does not include the complete ColumnDecimal definition, so the live macOS BE UT fails in column_execute_util.h with implicit instantiation of undefined template 'doris::ColumnDecimal<...>'. Please include core/column/column_decimal.h here (or make the view header provide the concrete type it requires) so every supported scalar instantiation compiles.

*dst = _apply_internal<PType, OP>(array_view, src_column, cmp);
return true;
};

Expand All @@ -195,27 +188,27 @@ class FunctionArrayApply : public IFunction {
}
}
// need exception safety
Status _execute(const IColumn& nested_src, DataTypePtr nested_type,
const ColumnArray::Offsets64& offsets, const std::string& condition,
Status _execute(const ColumnPtr& array_column, const IColumn& nested_src,
DataTypePtr nested_type, const std::string& condition,
const ColumnConst& rhs_value_column, ColumnPtr* dst) const {
if (condition == "=") {
dispatch_array_scalar<ApplyOp::EQ>(nested_type, nested_src, offsets, rhs_value_column,
dst);
dispatch_array_scalar<ApplyOp::EQ>(array_column, nested_type, nested_src,
rhs_value_column, dst);
} else if (condition == "!=") {
dispatch_array_scalar<ApplyOp::NE>(nested_type, nested_src, offsets, rhs_value_column,
dst);
dispatch_array_scalar<ApplyOp::NE>(array_column, nested_type, nested_src,
rhs_value_column, dst);
} else if (condition == "<") {
dispatch_array_scalar<ApplyOp::LT>(nested_type, nested_src, offsets, rhs_value_column,
dst);
dispatch_array_scalar<ApplyOp::LT>(array_column, nested_type, nested_src,
rhs_value_column, dst);
} else if (condition == "<=") {
dispatch_array_scalar<ApplyOp::LE>(nested_type, nested_src, offsets, rhs_value_column,
dst);
dispatch_array_scalar<ApplyOp::LE>(array_column, nested_type, nested_src,
rhs_value_column, dst);
} else if (condition == ">") {
dispatch_array_scalar<ApplyOp::GT>(nested_type, nested_src, offsets, rhs_value_column,
dst);
dispatch_array_scalar<ApplyOp::GT>(array_column, nested_type, nested_src,
rhs_value_column, dst);
} else if (condition == ">=") {
dispatch_array_scalar<ApplyOp::GE>(nested_type, nested_src, offsets, rhs_value_column,
dst);
dispatch_array_scalar<ApplyOp::GE>(array_column, nested_type, nested_src,
rhs_value_column, dst);
} else {
return Status::RuntimeError(
fmt::format("execute failed, unsupported op {} for function {})", condition,
Expand Down
53 changes: 12 additions & 41 deletions be/src/exprs/function/array/function_array_count.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
// specific language governing permissions and limitations
// under the License.

#include "core/column/column_array.h"
#include "core/column/column_nullable.h"
#include "core/data_type/data_type_array.h"
#include "core/column/column_array_view.h"
#include "core/data_type/data_type_number.h"
#include "exprs/function/function.h"
#include "exprs/function/function_helpers.h"
Expand Down Expand Up @@ -48,52 +46,25 @@ class FunctionArrayCount : public IFunction {

Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const override {
const auto& [src_column, src_const] =
unpack_if_const(block.get_by_position(arguments[0]).column);
const ColumnArray* array_column = nullptr;
const UInt8* array_null_map = nullptr;
if (const auto* nullable_array = check_and_get_column<ColumnNullable>(src_column.get())) {
array_column = assert_cast<const ColumnArray*>(&nullable_array->get_nested_column());
array_null_map = nullable_array->get_null_map_column().get_data().data();
} else {
array_column = assert_cast<const ColumnArray*>(src_column.get());
}

if (!array_column) {
return Status::RuntimeError("unsupported types for function {}({})", get_name(),
block.get_by_position(arguments[0]).type->get_name());
}

const auto& offsets = array_column->get_offsets();
ColumnPtr nested_column = nullptr;
const UInt8* nested_null_map = nullptr;
if (is_column_nullable(array_column->get_data())) {
const auto& nested_null_column =
assert_cast<const ColumnNullable&>(array_column->get_data());
nested_null_map = nested_null_column.get_null_map_column().get_data().data();
nested_column = nested_null_column.get_nested_column_ptr();
} else {
nested_column = array_column->get_data_ptr();
}

const auto& nested_data = assert_cast<const ColumnUInt8&>(*nested_column).get_data();

auto dst_column = ColumnInt64::create(offsets.size());
auto array_view =
ColumnArrayView<TYPE_BOOLEAN>::create(block.get_by_position(arguments[0]).column);
auto dst_column = ColumnInt64::create(array_view.size());
auto& dst_data = dst_column->get_data();

for (size_t row = 0; row < offsets.size(); ++row) {
for (size_t row = 0; row < array_view.size(); ++row) {
Int64 res = 0;
if (array_null_map && array_null_map[row]) {
if (array_view.is_null_at(row)) {
dst_data[row] = res;
continue;
}
size_t off = offsets[row - 1];
size_t len = offsets[row] - off;
for (size_t pos = 0; pos < len; ++pos) {
if (nested_null_map && nested_null_map[pos + off]) {
auto array_data = array_view[row];
const auto* data = array_data.get_data();
const auto* null_map = array_data.get_null_map_data();
for (size_t pos = 0; pos < array_data.size(); ++pos) {
if (null_map[pos]) {
continue;
}
if (nested_data[pos + off] != 0) {
if (data[pos] != 0) {
++res;
}
}
Expand Down
Loading
Loading