From fe1ae05df451bdaa3c3871d80374f134dd7a352c Mon Sep 17 00:00:00 2001 From: TengJianPing Date: Tue, 7 Jul 2026 17:35:24 +0800 Subject: [PATCH] [fix](be) Avoid scan executor shutdown use after free (#65220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue Number: None Related PR: None Problem Summary: UBSAN/ASAN reported a heap-use-after-free during BE shutdown in TimeSharingTaskExecutor destruction. The scan executor owned pending split runners, while the split runner function held ScannerContext, and ScannerContext cleaned up its task by copying the executor shared_ptr back from the scheduler. When the scheduler was already destroying that shared_ptr, copying the same shared_ptr member could re-enter the executor destruction path and release members such as MetricEntity twice. Keep ScannerContext cleanup tied to a weak reference of the executor captured when the task is created, and avoid capturing the scheduler this pointer in split cleanup lambdas. 旧代码的链路大致是： 1. TaskExecutorSimplifiedScanScheduler 析构，成员 _task_executor 是最后阶段要销毁的 std::shared_ptr。 2. 这个 shared_ptr 引用计数降到 0，于是开始执行 TimeSharingTaskExecutor::~TimeSharingTaskExecutor()。 3. TimeSharingTaskExecutor 析构时会释放它持有的 split/task 容器。 4. split 里有 ScannerSplitRunner，它的 std::function 捕获了 std::shared_ptr。 5. 释放 split 会释放这个 captured ScannerContext，于是进入 ScannerContext::~ScannerContext()。 6. 旧代码在 ScannerContext::~ScannerContext() 里又执行： task_executor_scheduler->task_executor()->remove_task(_task_handle) 而 task_executor() 返回的是： std::shared_ptr task_executor() const { return _task_executor; } 也就是从 scheduler 的 _task_executor 成员再复制一个 shared_ptr。 问题在于：第 1 步里，scheduler 的 _task_executor 这个 shared_ptr 正在析构，并且它已经触发了 TimeSharingTaskExecutor 的析构。此时再从同一个 member shared_ptr 复制引用，等价于在其析构过程中重新操作它的 control block。这个 control block 可能已经处于 release-last-use 路径，随后临时复制出来的 shared_ptr 析构又会再次 release 同一个 control block，导致 TimeSharingTaskExecutor 析构链被重入或二次释放其成员。 ``` /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1069:11: runtime error: member call on address 0x7c63e14891c0 which does not point to an object of type 'std::_Sp_counted_base<>' 0x7c63e14891c0: note: object has invalid vptr 00 00 00 00 e0 74 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 be be be be d0 26 2b e1 ^~~~~~~~~~~~~~~~~~~~~~~ invalid vptr #0 0x55e77069caee in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1069:11 #1 0x55e77069caee in std::__shared_ptr::~__shared_ptr() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1531:31 #2 0x55e77069caee in doris::TimeSharingTaskExecutor::~TimeSharingTaskExecutor() be/build_ASAN/./be/build_ASAN/../src/exec/scan/task_executor/time_sharing/time_sharing_task_executor.cpp:306:1 #3 0x55e75b584397 in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release_last_use() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:174:2 #4 0x55e7706486ac in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1069:11 #5 0x55e7706486ac in std::__shared_ptr::~__shared_ptr() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1531:31 #6 0x55e7706486ac in doris::ScannerContext::~ScannerContext() be/build_ASAN/./be/build_ASAN/../src/exec/scan/scanner_context.cpp:277:13 #7 0x55e75b584397 in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release_last_use() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:174:2 #8 0x55e7706f288d in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1069:11 #9 0x55e7706f288d in std::__shared_ptr::~__shared_ptr() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1531:31 #10 0x55e7706f288d in doris::ScannerScheduler::submit(std::shared_ptr, std::shared_ptr)::$_0::operator()() const::'lambda'()::~() be/build_ASAN/./be/build_ASAN/../src/exec/scan/scanner_scheduler.cpp:77:26 #11 0x55e7706f39bb in std::_Function_base::_Base_manager, std::shared_ptr)::$_0::operator()() const::'lambda'()>::_M_destroy(std::_Any_data&, std::integral_constant) /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:177:4 #12 0x55e7706f39bb in std::_Function_base::_Base_manager, std::shared_ptr)::$_0::operator()() const::'lambda'()>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation) /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:205:8 #13 0x55e7706f39bb in std::_Function_handler, std::shared_ptr)::$_0::operator()() const::'lambda'()>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation) /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:284:6 #14 0x55e770702246 in std::_Function_base::~_Function_base() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/std_function.h:246:2 #15 0x55e770702246 in doris::ScannerSplitRunner::~ScannerSplitRunner() be/build_ASAN/../src/exec/scan/scanner_scheduler.h:62:7 #16 0x55e75b584397 in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release_last_use() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:174:2 #17 0x55e7706e11e9 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1069:11 #18 0x55e7706e11e9 in std::__shared_ptr::~__shared_ptr() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1531:31 #19 0x55e7706e11e9 in doris::PrioritizedSplitRunner::~PrioritizedSplitRunner() be/build_ASAN/../src/exec/scan/task_executor/time_sharing/prioritized_split_runner.h:54:47 #20 0x55e75b584397 in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release_last_use() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:174:2 #21 0x55e7706b8ba4 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1069:11 #22 0x55e7706b8ba4 in std::__shared_ptr::~__shared_ptr() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1531:31 #23 0x55e7706b8ba4 in void std::destroy_at >(std::shared_ptr*) /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/stl_construct.h:88:15 #24 0x55e7706b8ba4 in void std::allocator_traits, false> > >::destroy >(std::allocator, false> >&, std::shared_ptr*) /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/alloc_traits.h:698:4 #25 0x55e7706b8ba4 in std::__detail::_Hashtable_alloc, false> > >::_M_deallocate_node(std::__detail::_Hash_node, false>*) /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/hashtable_policy.h:1572:7 #26 0x55e7706b89ff in std::__detail::_Hashtable_alloc, false> > >::_M_deallocate_nodes(std::__detail::_Hash_node, false>*) /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/hashtable_policy.h:1594:4 #27 0x55e7706b89ff in std::_Hashtable, std::shared_ptr, std::allocator >, std::__detail::_Identity, std::equal_to >, std::hash >, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::~_Hashtable() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/hashtable.h:1852:13 #28 0x55e77069c6ca in std::unordered_set, std::hash >, std::equal_to >, std::allocator > >::~unordered_set() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/unordered_set.h:107:11 #29 0x55e77069c6ca in doris::TimeSharingTaskExecutor::~TimeSharingTaskExecutor() be/build_ASAN/./be/build_ASAN/../src/exec/scan/task_executor/time_sharing/time_sharing_task_executor.cpp:306:1 #30 0x55e75b583fa1 in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:345:8 #31 0x55e770688ecf in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1069:11 #32 0x55e770688ecf in std::__shared_ptr::~__shared_ptr() /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1531:31 #33 0x55e770688ecf in doris::TaskExecutorSimplifiedScanScheduler::~TaskExecutorSimplifiedScanScheduler() be/build_ASAN/../src/exec/scan/scanner_scheduler.h:266:5 #34 0x55e7706891ed in doris::TaskExecutorSimplifiedScanScheduler::~TaskExecutorSimplifiedScanScheduler() be/build_ASAN/../src/exec/scan/scanner_scheduler.h:261:53 #35 0x55e7744d5afa in std::default_delete::operator()(doris::ScannerScheduler*) const /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/unique_ptr.h:93:2 #36 0x55e7744d5afa in std::__uniq_ptr_impl >::reset(doris::ScannerScheduler*) /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/unique_ptr.h:205:4 #37 0x55e7744d5afa in std::unique_ptr >::reset(doris::ScannerScheduler*) /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/unique_ptr.h:512:7 #38 0x55e7744d5afa in doris::WorkloadGroup::destroy_schedulers() be/build_ASAN/./be/build_ASAN/../src/runtime/workload_group/workload_group.cpp:776:22 #39 0x55e774501af0 in doris::WorkloadGroupMgr::destroy_schedulers() be/build_ASAN/./be/build_ASAN/../src/runtime/workload_group/workload_group_manager.cpp:977:23 #40 0x55e7740e0ddc in doris::ExecEnv::destroy() be/build_ASAN/./be/build_ASAN/../src/runtime/exec_env_init.cpp:892:34 #41 0x55e75b5601f2 in main be/build_ASAN/./be/build_ASAN/../src/service/doris_main.cpp:734:15 #42 0x7f53e1e6b082 in __libc_start_main /build/glibc-SzIz7B/glibc-2.31/csu/../csu/libc-start.c:308:16 #43 0x55e75b470029 in _start (/mnt/ssd01/pipline/OpenSourceDoris/clusterEnv/P0/Cluster0/be/lib/doris_be+0x247b8029) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /usr/local/ldb-toolchain-v0.26/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/shared_ptr_base.h:1069:11 ``` ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/exec/scan/scanner_context.cpp | 14 ++++++++------ be/src/exec/scan/scanner_context.h | 1 + be/src/exec/scan/scanner_scheduler.h | 8 ++++++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/be/src/exec/scan/scanner_context.cpp b/be/src/exec/scan/scanner_context.cpp index 3371af5bf8e829..279040c5168279 100644 --- a/be/src/exec/scan/scanner_context.cpp +++ b/be/src/exec/scan/scanner_context.cpp @@ -40,6 +40,7 @@ #include "exec/operator/scan_operator.h" #include "exec/scan/scan_node.h" #include "exec/scan/scanner_scheduler.h" +#include "exec/scan/task_executor/task_executor.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "runtime/runtime_profile.h" @@ -123,6 +124,7 @@ Status ScannerContext::init() { if (auto* task_executor_scheduler = dynamic_cast(_scanner_scheduler)) { std::shared_ptr task_executor = task_executor_scheduler->task_executor(); + _task_executor = task_executor; TaskId task_id(fmt::format("{}-{}", print_id(_state->query_id()), ctx_id)); _task_handle = DORIS_TRY(task_executor->create_task( task_id, []() { return 0.0; }, @@ -187,11 +189,11 @@ ScannerContext::~ScannerContext() { block.reset(); DorisMetrics::instance()->scanner_ctx_cnt->increment(-1); if (_task_handle) { - if (auto* task_executor_scheduler = - dynamic_cast(_scanner_scheduler)) { - static_cast(task_executor_scheduler->task_executor()->remove_task(_task_handle)); + if (auto task_executor = _task_executor.lock()) { + static_cast(task_executor->remove_task(_task_handle)); } _task_handle = nullptr; + _task_executor.reset(); } } @@ -389,11 +391,11 @@ void ScannerContext::stop_scanners(RuntimeState* state) { } _tasks_queue.clear(); if (_task_handle) { - if (auto* task_executor_scheduler = - dynamic_cast(_scanner_scheduler)) { - static_cast(task_executor_scheduler->task_executor()->remove_task(_task_handle)); + if (auto task_executor = _task_executor.lock()) { + static_cast(task_executor->remove_task(_task_handle)); } _task_handle = nullptr; + _task_executor.reset(); } // TODO yiguolei, call mark close to scanners if (state->enable_profile()) { diff --git a/be/src/exec/scan/scanner_context.h b/be/src/exec/scan/scanner_context.h index 857dfb736e91bd..ba1c323566062b 100644 --- a/be/src/exec/scan/scanner_context.h +++ b/be/src/exec/scan/scanner_context.h @@ -239,6 +239,7 @@ class ScannerContext : public std::enable_shared_from_this, std::shared_ptr _resource_ctx; std::shared_ptr _dependency = nullptr; std::shared_ptr _task_handle; + std::weak_ptr _task_executor; std::atomic _block_memory_usage = 0; diff --git a/be/src/exec/scan/scanner_scheduler.h b/be/src/exec/scan/scanner_scheduler.h index f1939409bda579..66d5fd55f65cb7 100644 --- a/be/src/exec/scan/scanner_scheduler.h +++ b/be/src/exec/scan/scanner_scheduler.h @@ -351,10 +351,14 @@ class TaskExecutorSimplifiedScanScheduler final : public ScannerScheduler { : std::max(48, CpuInfo::num_cores() * 2), std::chrono::milliseconds(100), std::nullopt)); - auto wrapped_scan_func = [this, task_handle, scan_func = scan_task.scan_func]() { + std::weak_ptr task_executor = _task_executor; + auto wrapped_scan_func = [task_executor, task_handle, + scan_func = scan_task.scan_func]() { bool result = scan_func(); if (result) { - static_cast(_task_executor->remove_task(task_handle)); + if (auto executor = task_executor.lock()) { + static_cast(executor->remove_task(task_handle)); + } } return result; };