diff --git a/.github/workflows/test-windows.ps1 b/.github/workflows/test-windows.ps1 index 809542815..c20ecc521 100644 --- a/.github/workflows/test-windows.ps1 +++ b/.github/workflows/test-windows.ps1 @@ -21,10 +21,11 @@ New-Item -ItemType Directory $BuildDirectory | Out-Null Invoke-NativeCommand cmake -B $BuildDirectory -G Ninja ` "-DCMAKE_BUILD_TYPE=$Config" ` + "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" ` "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT:STRING=Embedded" ` + "-DCMAKE_CXX_FLAGS:STRING=/fsanitize=address /EHsc" ` "-DSTDEXEC_ENABLE_ASIO:BOOL=TRUE" ` "-DSTDEXEC_ASIO_IMPLEMENTATION:STRING=boost" ` - "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" ` "-DSTDEXEC_BUILD_TESTS:BOOL=TRUE" . Invoke-NativeCommand cmake --build $BuildDirectory Invoke-NativeCommand ctest --test-dir $BuildDirectory --output-on-failure --verbose --timeout 60 diff --git a/CMakeLists.txt b/CMakeLists.txt index f603401a6..c574a5341 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -252,7 +252,7 @@ target_compile_options(stdexec INTERFACE # Do you want a preprocessor that works? Picky, picky. target_compile_options(stdexec INTERFACE - $<$:/Zc:__cplusplus /Zc:preprocessor /Zc:externConstexpr> + $<$:/Zc:__cplusplus /Zc:preprocessor /Zc:externConstexpr /bigobj> ) set(STDEXEC_NAMESPACE "stdexec" CACHE STRING "The name of the top-level namespace for stdexec") diff --git a/include/exec/at_coroutine_exit.hpp b/include/exec/at_coroutine_exit.hpp index a54668d63..9a872f67f 100644 --- a/include/exec/at_coroutine_exit.hpp +++ b/include/exec/at_coroutine_exit.hpp @@ -141,7 +141,7 @@ namespace experimental::execution ~__task() { if (__coro_) - __coro_.destroy(); + STDEXEC::__coroutine_destroy_nothrow(__coro_); } [[nodiscard]] @@ -189,7 +189,7 @@ namespace experimental::execution { auto __cont = __h.promise().continuation(); auto __coro = __h.promise().__is_stopped_ ? __cont.unhandled_stopped() : __cont.handle(); - return STDEXEC_CORO_DESTROY_AND_CONTINUE(__h, __coro); + return STDEXEC::__coroutine_destroy_and_continue(__h, __coro); } static constexpr void await_resume() noexcept {} diff --git a/include/exec/on_coro_disposition.hpp b/include/exec/on_coro_disposition.hpp index bfd8385cd..edb8b2ed1 100644 --- a/include/exec/on_coro_disposition.hpp +++ b/include/exec/on_coro_disposition.hpp @@ -121,7 +121,7 @@ namespace experimental::execution { auto __cont = __h.promise().continuation(); auto __coro = __h.promise().__is_stopped_ ? __cont.unhandled_stopped() : __cont.handle(); - return STDEXEC_CORO_DESTROY_AND_CONTINUE(__h, __coro); + return STDEXEC::__coroutine_destroy_and_continue(__h, __coro); } void await_resume() const noexcept {} diff --git a/include/exec/task.hpp b/include/exec/task.hpp index c73746769..e0a3f4af3 100644 --- a/include/exec/task.hpp +++ b/include/exec/task.hpp @@ -531,7 +531,7 @@ namespace experimental::execution constexpr ~basic_task() { if (__coro_) - __coro_.destroy(); + STDEXEC::__coroutine_destroy_nothrow(__coro_); } private: @@ -653,7 +653,7 @@ namespace experimental::execution constexpr ~__task_awaiter() { if (__coro_) - __coro_.destroy(); + STDEXEC::__coroutine_destroy_nothrow(__coro_); } static constexpr auto await_ready() noexcept -> bool @@ -672,7 +672,8 @@ namespace experimental::execution if constexpr (requires { __coro_.promise().stop_requested() ? 0 : 1; }) { if (__coro_.promise().stop_requested()) - return __parent.promise().unhandled_stopped(); + return STDEXEC::__coroutine_destroy_and_continue( + __parent.promise().unhandled_stopped()); } return __coro_; } diff --git a/include/stdexec/__detail/__as_awaitable.hpp b/include/stdexec/__detail/__as_awaitable.hpp index 1d154e29b..e489263c6 100644 --- a/include/stdexec/__detail/__as_awaitable.hpp +++ b/include/stdexec/__detail/__as_awaitable.hpp @@ -133,8 +133,14 @@ namespace STDEXEC // If the operation was stopped (__result_ is valueless), we should use the // unhandled_stopped() continuation. Otherwise, should resume the __continuation_ // as normal. - return __result_.__is_valueless() ? __continuation_.unhandled_stopped() - : __continuation_.handle(); + if (__result_.__is_valueless()) + { + return STDEXEC::__coroutine_destroy_and_continue(__continuation_.unhandled_stopped()); + } + else + { + return __continuation_.handle(); + } } __coroutine_handle<> __continuation_; diff --git a/include/stdexec/__detail/__config.hpp b/include/stdexec/__detail/__config.hpp index 3106b08fa..ad3643cad 100644 --- a/include/stdexec/__detail/__config.hpp +++ b/include/stdexec/__detail/__config.hpp @@ -142,6 +142,13 @@ # define STDEXEC_HAS_ATTRIBUTE(...) 0 #endif +//////////////////////////////////////////////////////////////////////////////////////////////////// +#if defined(__has_cpp_attribute) +# define STDEXEC_HAS_CPP_ATTRIBUTE(...) __has_cpp_attribute(__VA_ARGS__) +#else +# define STDEXEC_HAS_CPP_ATTRIBUTE(...) 0 +#endif + //////////////////////////////////////////////////////////////////////////////////////////////////// #if STDEXEC_CLANG() && STDEXEC_CUDA_COMPILATION() # define STDEXEC_HOST_DEVICE_DEDUCTION_GUIDE __host__ __device__ @@ -321,7 +328,7 @@ namespace STDEXEC::__std # define STDEXEC_ATTR_WHICH_4(_ATTR) __forceinline #elif STDEXEC_CLANG() # define STDEXEC_ATTR_WHICH_4(_ATTR) \ - inline //__attribute__((__always_inline__, __artificial__, __nodebug__)) inline + __attribute__((__always_inline__, __artificial__, __nodebug__)) inline #elif STDEXEC_GCC() # define STDEXEC_ATTR_WHICH_4(_ATTR) __attribute__((__always_inline__, __artificial__)) inline #else @@ -370,6 +377,16 @@ namespace STDEXEC::__std #define STDEXEC_ATTR_noinline STDEXEC_PP_PROBE(~, 9) #define STDEXEC_ATTR___noinline__ STDEXEC_PP_PROBE(~, 9) +#if STDEXEC_MSVC() && !STDEXEC_CLANG_CL() && STDEXEC_MSVC_VERSION >= 1950 +# define STDEXEC_ATTR_WHICH_10(_ATTR) [[msvc::musttail]] +#elif STDEXEC_HAS_CPP_ATTRIBUTE(gnu::musttail) +# define STDEXEC_ATTR_WHICH_10(_ATTR) [[gnu::musttail]] +#else +# define STDEXEC_ATTR_WHICH_10(_ATTR) /*nothing*/ +#endif +#define STDEXEC_ATTR_musttail STDEXEC_PP_PROBE(~, 10) +#define STDEXEC_ATTR___musttail__ STDEXEC_PP_PROBE(~, 10) + //////////////////////////////////////////////////////////////////////////////////////////////////// // warning push/pop portability macros #if STDEXEC_NVCC() diff --git a/include/stdexec/__detail/__connect_awaitable.hpp b/include/stdexec/__detail/__connect_awaitable.hpp index efbe443cf..9b24af9ec 100644 --- a/include/stdexec/__detail/__connect_awaitable.hpp +++ b/include/stdexec/__detail/__connect_awaitable.hpp @@ -182,7 +182,7 @@ namespace STDEXEC ~__state() { - // make sure to destroy in the reverse order of construction + // make sure to __destroy in the reverse order of construction __awaiter_.__destroy(); __awaitable_.__destroy(); } @@ -206,12 +206,12 @@ namespace STDEXEC : __source_awaitable_(static_cast<_Awaitable2&&>(__awaitable)) {} - constexpr void construct(__std::coroutine_handle<_Promise> __coro) noexcept(__is_nothrow) + constexpr void __construct(__std::coroutine_handle<_Promise> __coro) noexcept(__is_nothrow) { __awaiter_.__construct(static_cast<_Awaitable&&>(__source_awaitable_), __coro); } - constexpr void destroy() noexcept + constexpr void __destroy() noexcept { __awaiter_.__destroy(); } @@ -263,12 +263,12 @@ namespace STDEXEC : __source_awaitable_(static_cast<_Awaitable2&&>(__awaitable)) {} - constexpr void construct(__std::coroutine_handle<_Promise> __coro) noexcept(__is_nothrow) + constexpr void __construct(__std::coroutine_handle<_Promise> __coro) noexcept(__is_nothrow) { __awaiter_.__construct(static_cast<_Awaitable&&>(__source_awaitable_), __coro); } - constexpr void destroy() noexcept + constexpr void __destroy() noexcept { __awaiter_.__destroy(); } @@ -328,12 +328,12 @@ namespace STDEXEC : __source_awaitable_(static_cast<_Awaitable2&&>(__awaitable)) {} - constexpr void construct(__std::coroutine_handle<_Promise> __coro) noexcept(__is_nothrow) + constexpr void __construct(__std::coroutine_handle<_Promise> __coro) noexcept(__is_nothrow) { __awaiter_.__construct(static_cast<_Awaitable&&>(__source_awaitable_), __coro); } - constexpr void destroy() noexcept + constexpr void __destroy() noexcept { __awaiter_.__destroy(); } @@ -363,12 +363,12 @@ namespace STDEXEC __awaiter_.__destroy(); } - static constexpr void construct(__std::coroutine_handle<_Promise>) noexcept + static constexpr void __construct(__std::coroutine_handle<_Promise>) noexcept { // no-op } - static constexpr void destroy() noexcept + static constexpr void __destroy() noexcept { // no-op } @@ -389,7 +389,7 @@ namespace STDEXEC { if (__started_) { - __awaiter_.destroy(); + __awaiter_.__destroy(); } } @@ -399,7 +399,7 @@ namespace STDEXEC STDEXEC_TRY { - __awaiter_.construct(__coro); + __awaiter_.__construct(__coro); __started_ = true; if (!__awaiter_.await_ready()) @@ -439,7 +439,7 @@ namespace STDEXEC } STDEXEC_CATCH_ALL { - if constexpr (!noexcept(__awaiter_.construct(__coro)) + if constexpr (!noexcept(__awaiter_.__construct(__coro)) || !noexcept(__awaiter_.await_ready()) || !noexcept(__awaiter_.await_suspend(__coro))) { diff --git a/include/stdexec/__detail/__task.hpp b/include/stdexec/__detail/__task.hpp index 9a8120569..a2350be9e 100644 --- a/include/stdexec/__detail/__task.hpp +++ b/include/stdexec/__detail/__task.hpp @@ -287,7 +287,7 @@ namespace STDEXEC constexpr ~task() { if (__coro_) - __coro_.destroy(); + STDEXEC::__coroutine_destroy_nothrow(__coro_); } [[nodiscard]] diff --git a/include/stdexec/coroutine.hpp b/include/stdexec/coroutine.hpp index f26378a97..06d9eda32 100644 --- a/include/stdexec/coroutine.hpp +++ b/include/stdexec/coroutine.hpp @@ -33,34 +33,44 @@ namespace STDEXEC return __std::coroutine_handle<_Tp>::from_address(__h.address()); } - inline void __coroutine_resume_nothrow(__std::coroutine_handle<> __h) noexcept + STDEXEC_ATTRIBUTE(always_inline) + void __coroutine_resume_nothrow(void* __address) noexcept { STDEXEC_TRY { - STDEXEC_ASSERT(__h); - __h.resume(); + __builtin_coro_resume(__address); } STDEXEC_CATCH_ALL { - STDEXEC_ASSERT(!"Coroutine resume threw an exception!"); __std::unreachable(); } } - inline void __coroutine_destroy_nothrow(__std::coroutine_handle<> __h) noexcept + STDEXEC_ATTRIBUTE(always_inline) + void __coroutine_resume_nothrow(__std::coroutine_handle<> __h) noexcept + { + STDEXEC::__coroutine_resume_nothrow(__h.address()); + } + + STDEXEC_ATTRIBUTE(always_inline) + void __coroutine_destroy_nothrow(void* __address) noexcept { STDEXEC_TRY { - STDEXEC_ASSERT(__h); - __h.destroy(); + __builtin_coro_destroy(__address); } STDEXEC_CATCH_ALL { - STDEXEC_ASSERT(!"Coroutine destroy threw an exception!"); __std::unreachable(); } } + STDEXEC_ATTRIBUTE(always_inline) + void __coroutine_destroy_nothrow(__std::coroutine_handle<> __h) noexcept + { + STDEXEC::__coroutine_destroy_nothrow(__h.address()); + } + // A coroutine handle that also supports unhandled_stopped() for propagating stop // signals through co_awaits of senders. template @@ -157,24 +167,19 @@ namespace STDEXEC { struct __synthetic_coro_frame { - void (*__resume_)(void*) noexcept; - // we never invoke __destroy_ so a no-op implementation is fine; we've chosen the - // address of a no-op function rather than nullptr in case some rogue awaitable - // *does* invoke destroy on the synthesized handle that it receives in its - // await_suspend function - void (*__destroy_)(void*) noexcept = &__noop_destroy; + using __callback_fn_t = void(void*) noexcept; - static void __noop_destroy(void*) noexcept - { - STDEXEC_ASSERT(!"Attempt to destroy a synthetic coroutine!"); - } + __callback_fn_t* __resume_ = &__noop_fn; + __callback_fn_t* __destroy_ = &__noop_fn; + + static void __noop_fn(void*) noexcept {} }; static constexpr std::ptrdiff_t __coro_promise_offset = static_cast( sizeof(__synthetic_coro_frame)); } // namespace __detail -# if STDEXEC_MSVC() && STDEXEC_MSVC_VERSION <= 1939 +# if STDEXEC_MSVC() && STDEXEC_MSVC_VERSION < 1950 // MSVCBUG https://developercommunity.visualstudio.com/t/destroy-coroutine-from-final_suspend-r/10096047 // Prior to Visual Studio 17.9 (Feb, 2024), aka MSVC 19.39, MSVC incorrectly allocates @@ -192,17 +197,14 @@ namespace STDEXEC struct __destroy_and_continue_frame : __detail::__synthetic_coro_frame { - constexpr __destroy_and_continue_frame() noexcept - : __detail::__synthetic_coro_frame{&__destroy_and_continue_frame::__resume} - {} - static void __resume(void* __address) noexcept { // Make a local copy of the promise to ensure we can safely destroy the suspended // coroutine after resuming the continuation. auto __promise = static_cast<__destroy_and_continue_frame*>(__address)->__promise_; STDEXEC::__coroutine_resume_nothrow(__promise.__continue_); - STDEXEC::__coroutine_destroy_nothrow(__promise.__destroy_); + STDEXEC_ATTRIBUTE(musttail) + return STDEXEC::__coroutine_destroy_nothrow(__promise.__destroy_.address()); } struct __promise @@ -210,24 +212,72 @@ namespace STDEXEC __std::coroutine_handle<> __destroy_{}; __std::coroutine_handle<> __continue_{}; } __promise_; + + static thread_local __destroy_and_continue_frame value; }; + inline thread_local __destroy_and_continue_frame __destroy_and_continue_frame::value{ + {&__destroy_and_continue_frame::__resume}, + {}}; + + struct __symmetric_transfer_frame : __detail::__synthetic_coro_frame + { + static void __resume(void* __address) noexcept + { + // Make a local copy of the promise to ensure we can safely destroy the suspended + // coroutine after resuming the continuation. + auto __promise = static_cast<__symmetric_transfer_frame*>(__address)->__promise_; + STDEXEC_ATTRIBUTE(musttail) + return STDEXEC::__coroutine_resume_nothrow(__promise.__continue_.address()); + } + + struct __promise + { + __std::coroutine_handle<> __continue_{}; + } __promise_; + + static thread_local __symmetric_transfer_frame value; + }; + + inline thread_local __symmetric_transfer_frame __symmetric_transfer_frame::value{ + {&__symmetric_transfer_frame::__resume}, + {}}; + inline auto __coroutine_destroy_and_continue(__std::coroutine_handle<> __destroy, // __std::coroutine_handle<> __continue) noexcept // -> __std::coroutine_handle<> { - static constinit thread_local __destroy_and_continue_frame __fr; - __fr.__promise_.__destroy_ = __destroy; - __fr.__promise_.__continue_ = __continue; - return __std::coroutine_handle<>::from_address(&__fr); + __destroy_and_continue_frame::value.__promise_.__destroy_ = __destroy; + __destroy_and_continue_frame::value.__promise_.__continue_ = __continue; + return __std::coroutine_handle<>::from_address(&__destroy_and_continue_frame::value); + } + + inline auto __coroutine_destroy_and_continue(__std::coroutine_handle<> __continue) noexcept // + -> __std::coroutine_handle<> + { + __symmetric_transfer_frame::value.__promise_.__continue_ = __continue; + return __std::coroutine_handle<>::from_address(&__symmetric_transfer_frame::value); } -# define STDEXEC_CORO_DESTROY_AND_CONTINUE(__destroy, __continue) \ - ::STDEXEC::__coroutine_destroy_and_continue(__destroy, __continue) # else -# define STDEXEC_CORO_DESTROY_AND_CONTINUE(__destroy, __continue) \ - (__destroy.destroy(), __continue) -# endif + + STDEXEC_ATTRIBUTE(always_inline) + auto __coroutine_destroy_and_continue(__std::coroutine_handle<> __destroy, // + __std::coroutine_handle<> __continue) noexcept // + -> __std::coroutine_handle<> + { + ::STDEXEC::__coroutine_destroy_nothrow(__destroy); + return __continue; + } + + STDEXEC_ATTRIBUTE(always_inline) + auto __coroutine_destroy_and_continue(__std::coroutine_handle<> __continue) noexcept // + -> __std::coroutine_handle<> + { + return __continue; + } + +# endif // STDEXEC_MSVC() && STDEXEC_MSVC_VERSION < 1950 } // namespace STDEXEC #endif // !STDEXEC_NO_STDCPP_COROUTINES()