refactor engine: model Awaiter with unique ownership

No changelog entry intended, no effects on API or measurable effects on performance.

* `engine::impl::Awaiter` is now passed around in `AwaiterPtr`, which is a `unique_ptr` with a custom deleter that decrements the intrusive refcounter of `TaskContext` calls virtual `PolymorphicAwaiter::DisposeWithoutNotification` on destruction.
   * Rationale:
      * `unique_ptr` better suits `AwaitableBase` API, which are the majority of awaitables. Only legacy `WaitAny` actually copies the `intrusive_ptr`.
      * `unique_ptr` better suits most awaiters, which don't support being notified concurrently. And those that do can easily add an explicit `AsAwaiterPtr` method, see e.g. `WaitAnyContext::Impl`.
         * Making an expensive locked increment an explicit operation can be seen as an additional benefit.
      * The existing performance of `TaskContext` awaiter is maintained, there are still no virtual calls for this kind of `Awaiter`.

* Moved the atomic refcount from `Awaiter` to `TaskContext`.

* Added `IntrusiveRefCounterOne` as a common place for intrusive refcounter that initializes to 1 instead of 0 (that place was `Awaiter` previously).
commit_hash:3170fb506a660a90af55e129ff22f8c9a9b677bb

Author: Anton3

.mapping.json

@@ -1147,6 +1147,7 @@
   "core/include/userver/engine/get_all.hpp":"taxi/uservices/userver/core/include/userver/engine/get_all.hpp",
   "core/include/userver/engine/impl/actor.hpp":"taxi/uservices/userver/core/include/userver/engine/impl/actor.hpp",
   "core/include/userver/engine/impl/awaiter.hpp":"taxi/uservices/userver/core/include/userver/engine/impl/awaiter.hpp",
+  "core/include/userver/engine/impl/awaiter_fwd.hpp":"taxi/uservices/userver/core/include/userver/engine/impl/awaiter_fwd.hpp",
   "core/include/userver/engine/impl/condition_variable_any.hpp":"taxi/uservices/userver/core/include/userver/engine/impl/condition_variable_any.hpp",
   "core/include/userver/engine/impl/context_accessor.hpp":"taxi/uservices/userver/core/include/userver/engine/impl/context_accessor.hpp",
   "core/include/userver/engine/impl/detach.hpp":"taxi/uservices/userver/core/include/userver/engine/impl/detach.hpp",
@@ -6003,6 +6004,7 @@
   "universal/include/userver/utils/impl/internal_tag.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/internal_tag.hpp",
   "universal/include/userver/utils/impl/internal_tag_fwd.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/internal_tag_fwd.hpp",
   "universal/include/userver/utils/impl/intrusive_link_mode.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/intrusive_link_mode.hpp",
+  "universal/include/userver/utils/impl/intrusive_ref_counter_one.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/intrusive_ref_counter_one.hpp",
   "universal/include/userver/utils/impl/projecting_view.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/projecting_view.hpp",
   "universal/include/userver/utils/impl/source_location.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/source_location.hpp",
   "universal/include/userver/utils/impl/static_registration.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/static_registration.hpp",

core/include/userver/engine/impl/awaiter.hpp

@@ -1,40 +1,33 @@
 #pragma once
 
-#include <atomic>
 #include <cstddef>
 #include <cstdint>
 
 #include <boost/intrusive/list_hook.hpp>
-#include <boost/smart_ptr/intrusive_ptr.hpp>
-#include <boost/smart_ptr/intrusive_ref_counter.hpp>
 
+#include <userver/engine/impl/awaiter_fwd.hpp>
 #include <userver/engine/impl/epoch.hpp>
 #include <userver/utils/assert.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
 namespace engine::impl {
 
-class PolymorphicAwaiter;
+class TaskContext;
 
 class Awaiter {
 public:
-    enum InitialRefCounter : std::size_t { kZero = 0, kOne = 1 };  // NOLINT(performance-enum-size)
-
     Awaiter(const Awaiter&) = delete;
     Awaiter(Awaiter&&) = delete;
     Awaiter& operator=(const Awaiter&) = delete;
     Awaiter& operator=(Awaiter&&) = delete;
 
-    std::size_t UseCount() const noexcept;
-
-    friend void intrusive_ptr_add_ref(Awaiter* awaiter) noexcept;  // NOLINT(readability-identifier-naming)
-    friend void intrusive_ptr_release(Awaiter* awaiter) noexcept;  // NOLINT(readability-identifier-naming)
+    TaskContext& CastToTaskContext() noexcept;
 
 protected:
-    enum StaticType : std::uint8_t { kPolymorphic, kTaskContext };
+    enum class StaticType : std::uint8_t { kPolymorphic, kTaskContext };
 
-    Awaiter(StaticType type, InitialRefCounter initial_ref_counter);
+    explicit Awaiter(StaticType type) noexcept;
     ~Awaiter() = default;
 
 private:
@@ -47,46 +40,60 @@ class Awaiter {
         std::uintptr_t context{0};
     };
 
+    friend struct AwaiterDeleter;
     friend class WaitList;
-    friend void Notify(boost::intrusive_ptr<Awaiter>&& awaiter, std::uintptr_t context) noexcept;
 
-    static void NotifyTaskContext(boost::intrusive_ptr<Awaiter> self, std::uintptr_t context) noexcept;
+    friend void NotifyAndDispose(AwaiterPtr&& awaiter, std::uintptr_t context) noexcept;
 
-    PolymorphicAwaiter& CastToPolymorphic() noexcept;
-
-    // refcounter for resources and memory deallocation
-    std::atomic<std::size_t> intrusive_refcount_{0};
+    static void NotifyAndDisposeTaskContext(Awaiter* self, std::uintptr_t context) noexcept;
+    static void DisposeWithoutNotificationTaskContext(Awaiter* self) noexcept;
 
     StaticType type_;
-
     WaitListData wait_list_data_;
 };
 
+struct AwaiterDeleter {
+    void operator()(Awaiter* awaiter) noexcept;
+};
+
+using AwaiterPtr = std::unique_ptr<Awaiter, AwaiterDeleter>;
+
 class PolymorphicAwaiter : public Awaiter {
 public:
-    virtual void DoNotify(boost::intrusive_ptr<PolymorphicAwaiter> self, std::uintptr_t context) noexcept = 0;
+    virtual void NotifyAndDispose(std::uintptr_t context) noexcept = 0;
+
+    virtual void DisposeWithoutNotification() noexcept = 0;
 
 protected:
     PolymorphicAwaiter();
-    explicit PolymorphicAwaiter(InitialRefCounter initial_ref_counter);
-
     ~PolymorphicAwaiter() = default;
+};
 
-    // Called from intrusive_ptr_release. Should delete the instance
-    virtual void Destroy() noexcept = 0;
+inline void NotifyAndDispose(AwaiterPtr&& awaiter, std::uintptr_t context) noexcept {
+    UASSERT(awaiter);
 
-private:
-    friend void intrusive_ptr_release(Awaiter* awaiter) noexcept;  // NOLINT(readability-identifier-naming)
-};
+    switch (awaiter->type_) {
+        case Awaiter::StaticType::kTaskContext:
+            Awaiter::NotifyAndDisposeTaskContext(awaiter.release(), context);
+            break;
+        case Awaiter::StaticType::kPolymorphic:
+            // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast)
+            static_cast<PolymorphicAwaiter&>(*awaiter.release()).NotifyAndDispose(context);
+            break;
+    }
+}
 
-inline void Notify(boost::intrusive_ptr<Awaiter>&& awaiter, std::uintptr_t context) noexcept {
+inline void AwaiterDeleter::operator()(Awaiter* awaiter) noexcept {
     UASSERT(awaiter);
 
-    if (awaiter->type_ == Awaiter::StaticType::kTaskContext) {
-        Awaiter::NotifyTaskContext(std::move(awaiter), context);
-    } else {
-        auto polymorphic_awaiter = boost::static_pointer_cast<PolymorphicAwaiter>(std::move(awaiter));
-        polymorphic_awaiter->DoNotify(std::move(polymorphic_awaiter), context);
+    switch (awaiter->type_) {
+        case Awaiter::StaticType::kTaskContext:
+            Awaiter::DisposeWithoutNotificationTaskContext(awaiter);
+            break;
+        case Awaiter::StaticType::kPolymorphic:
+            // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast)
+            static_cast<PolymorphicAwaiter&>(*awaiter).DisposeWithoutNotification();
+            break;
     }
 }
 

core/include/userver/engine/impl/awaiter_fwd.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <memory>
+
+USERVER_NAMESPACE_BEGIN
+
+namespace engine::impl {
+
+class Awaiter;
+struct AwaiterDeleter;
+
+using AwaiterPtr = std::unique_ptr<Awaiter, AwaiterDeleter>;
+
+}  // namespace engine::impl
+
+USERVER_NAMESPACE_END

core/include/userver/engine/impl/context_accessor.hpp

@@ -2,19 +2,14 @@
 
 #include <cstdint>
 #include <exception>
+#include <memory>
 
-#include <boost/smart_ptr/intrusive_ptr.hpp>
+#include <userver/engine/impl/awaiter_fwd.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
 namespace engine::impl {
 
-class Awaiter;
-class TaskContext;
-
-void intrusive_ptr_add_ref(Awaiter* awaiter) noexcept;  // NOLINT(readability-identifier-naming)
-void intrusive_ptr_release(Awaiter* awaiter) noexcept;  // NOLINT(readability-identifier-naming)
-
 class WeakAwaitable {
 public:
     // Auxiliary method to check if the awaitable is ready.
@@ -31,7 +26,7 @@ class WeakAwaitable {
     //    * do not notify `awaiter`.
     //
     // You may not sleep in `TryAppendAwaiter`.
-    virtual void TryAppendAwaiter(boost::intrusive_ptr<Awaiter>& awaiter, std::uintptr_t context) = 0;
+    virtual void TryAppendAwaiter(AwaiterPtr& awaiter, std::uintptr_t context) = 0;
 
     // Atomically:
     //
@@ -48,7 +43,7 @@ class WeakAwaitable {
     //
     // Depending on a wait list implementation `context` match also could be required for the awaiter removal.
     // You may not sleep in `RemoveAwaiter`.
-    virtual boost::intrusive_ptr<Awaiter> RemoveAwaiter(Awaiter& awaiter, std::uintptr_t context) noexcept = 0;
+    virtual AwaiterPtr RemoveAwaiter(Awaiter& awaiter, std::uintptr_t context) noexcept = 0;
 
     // Returns the stored exception if present.
     // Precondition: IsReady.

core/include/userver/engine/impl/future_state.hpp

@@ -38,8 +38,8 @@ class FutureStateBase : private AwaitableBase {
     void WaitForResult();
 
 private:
-    void TryAppendAwaiter(boost::intrusive_ptr<Awaiter>& awaiter, std::uintptr_t context) final;
-    boost::intrusive_ptr<Awaiter> RemoveAwaiter(Awaiter& awaiter, std::uintptr_t context) noexcept final;
+    void TryAppendAwaiter(AwaiterPtr& awaiter, std::uintptr_t context) final;
+    AwaiterPtr RemoveAwaiter(Awaiter& awaiter, std::uintptr_t context) noexcept final;
 
     FastPimplWaitListLight finish_awaiters_;
     std::atomic<bool> is_result_store_locked_;

core/include/userver/engine/multi_consumer_event.hpp

@@ -66,8 +66,8 @@ class MultiConsumerEvent final : private impl::AwaitableBase {
     }
 
 private:
-    void TryAppendAwaiter(boost::intrusive_ptr<impl::Awaiter>& awaiter, std::uintptr_t context) override;
-    boost::intrusive_ptr<impl::Awaiter> RemoveAwaiter(impl::Awaiter& awaiter, std::uintptr_t context) noexcept override;
+    void TryAppendAwaiter(impl::AwaiterPtr& awaiter, std::uintptr_t context) override;
+    impl::AwaiterPtr RemoveAwaiter(impl::Awaiter& awaiter, std::uintptr_t context) noexcept override;
 
     std::atomic<bool> is_ready_{false};
     impl::FastPimplWaitList awaiters_;

core/include/userver/engine/single_use_event.hpp

@@ -86,8 +86,8 @@ class SingleUseEvent final : private impl::AwaitableBase {
     }
 
 private:
-    void TryAppendAwaiter(boost::intrusive_ptr<impl::Awaiter>& awaiter, std::uintptr_t context) override;
-    boost::intrusive_ptr<impl::Awaiter> RemoveAwaiter(impl::Awaiter& awaiter, std::uintptr_t context) noexcept override;
+    void TryAppendAwaiter(impl::AwaiterPtr& awaiter, std::uintptr_t context) override;
+    impl::AwaiterPtr RemoveAwaiter(impl::Awaiter& awaiter, std::uintptr_t context) noexcept override;
 
     impl::FastPimplWaitListLight awaiters_;
 };

core/src/engine/awaitable.cpp

@@ -17,9 +17,9 @@ class ReadyAwaitable final : private impl::AwaitableBase {
 private:
     bool IsReady() const noexcept override { return true; }
 
-    void TryAppendAwaiter(boost::intrusive_ptr<impl::Awaiter>&, std::uintptr_t) override {}
+    void TryAppendAwaiter(impl::AwaiterPtr&, std::uintptr_t) override {}
 
-    boost::intrusive_ptr<impl::Awaiter> RemoveAwaiter(impl::Awaiter&, std::uintptr_t) noexcept override {
+    impl::AwaiterPtr RemoveAwaiter(impl::Awaiter&, std::uintptr_t) noexcept override {
         // The awaiter is always notified early by TryAppendAwaiter, nothing to remove.
         return {};
     }

core/src/engine/impl/awaiter.cpp

@@ -7,64 +7,29 @@ USERVER_NAMESPACE_BEGIN
 
 namespace engine::impl {
 
-Awaiter::Awaiter(StaticType type, InitialRefCounter initial_ref_counter)
-    : intrusive_refcount_(initial_ref_counter),
-      type_(type)
-{}
-
-std::size_t Awaiter::UseCount() const noexcept {
-    // memory order could potentially be less restrictive, but it gets very
-    // complicated to reason about
-    return intrusive_refcount_.load(std::memory_order_seq_cst);
-}
-
-void Awaiter::NotifyTaskContext(boost::intrusive_ptr<Awaiter> self, std::uintptr_t context) noexcept {
-    UASSERT(self);
-    TaskContext::Wakeup(
-        boost::static_pointer_cast<TaskContext>(std::move(self)),
-        TaskContext::WakeupSource::kNotify,
-        static_cast<Epoch>(context)
-    );
-}
-
-PolymorphicAwaiter& Awaiter::CastToPolymorphic() noexcept {
-    UASSERT_MSG(type_ == StaticType::kPolymorphic, "Unexpected awaiter type");
+TaskContext& Awaiter::CastToTaskContext() noexcept {
+    UASSERT(type_ == StaticType::kTaskContext);
     // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast)
-    return static_cast<PolymorphicAwaiter&>(*this);
+    return static_cast<TaskContext&>(*this);
 }
 
-void intrusive_ptr_add_ref(Awaiter* awaiter) noexcept {
-    UASSERT(awaiter);
+Awaiter::Awaiter(StaticType type) noexcept
+    : type_(type)
+{}
 
-    // memory order could potentially be less restrictive, but it gets very
-    // complicated to reason about
-    awaiter->intrusive_refcount_.fetch_add(1, std::memory_order_seq_cst);
+void Awaiter::NotifyAndDisposeTaskContext(Awaiter* self, std::uintptr_t context) noexcept {
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast)
+    boost::intrusive_ptr<TaskContext> task_context(&static_cast<TaskContext&>(*self), /*add_ref=*/false);
+    TaskContext::Wakeup(std::move(task_context), TaskContext::WakeupSource::kNotify, static_cast<Epoch>(context));
 }
 
-void intrusive_ptr_release(Awaiter* awaiter) noexcept {
-    UASSERT(awaiter);
-
-    // memory order could potentially be less restrictive, but it gets very
-    // complicated to reason about
-    if (awaiter->intrusive_refcount_.fetch_sub(1, std::memory_order_seq_cst) != 1) {
-        return;
-    }
-
-    if (awaiter->type_ == Awaiter::StaticType::kTaskContext) {
-        // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast)
-        static_cast<TaskContext*>(awaiter)->Destroy();
-        return;
-    }
-
-    awaiter->CastToPolymorphic().Destroy();
+void Awaiter::DisposeWithoutNotificationTaskContext(Awaiter* self) noexcept {
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast)
+    intrusive_ptr_release(&static_cast<TaskContext&>(*self));
 }
 
 PolymorphicAwaiter::PolymorphicAwaiter()
-    : PolymorphicAwaiter(InitialRefCounter::kZero)
-{}
-
-PolymorphicAwaiter::PolymorphicAwaiter(InitialRefCounter initial_ref_counter)
-    : Awaiter(StaticType::kPolymorphic, initial_ref_counter)
+    : Awaiter(StaticType::kPolymorphic)
 {}
 
 }  // namespace engine::impl

core/src/engine/impl/condition_variable_any.cpp

@@ -23,7 +23,7 @@ class CvAwaitable final : public WeakAwaitable {
 
     bool IsReady() const noexcept override { return false; }
 
-    void TryAppendAwaiter(boost::intrusive_ptr<Awaiter>& awaiter, std::uintptr_t context) override {
+    void TryAppendAwaiter(engine::impl::AwaiterPtr& awaiter, std::uintptr_t context) override {
         UASSERT(mutex_lock_);
         {
             WaitList::Lock awaiters_lock{awaiters_};
@@ -36,7 +36,7 @@ class CvAwaitable final : public WeakAwaitable {
         // by user under mutex_lock_.
     }
 
-    boost::intrusive_ptr<Awaiter> RemoveAwaiter(Awaiter& awaiter, std::uintptr_t context) noexcept override {
+    engine::impl::AwaiterPtr RemoveAwaiter(engine::impl::Awaiter& awaiter, std::uintptr_t context) noexcept override {
         WaitList::Lock awaiters_lock{awaiters_};
         return awaiters_.Remove(awaiters_lock, awaiter, context);
     }