Thread TargetSetupOptions through RCTarget; fix in_options doc

Signed-off-by: Maciej Oleksy <maciej.oleksy@intel.com>

Author: moleksy

lib/fabrics/include/mxl/fabrics.h

@@ -156,7 +156,7 @@ extern "C"
      * \param in_target A valid fabrics target
      * \param in_config The target configuration. This will be used to create an endpoint and register a memory region. The memory region
      * corresponds to the one that will be written to by the initiator.
-     * \param options An optional json-formatted string with options. May be NULL or empty. Recognized fields:
+     * \param in_options An optional json-formatted string with options. May be NULL or empty. Recognized fields:
      *  - "cqDepth" (number >= 1): the depth of the target's completion queue. Increase this for high-frame-rate or many-stream
      *    receivers, or when per-completion processing is slow, to avoid completion queue overflow. See the "Receiving grains" section
      *    of docs/Fabrics.md. When omitted, an implementation default is used.

lib/fabrics/ofi/src/internal/RCTarget.cpp

@@ -64,24 +64,22 @@ namespace mxl::lib::fabrics::ofi
         auto proto = selectIngressProtocol(mxlRegions.dataLayout(), mxlRegions.regions(), mxlRegions.maxSyncBatchSize());
         auto targetInfo = std::make_unique<TargetInfo>(pep.id(), pep.localAddress(), proto->registerMemory(domain), proto->bounceBufferInfo());
 
-        auto const cqSize = options.cqDepth.value_or(CompletionQueue::Attributes::DEFAULT_SIZE);
-
         // Helper struct to enable the std::make_unique function to access the private constructor of this class.
         struct MakeUniqueEnabler : RCTarget
         {
-            MakeUniqueEnabler(PassiveEndpoint pep, std::unique_ptr<IngressProtocol> proto, std::shared_ptr<Domain> domain, std::size_t cqDepth)
-                : RCTarget(std::move(pep), std::move(proto), std::move(domain), cqDepth)
+            MakeUniqueEnabler(PassiveEndpoint pep, std::unique_ptr<IngressProtocol> proto, std::shared_ptr<Domain> domain, TargetSetupOptions options)
+                : RCTarget(std::move(pep), std::move(proto), std::move(domain), std::move(options))
             {}
         };
 
         // Return the constructed RCTarget and associated TargetInfo for remote peers to connect.
-        return {std::make_unique<MakeUniqueEnabler>(std::move(pep), std::move(proto), std::move(domain), cqSize), std::move(targetInfo)};
+        return {std::make_unique<MakeUniqueEnabler>(std::move(pep), std::move(proto), std::move(domain), options), std::move(targetInfo)};
     }
 
-    RCTarget::RCTarget(PassiveEndpoint pep, std::unique_ptr<IngressProtocol> proto, std::shared_ptr<Domain> domain, std::size_t cqDepth)
+    RCTarget::RCTarget(PassiveEndpoint pep, std::unique_ptr<IngressProtocol> proto, std::shared_ptr<Domain> domain, TargetSetupOptions options)
         : _proto(std::move(proto))
         , _domain(std::move(domain))
-        , _cqDepth(cqDepth)
+        , _setupOptions(std::move(options))
         , _state(WaitForConnectionRequest{std::move(pep)})
     {}
 
@@ -162,7 +160,7 @@ namespace mxl::lib::fabrics::ofi
                         auto endpoint = Endpoint::create(_domain, state.pep.id(), event->connReq().info());
 
                         auto cqAttr = CompletionQueue::Attributes::defaults();
-                        cqAttr.size = _cqDepth;
+                        cqAttr.size = _setupOptions.cqDepth.value_or(CompletionQueue::Attributes::DEFAULT_SIZE);
                         auto cq = CompletionQueue::open(_domain, cqAttr);
                         endpoint.bind(cq, FI_RECV);
 

lib/fabrics/ofi/src/internal/RCTarget.hpp

@@ -90,9 +90,9 @@ namespace mxl::lib::fabrics::ofi
          *
          * \param domain The domain to create the RCTarget on.
          * \param pep The passive endpoint to use for listening for incoming connection requests.
-         * \param cqDepth The depth of the completion queue created for the accepted connection.
+         * \param options The setup options applied when the completion queue is created for the accepted connection.
          */
-        RCTarget(PassiveEndpoint pep, std::unique_ptr<IngressProtocol> proto, std::shared_ptr<Domain> domain, std::size_t cqDepth);
+        RCTarget(PassiveEndpoint pep, std::unique_ptr<IngressProtocol> proto, std::shared_ptr<Domain> domain, TargetSetupOptions options);
 
         /** \brief Internal method to drive progress based on the current state.
          *
@@ -108,7 +108,7 @@ namespace mxl::lib::fabrics::ofi
     private:
         std::unique_ptr<IngressProtocol> _proto;
         std::shared_ptr<Domain> _domain;
-        std::size_t _cqDepth; /**< Completion queue depth for accepted connections. */
-        State _state;         /**< The current state of the RCTarget. */
+        TargetSetupOptions _setupOptions; /**< Setup options applied when accepting connections. */
+        State _state;                     /**< The current state of the RCTarget. */
     };
 }