feat: Add hybrid agent support for grpc-dotnet.

Author: jaffinito

src/Agent/NewRelic/Agent/Core/Attributes/AttributeDefinitionService.cs

@@ -62,9 +62,11 @@ public interface IAttributeDefinitions
     AttributeDefinition<string, string> Guid { get; }
     AttributeDefinition<string, string> HostDisplayName { get; }
     AttributeDefinition<string, string> HttpMethod { get; }
+    AttributeDefinition<string, string> HttpRequestMethod { get; }
     AttributeDefinition<long?, long> HttpStatusCode { get; }
     AttributeDefinition<string, string> HttpStatusText { get; }
     AttributeDefinition<Uri, string> HttpUrl { get; }
+    AttributeDefinition<long?, long> GrpcStatusCode { get; }
     AttributeDefinition<bool, bool> IsError { get; }
     AttributeDefinition<string, string> NameForSpan { get; }
     AttributeDefinition<bool, bool> NrEntryPoint { get; }
@@ -475,6 +477,18 @@ public AttributeDefinition<TypeAttributeValue, string> GetTypeAttribute(TypeAttr
             .AppliesTo(AttributeDestinations.TransactionTrace)
             .Build(_attribFilter);
 
+    private AttributeDefinition<long?, long> _grpcStatusCode;
+    public AttributeDefinition<long?, long> GrpcStatusCode => _grpcStatusCode ?? (_grpcStatusCode =
+        AttributeDefinitionBuilder.CreateLong<long?>("grpc.statusCode", AttributeClassification.AgentAttributes)
+            .AppliesTo(AttributeDestinations.ErrorEvent)
+            .AppliesTo(AttributeDestinations.ErrorTrace)
+            .AppliesTo(AttributeDestinations.TransactionEvent)
+            .AppliesTo(AttributeDestinations.ErrorEvent)
+            .AppliesTo(AttributeDestinations.SpanEvent)
+            .AppliesTo(AttributeDestinations.TransactionTrace)
+            .WithConvert(x => x.GetValueOrDefault())                //This is ok b/c we check for null input earlier
+            .Build(_attribFilter));
+
     private AttributeDefinition<string, string> _clientCrossProcessId;
     public AttributeDefinition<string, string> ClientCrossProcessId => _clientCrossProcessId ?? (_clientCrossProcessId =
         AttributeDefinitionBuilder.CreateString("client_cross_process_id", AttributeClassification.Intrinsics)
@@ -669,6 +683,12 @@ public AttributeDefinition<TypeAttributeValue, string> GetTypeAttribute(TypeAttr
 
     private AttributeDefinition<string, string> _httpMethod;
     public AttributeDefinition<string, string> HttpMethod => _httpMethod ?? (_httpMethod =
+        AttributeDefinitionBuilder.CreateString("http.method", AttributeClassification.AgentAttributes)
+            .AppliesTo(AttributeDestinations.SpanEvent)
+            .Build(_attribFilter));
+
+    private AttributeDefinition<string, string> _httpRequestMethod;
+    public AttributeDefinition<string, string> HttpRequestMethod => _httpRequestMethod ?? (_httpRequestMethod =
         AttributeDefinitionBuilder.CreateString("http.request.method", AttributeClassification.AgentAttributes)
             .AppliesTo(AttributeDestinations.SpanEvent)
             .Build(_attribFilter));

src/Agent/NewRelic/Agent/Core/OpenTelemetryBridge/Tracing/ActivityBridgeSegmentHelpers.cs

@@ -58,6 +58,10 @@ public static void ProcessActivityTags(this ISegment segment, object originalAct
                 {
                     ProcessRpcClientTags(segment, agent, errorService, tags, activityLogPrefix, rpcSystem);
                 }
+                else if (tags.TryGetTag<string>(["grpc.method"], out _)) // Its gRPC - likely grpc-dotnet
+                {
+                    ProcessRpcClientTags(segment, agent, errorService, tags, activityLogPrefix, "grpc");
+                }
                 else if (tags.TryGetAndRemoveTag<string>(["http.request.method", "http.method"], out var method)) // it's an HTTP call
                 {
                     ProcessClientExternalTags(segment, agent, tags, activityLogPrefix, method);
@@ -136,25 +140,34 @@ private static void GetInstrumentationScopeAttributes(ISegment segment, dynamic
 
     private static void ProcessRpcClientTags(ISegment segment, IAgent agent, IErrorService errorService, Dictionary<string, object> tags, string activityLogPrefix, string rpcSystem)
     {
-        tags.TryGetAndRemoveTag<int?>(["rpc.grpc.status_code"], out var statusCode);
+        tags.TryGetAndRemoveTag<int?>(["rpc.grpc.status_code", "grpc.status_code"], out var statusCode);
 
         tags.TryGetAndRemoveTag<string>(["rpc.method"], out var method);
-        tags.TryGetAndRemoveTag<string>(["rpc.service"], out var service);
+        tags.TryGetAndRemoveTag<string>(["rpc.service"], out var service); // deprecated attribute, rpc.method includes this information in the format /service/method, but some instrumentation may still use this tag so we'll look for it as a fallback
         tags.TryGetAndRemoveTag<string>(["grpc.method"], out var grpcMethod);
+        var cleanGrpcMethod = grpcMethod?.TrimStart('/'); // per spec, we need to strip leading slashes.
 
         tags.TryGetAndRemoveTag<string>(["server.address", "network.peer.address"], out var host);
+
+        if (string.IsNullOrWhiteSpace(host))
+        {
+            host = ((Segment)segment).GetCacheItem("server.address") as string;
+        }
+
         tags.TryGetAndRemoveTag<int?>(["server.port", "network.peer.port"], out var port);
+        port ??= ((Segment)segment).GetCacheItem("server.port") as int?;
 
-        var path = BuildRpcPath(host ?? "unknown", port ?? 0, service, method, grpcMethod);
+        var path = BuildRpcPath(host ?? "unknown", port ?? 0, service, method, cleanGrpcMethod);
         Uri uri = new Uri(path);
-        var externalSegmentData = new ExternalSegmentData(uri, method, componentOverride: rpcSystem);
+        var externalSegmentData = new ExternalGrpcSegmentData(uri: uri, method: method ?? cleanGrpcMethod, componentOverride: rpcSystem);
 
         if (statusCode.HasValue)
-            externalSegmentData.SetHttpStatus(statusCode.Value);
+            externalSegmentData.SetGrpcStatus(statusCode.Value);
 
-        Log.Finest($"Created ExternalSegmentData for {activityLogPrefix}.");
+        Log.Finest($"{activityLogPrefix} Created ExternalGrpcSegmentData.");
 
-        segment.GetExperimentalApi().SetSegmentData(externalSegmentData);
+        segment.GetExperimentalApi().SetSegmentData(externalSegmentData)
+            .MakeLeaf();
 
         // per spec, a non-zero status code must be recorded as an exception.
         // TODO: This behavior is supposed to be configurable by the customer but currently is not
@@ -203,7 +216,7 @@ private static string BuildRpcPath(string host, int? port, string service, strin
     {
         // construct the path according to gRPC spec
         // if service is missing, grpcMethod is the full path
-        return !string.IsNullOrEmpty(service) ? $"grpc://{host}:{port}/{service}/{method}" : $"grpc://{host}:{port}{grpcMethod}";
+        return !string.IsNullOrEmpty(service) ? $"grpc://{host}:{port}/{service}/{method}" : $"grpc://{host}:{port}/{grpcMethod}";
     }
 
     private static void RecordGrpcException(ISegment segment, IAgent agent, IErrorService errorService, int statusCode, string path, string activityLogPrefix)

src/Agent/NewRelic/Agent/Core/Segments/ExternalSegmentData.cs

@@ -15,29 +15,20 @@
 
 namespace NewRelic.Agent.Core.Segments;
 
-public class ExternalSegmentData : AbstractSegmentData, IExternalSegmentData
+public class ExternalSegmentData(Uri uri, string method, string library = "Stream", CrossApplicationResponseData crossApplicationResponseData = null, string componentOverride = null) : AbstractSegmentData, IExternalSegmentData
 {
     private const string TransactionGuidSegmentParameterKey = "transaction_guid";
 
     private int? _httpStatusCode;
     private string _httpStatusText;
-    private readonly string _componentOverride;
+    private readonly string _componentOverride = componentOverride;
 
     public override SpanCategory SpanCategory => SpanCategory.Http;
 
-    public Uri Uri { get; }
-    public string Method { get; }
-    public string Type { get; }
-
-    public ExternalSegmentData(Uri uri, string method, CrossApplicationResponseData crossApplicationResponseData = null, string componentOverride = null)
-    {
-        Uri = uri;
-        Method = method;
-        CrossApplicationResponseData = crossApplicationResponseData;
-        _componentOverride = componentOverride;
-    }
-
-    public CrossApplicationResponseData CrossApplicationResponseData { get; set; }
+    public Uri Uri { get; } = uri;
+    public string Method { get; } = method;
+    
+    public CrossApplicationResponseData CrossApplicationResponseData { get; set; } = crossApplicationResponseData;
 
     public void SetHttpStatus(int httpStatusCode, string httpStatusText = null)
     {
@@ -71,12 +62,12 @@ public override void AddMetricStats(Segment segment, TimeSpan durationOfChildren
         if (CrossApplicationResponseData == null)
         {
             // Generate scoped and unscoped external metrics as CAT not present.
-            MetricBuilder.TryBuildExternalSegmentMetric(Uri.Host, Method, duration, exclusiveDuration, txStats, false);
+            MetricBuilder.TryBuildExternalSegmentMetric(Uri.Host, Method, library, duration, exclusiveDuration, txStats, false);
         }
         else
         {
             // Only generate unscoped metric for response with CAT headers because segments should only produce a single scoped metric and the CAT metric is more interesting than the external segment metric.
-            MetricBuilder.TryBuildExternalSegmentMetric(Uri.Host, Method, duration, exclusiveDuration, txStats, true);
+            MetricBuilder.TryBuildExternalSegmentMetric(Uri.Host, Method, library, duration, exclusiveDuration, txStats, true);
 
             var externalCrossProcessId = CrossApplicationResponseData.CrossProcessId;
             var externalTransactionName = CrossApplicationResponseData.TransactionName;
@@ -92,7 +83,7 @@ public override void SetSpanTypeSpecificAttributes(SpanAttributeValueCollection
     {
         AttribDefs.SpanCategory.TrySetValue(attribVals, SpanCategory.Http);
         AttribDefs.HttpUrl.TrySetValue(attribVals, Uri);
-        AttribDefs.HttpMethod.TrySetValue(attribVals, Method);
+        AttribDefs.HttpRequestMethod.TrySetValue(attribVals, Method);
         AttribDefs.Component.TrySetValue(attribVals, _componentOverride ?? _segmentState.TypeName);
         AttribDefs.SpanKind.TrySetDefault(attribVals);
         AttribDefs.HttpStatusCode.TrySetValue(attribVals, _httpStatusCode);   //Attrib handles null
@@ -105,7 +96,7 @@ public override string GetTransactionTraceName()
     {
         // APM expects metric names to be used for external segment trace names
         var name = CrossApplicationResponseData == null
-            ? MetricNames.GetExternalHost(Uri.Host, "Stream", Method)
+            ? MetricNames.GetExternalHost(Uri.Host, library, Method)
             : MetricNames.GetExternalTransaction(Uri.Host, CrossApplicationResponseData.CrossProcessId, CrossApplicationResponseData.TransactionName);
         return name.ToString();
     }
@@ -125,3 +116,27 @@ public override bool IsCombinableWith(AbstractSegmentData otherData)
         return true;
     }
 }
+
+public class ExternalGrpcSegmentData(Uri uri, string method, CrossApplicationResponseData crossApplicationResponseData = null, string componentOverride = null)
+    : ExternalSegmentData(uri, method, "gRPC", crossApplicationResponseData)
+{
+    private int? _grpcStatusCode;
+
+    public void SetGrpcStatus(int grpcStatusCode)
+    {
+        _grpcStatusCode = grpcStatusCode;
+    }
+
+    public override void SetSpanTypeSpecificAttributes(SpanAttributeValueCollection attribVals)
+    {
+        AttribDefs.SpanCategory.TrySetValue(attribVals, SpanCategory.Http);
+        AttribDefs.HttpUrl.TrySetValue(attribVals, Uri);
+        AttribDefs.HttpMethod.TrySetValue(attribVals, Method); //TODO gRPC spec says this is the correct "method" attribute
+        AttribDefs.HttpRequestMethod.TrySetValue(attribVals, Method); //TODO Otel spec says this should be "procedure", but externals use this
+        AttribDefs.Component.TrySetValue(attribVals, componentOverride ?? _segmentState.TypeName);
+        AttribDefs.SpanKind.TrySetDefault(attribVals);
+        AttribDefs.ServerAddress.TrySetValue(attribVals, Uri.Host);
+        AttribDefs.ServerPort.TrySetValue(attribVals, Uri.Port);
+        AttribDefs.GrpcStatusCode.TrySetValue(attribVals, _grpcStatusCode);
+    }
+}

src/Agent/NewRelic/Agent/Core/Segments/NoOpSegment.cs

@@ -5,6 +5,7 @@
 using NewRelic.Agent.Api;
 using NewRelic.Agent.Api.Experimental;
 using NewRelic.Agent.Core.Attributes;
+using NewRelic.Agent.Extensions.Api.Experimental;
 
 namespace NewRelic.Agent.Core.Segments;
 
@@ -83,5 +84,15 @@ public string GetCategory()
         return string.Empty;
     }
 
+    public INewRelicActivity GetActivity()
+    {
+        return null;
+    }
+
+    public void AddCacheItem(string key, object value)
+    {
+        return;
+    }
+
     public TimeSpan DurationOrZero => TimeSpan.Zero;
-}
+}

src/Agent/NewRelic/Agent/Core/Segments/Segment.cs

@@ -97,6 +97,9 @@ public class Segment : IInternalSpan, ISegmentDataState, IHybridAgentSegment
 {
     private static ConfigurationSubscriber _configurationSubscriber = new ConfigurationSubscriber();
 
+    // Used to store _small_ amounts of data for use when creating segment data or other attributes.  This is not intended to be a general purpose storage mechanism and should not be used to store large objects or large amounts of data.
+    private Dictionary<string, object> _dataCache;
+
     public IAttributeDefinitions AttribDefs => _transactionSegmentState.AttribDefs;
     public string TypeName => MethodCallData.TypeName;
 
@@ -124,6 +127,7 @@ public Segment(ITransactionSegmentState transactionSegmentState, MethodCallData
         Combinable = false;
         IsLeaf = false;
         IsAsync = methodCallData.IsAsync;
+        _dataCache = new Dictionary<string, object>();
     }
 
     /// <summary>
@@ -147,6 +151,7 @@ public Segment(ITransactionSegmentState transactionSegmentState, MethodCallData
         Combinable = false;
         IsLeaf = true;
         IsAsync = methodCallData.IsAsync;
+        _dataCache = new Dictionary<string, object>();
     }
 
     /// <summary>
@@ -178,6 +183,7 @@ public Segment(TimeSpan relativeStartTime, TimeSpan? duration, Segment segment,
 
         SpanId = segment.SpanId;
         IsAsync = segment.IsAsync;
+        _dataCache = new Dictionary<string, object>();
     }
 
     public bool IsDone
@@ -283,6 +289,11 @@ public void SetActivity(INewRelicActivity activity)
         _activity = activity;
     }
 
+    public INewRelicActivity GetActivity()
+    {
+        return _activity;
+    }
+
     public void MakeCombinable()
     {
         Combinable = true;
@@ -653,6 +664,31 @@ public string GetCategory()
         return EnumNameCache<SpanCategory>.GetName(Data.SpanCategory);
     }
 
+    // See ISegmentExperimental for details on this method.
+    public void AddCacheItem(string key, object value)
+    {
+        if (string.IsNullOrWhiteSpace(key))
+        {
+            return;
+        }
+
+        _dataCache[key] = value;
+    }
+
+    /// <summary>
+    /// Gets an item from the segment's cache with the specified key.  Returns null if the key is not found or if the key is null or whitespace.
+    /// </summary>
+    /// <param name="key"></param>
+    /// <returns></returns>
+    internal object GetCacheItem(string key)
+    {
+        if (string.IsNullOrWhiteSpace(key) || !_dataCache.TryGetValue(key, out var item))
+        {
+            return null;
+        }
+
+        return item;
+    }
 }
 
 // TODO: Rename this experimental to something else, or find a better way to solve this problem.

src/Agent/NewRelic/Agent/Core/WireModels/MetricWireModel.cs

@@ -361,10 +361,10 @@ public static void TryBuildMessageBrokerSerializationSegmentMetric(string vendor
             txStats.MergeUnscopedStats(proposedName, data);
         }
 
-        public static void TryBuildExternalSegmentMetric(string host, string method, TimeSpan totalTime,
+        public static void TryBuildExternalSegmentMetric(string host, string method, string library, TimeSpan totalTime,
             TimeSpan totalExclusiveTime, TransactionMetricStatsCollection txStats, bool unscopedOnly)
         {
-            var proposedName = MetricNames.GetExternalHost(host, "Stream", method);
+            var proposedName = MetricNames.GetExternalHost(host, library, method);
             var data = MetricDataWireModel.BuildTimingData(totalTime, totalExclusiveTime);
             txStats.MergeUnscopedStats(proposedName, data);
             if (!unscopedOnly)

src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Api/Experimental/ISegmentExperimental.cs

@@ -1,6 +1,8 @@
 // Copyright 2020 New Relic, Inc. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+using NewRelic.Agent.Extensions.Api.Experimental;
+
 namespace NewRelic.Agent.Api.Experimental;
 
 /// <summary>
@@ -54,4 +56,25 @@ public interface ISegmentExperimental
     /// Will be true if a relative end time has been set on the segment.  In most situations, this is only set when a segment is ended.
     /// </summary>
     bool IsDone { get; }
-}
+
+    /// <summary>
+    /// Returns the activity associated with the segment.
+    /// </summary>
+    /// <returns></returns>
+    INewRelicActivity GetActivity();
+
+    /// <summary>
+    /// Add object to the segment's data cache with the specified key.
+    /// This is intended for use in cases where you need to store some data during the execution of a segment that will be used when creating the segment data or attributes.
+    /// This should not be used as a general purpose storage mechanism and should not be used to store large objects or large amounts of data.
+    /// This will overwrite any existing value in the cache with the same key, so it should be used with unique keys to avoid collisions.
+    /// It is the caller's responsibility to manage the keys and ensure they are unique within the context of a segment.
+    ///
+    /// Notes:
+    /// Null keys are not allowed and will be ignored.  If a null key is passed to this method, the method will return without adding anything to the cache.
+    /// Null values are allowed and will be stored in the cache.  If a null value is added to the cache, it will overwrite any existing value with the same key, and a subsequent retrieval of that key from the cache will return null.
+    /// </summary>
+    /// <param name="key"></param>
+    /// <param name="value"></param>
+    void AddCacheItem(string key, object value);
+}

src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/HttpClient/SendAsync.cs

@@ -59,6 +59,19 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins
 
         var transactionExperimental = transaction.GetExperimentalApi();
 
+        // Where we need to find the previous segment or activity and if it is a gRPC segment/activity, we need to set the server.address and server.port attributes on the segment/activity.
+        // After we do this, we need to no-op this so that we only create a single external segment.
+        // This is required for distributed tracing to work properly for gRPC requests.
+        // No segment has been created in this wrapper at this point, so this should be the parent.
+        var currentSegment = (ISegmentExperimental)transaction.CurrentSegment;
+        if (currentSegment.GetActivity().DisplayName == "Grpc.Net.Client.GrpcOut")
+        {
+            // Using uri here since we ensure that it has both host and port information in TryGetAbsoluteUri;
+            currentSegment.AddCacheItem("server.address", uri.Host);
+            currentSegment.AddCacheItem("server.port", uri.Port);
+            return Delegates.NoOp;
+        }
+
         var externalSegmentData = transactionExperimental.CreateExternalSegmentData(uri, method);
         var segment = transactionExperimental.StartSegment(instrumentedMethodCall.MethodCall);
         segment.GetExperimentalApi()
@@ -208,4 +221,4 @@ private static KeyValuePair<string, string> Flatten(KeyValuePair<string, IEnumer
 
         return new KeyValuePair<string, string>(key, flattenedValues);
     }
-}
+}