Allow cross-registry refinements

CHANGELOG.md

@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.
 
 # Unreleased
 
+- Support signal refinements over a published dependency. ([#1587](https://github.com/open-telemetry/weaver/pull/1587) by @lmolkova)
 - Fix panic when a registry, policy, or template path uses a commit SHA. ([#1414](https://github.com/open-telemetry/weaver/pull/1414))
 - Add a stats dashboard with charts to the `serve` UI. ([#1570](https://github.com/open-telemetry/weaver/pull/1570) by @jerbly)
 - Add `semconv_grouped_entities` JQ helper. ([#1560](https://github.com/open-telemetry/weaver/pull/1560) by @lmolkova)

crates/weaver_resolved_schema/src/lineage.rs

@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
 
 use weaver_semconv::attribute::{AttributeRole, AttributeSpec, Examples, RequirementLevel};
 use weaver_semconv::deprecated::Deprecated;
+use weaver_semconv::group::GroupType;
 use weaver_semconv::provenance::Provenance;
 use weaver_semconv::stability::Stability;
 use weaver_semconv::YamlValue;
@@ -43,6 +44,18 @@ pub struct GroupLineage {
     #[serde(default)]
     pub extends_group: Option<String>,
 
+    /// The type of the extended group, resolved from the parent at extends
+    /// resolution time (the parent may live in a dependency). Used to tell a
+    /// refinement from a plain base signal without inferring from the id.
+    ///
+    /// In-memory only: it is derived during resolution and consumed by the
+    /// v1→v2 conversion in the same run, so it is never serialized (keeping
+    /// the resolved file format unchanged).
+    /// TODO: it's only necessary for v1->v2 conversion and
+    /// we should clean this up when removing v1
+    #[serde(skip)]
+    pub extends_group_type: Option<GroupType>,
+
     /// The lineage per attribute.
     ///
     /// Note: Use a BTreeMap to ensure a deterministic order of attributes.
@@ -481,14 +494,16 @@ impl GroupLineage {
         Self {
             provenance,
             extends_group: None,
+            extends_group_type: None,
             attributes: Default::default(),
             includes_group: Default::default(),
         }
     }
 
-    /// Declares this group extended another group.
-    pub fn extends(&mut self, extends_group: &str) {
+    /// Declares this group extended another group of the given type.
+    pub fn extends(&mut self, extends_group: &str, extends_group_type: GroupType) {
         self.extends_group = Some(extends_group.to_owned());
+        self.extends_group_type = Some(extends_group_type);
     }
 
     /// Records what attribute groups were included (v2 only).

crates/weaver_resolved_schema/src/v2/mod.rs

@@ -248,11 +248,6 @@ pub fn convert_v1_to_v2(
 
     let v2_catalog = Catalog::from_attributes(attributes.into_iter().collect());
 
-    // Create a lookup so we can check inheritance.
-    let mut group_type_lookup = HashMap::new();
-    for g in r.groups.iter() {
-        let _ = group_type_lookup.insert(g.id.clone(), g.r#type.clone());
-    }
     // Pull signals from the registry and create a new span-focused registry.
     let mut spans = Vec::new();
     let mut span_refinements = Vec::new();
@@ -269,9 +264,7 @@ pub fn convert_v1_to_v2(
                 let is_refinement = g
                     .lineage
                     .as_ref()
-                    .and_then(|l| l.extends_group.as_ref())
-                    .and_then(|parent| group_type_lookup.get(parent))
-                    .map(|t| *t == GroupType::Span)
+                    .map(|l| l.extends_group_type == Some(GroupType::Span))
                     .unwrap_or(false);
                 // Pull all the attribute references.
                 let mut span_attributes = Vec::new();
@@ -360,9 +353,7 @@ pub fn convert_v1_to_v2(
                 let is_refinement = g
                     .lineage
                     .as_ref()
-                    .and_then(|l| l.extends_group.as_ref())
-                    .and_then(|parent| group_type_lookup.get(parent))
-                    .map(|t| *t == GroupType::Event)
+                    .map(|l| l.extends_group_type == Some(GroupType::Event))
                     .unwrap_or(false);
                 let mut event_attributes = Vec::new();
                 for attr in g.attributes.iter().filter_map(|a| c.attribute(a)) {
@@ -419,9 +410,7 @@ pub fn convert_v1_to_v2(
                 let is_refinement = g
                     .lineage
                     .as_ref()
-                    .and_then(|l| l.extends_group.as_ref())
-                    .and_then(|parent| group_type_lookup.get(parent))
-                    .map(|t| *t == GroupType::Metric)
+                    .map(|l| l.extends_group_type == Some(GroupType::Metric))
                     .unwrap_or(false);
                 let mut metric_attributes = Vec::new();
                 for attr in g.attributes.iter().filter_map(|a| c.attribute(a)) {
@@ -743,7 +732,7 @@ mod tests {
         let v1_catalog = builder.build();
         let mut refinement_span_lineage =
             GroupLineage::new(Provenance::new(SchemaUrl::new_unknown(), "tmp"));
-        refinement_span_lineage.extends("span.my-span");
+        refinement_span_lineage.extends("span.my-span", GroupType::Span);
         refinement_span_lineage
             .add_attribute_lineage("test.key".to_owned(), AttributeLineage::new("span.my-span"));
         let v1_registry = crate::registry::Registry {
@@ -866,7 +855,7 @@ mod tests {
             };
         let mut refinement_lineage =
             GroupLineage::new(Provenance::new(SchemaUrl::new_unknown(), "tmp"));
-        refinement_lineage.extends("span.my-span");
+        refinement_lineage.extends("span.my-span", GroupType::Span);
         let v1_registry = crate::registry::Registry {
             registry_url: "my.schema.url".to_owned(),
             groups: vec![
@@ -961,7 +950,7 @@ mod tests {
         let v1_catalog = builder.build();
         let mut refinement_metric_lineage =
             GroupLineage::new(Provenance::new(SchemaUrl::new_unknown(), "tmp"));
-        refinement_metric_lineage.extends("metric.http");
+        refinement_metric_lineage.extends("metric.http", GroupType::Metric);
         refinement_metric_lineage
             .add_attribute_lineage("test.key".to_owned(), AttributeLineage::new("metric.http"));
         let v1_registry = crate::registry::Registry {

crates/weaver_resolver/data/registry-test-v2-dep/entity_registry/registry/manifest.yaml

@@ -0,0 +1,5 @@
+description: Consumer registry that refines an entity from the server V2 package.
+schema_url: https://entity-consumer.example.com/schemas/1.0.0
+dependencies:
+  - schema_url: https://server.example.com/schemas/1.0.0
+    registry_path: data/registry-test-v2-dep/published

crates/weaver_resolver/data/registry-test-v2-dep/entity_registry/registry/registry.yaml

@@ -0,0 +1,9 @@
+file_format: definition/2
+entity_refinements:
+  # Refines `host` from the dependency: only the brief is refined. The
+  # identity attributes (`host.id` required, `host.arch` recommended) and the
+  # description attribute (`host.name` recommended) are not mentioned and must
+  # be inherited from the dependency with their requirement levels and roles.
+  - id: refined.host
+    ref: host
+    brief: Refined host entity

crates/weaver_resolver/data/registry-test-v2-dep/event_registry/expected-schema.yaml

@@ -0,0 +1,39 @@
+file_format: resolved/2.0
+schema_url: https://event-consumer.example.com/schemas/1.0.0
+attribute_catalog:
+- key: server.address
+  type: string
+  examples:
+  - example.com
+  brief: Different brief
+  note: Different note
+  stability: stable
+- key: server.port
+  type: int
+  examples:
+  - 80
+  - 443
+  brief: Refined server port
+  stability: stable
+registry:
+  attributes: []
+  attribute_groups: []
+  spans: []
+  metrics: []
+  events: []
+  entities: []
+refinements:
+  spans: []
+  metrics: []
+  events:
+  - id: refined.error
+    name: refined.error
+    attributes:
+    - base: 0
+      requirement_level: required
+    - base: 1
+      requirement_level: recommended
+    brief: Refined server error event
+    stability: stable
+dependencies:
+- https://server.example.com/schemas/1.0.0

crates/weaver_resolver/data/registry-test-v2-dep/event_registry/registry/manifest.yaml

@@ -0,0 +1,5 @@
+description: Consumer registry that refines an event from the server V2 package.
+schema_url: https://event-consumer.example.com/schemas/1.0.0
+dependencies:
+  - schema_url: https://server.example.com/schemas/1.0.0
+    registry_path: data/registry-test-v2-dep/published

crates/weaver_resolver/data/registry-test-v2-dep/event_registry/registry/registry.yaml

@@ -0,0 +1,11 @@
+file_format: definition/2
+event_refinements:
+  # Refines `server.error` from the dependency: `server.port` gets a refined
+  # brief; `server.address` is not mentioned and must be inherited with its
+  # `required` requirement level.
+  - id: refined.error
+    ref: server.error
+    brief: Refined server error event
+    attributes:
+      - ref: server.port
+        brief: Refined server port

crates/weaver_resolver/data/registry-test-v2-dep/metric_registry/expected-schema.yaml

@@ -0,0 +1,41 @@
+file_format: resolved/2.0
+schema_url: https://metric-consumer.example.com/schemas/1.0.0
+attribute_catalog:
+- key: server.address
+  type: string
+  examples:
+  - example.com
+  brief: Different brief
+  note: Different note
+  stability: stable
+- key: server.port
+  type: int
+  examples:
+  - 80
+  - 443
+  brief: Refined server port
+  stability: stable
+registry:
+  attributes: []
+  attribute_groups: []
+  spans: []
+  metrics: []
+  events: []
+  entities: []
+refinements:
+  spans: []
+  metrics:
+  - id: refined.duration
+    name: server.request.duration
+    instrument: histogram
+    unit: s
+    attributes:
+    - base: 0
+      requirement_level: required
+    - base: 1
+      requirement_level: recommended
+    brief: Refined server request duration
+    stability: stable
+  events: []
+dependencies:
+- https://server.example.com/schemas/1.0.0

crates/weaver_resolver/data/registry-test-v2-dep/metric_registry/registry/manifest.yaml

@@ -0,0 +1,5 @@
+description: Consumer registry that refines a metric from the server V2 package.
+schema_url: https://metric-consumer.example.com/schemas/1.0.0
+dependencies:
+  - schema_url: https://server.example.com/schemas/1.0.0
+    registry_path: data/registry-test-v2-dep/published

crates/weaver_resolver/data/registry-test-v2-dep/metric_registry/registry/registry.yaml

@@ -0,0 +1,11 @@
+file_format: definition/2
+metric_refinements:
+  # Refines `server.request.duration` from the dependency: `server.port` gets
+  # a refined brief; `server.address` is not mentioned and must be inherited
+  # from the dependency with its `required` requirement level.
+  - id: refined.duration
+    ref: server.request.duration
+    brief: Refined server request duration
+    attributes:
+      - ref: server.port
+        brief: Refined server port

crates/weaver_resolver/data/registry-test-v2-dep/published/resolved.yaml

@@ -21,12 +21,45 @@ attribute_catalog:
   - example.com
   brief: Server address.
   stability: stable
+- key: host.id
+  type: string
+  examples:
+  - fdbf79e8af94cb7f9e8df36789187052
+  brief: Unique host ID.
+  stability: stable
+- key: host.name
+  type: string
+  examples:
+  - opentelemetry-test
+  brief: Name of the host.
+  stability: stable
+- key: host.arch
+  type: string
+  examples:
+  - amd64
+  brief: Host architecture.
+  stability: stable
 registry:
   attributes:
   - 2
   - 0
+  - 3
+  - 4
+  - 5
   attribute_groups: []
-  spans: []
+  spans:
+  - type: server.request
+    kind: server
+    name:
+      note: Server request span name.
+    attributes:
+    - base: 1
+      requirement_level: required
+      sampling_relevant: true
+    - base: 0
+      requirement_level: recommended
+    brief: A server request span.
+    stability: stable
   metrics:
   - name: server.request.duration
     instrument: histogram
@@ -38,10 +71,42 @@ registry:
       requirement_level: recommended
     brief: Duration of server requests.
     stability: stable
-  events: []
-  entities: []
+  events:
+  - name: server.error
+    attributes:
+    - base: 1
+      requirement_level: required
+    - base: 0
+      requirement_level: recommended
+    brief: A server error event.
+    stability: stable
+  entities:
+  - type: host
+    identity:
+    - base: 3
+      requirement_level: required
+    - base: 5
+      requirement_level: recommended
+    description:
+    - base: 4
+      requirement_level: recommended
+    brief: A host.
+    stability: stable
 refinements:
-  spans: []
+  spans:
+  - id: server.request
+    type: server.request
+    kind: server
+    name:
+      note: Server request span name.
+    attributes:
+    - base: 1
+      requirement_level: required
+      sampling_relevant: true
+    - base: 0
+      requirement_level: recommended
+    brief: A server request span.
+    stability: stable
   metrics:
   - id: server.request.duration
     name: server.request.duration
@@ -54,4 +119,13 @@ refinements:
       requirement_level: recommended
     brief: Duration of server requests.
     stability: stable
-  events: []
+  events:
+  - id: server.error
+    name: server.error
+    attributes:
+    - base: 1
+      requirement_level: required
+    - base: 0
+      requirement_level: recommended
+    brief: A server error event.
+    stability: stable