[mdatagen] Add semconv reference for attributes in metadata schema (#14646)

#### Description

This PR adds support in mdatagen for defining semantic conventions
references for attributes.

Updating your attribute in `metadata.yaml`:

```yaml
attributes:
  state:
    description: Breakdown of memory usage by type.
    type: string
    enum: [buffered, cached, inactive, free, slab_reclaimable, slab_unreclaimable, used]
    semantic_convention:
      ref: system.md#system-memory-state
```

> [!IMPORTANT]
> This work also updates the `semantic_convention: ref:` to use a
relative URL, this was done as we want to ensure the same semconv
version from the metadata schema is used, otherwise the end user could
have a full url in `ref` with one semconv version and the schema version
being a different version. A new method has been introduced in mdatagen
that builds the semconv url at runtime.

Creates this table in the generated docs for that component:

#### Attributes

| Name | Description | Values | Requirement Level | Semantic Convention
|
| ---- | ----------- | ------ | ----------------- | -------------------
|
| state | Breakdown of memory usage by type. | Str: ``buffered``,
``cached``, ``inactive``, ``free``, ``slab_reclaimable``,
``slab_unreclaimable``, ``used`` | Recommended |
[state](https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/registry/attributes/system.md#system-memory-state)
|

<!-- Issue number if applicable -->
#### Link to tracking issue
This work is linked to #13297 but doesn't close it.

<!--Describe what testing was performed and which tests were added.-->
#### Testing 

- loader_test.go was updated with attributes for `cpu` and `state`.
- Also `cmd/mdatagen/internal/samplereciever` was updated and generate
code created which creates tests.

<!--Describe the documentation added.-->
#### Documentation

The documentation template has been updated to reference the semconv
definition for attributes if provided in the schema.

Author: osullivandonal

.chloggen/mdatagen-add-attribute-semconv-refs.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/otlp)
+component: cmd/mdatagen
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add semconv reference for attributes
+
+# One or more tracking issues or pull requests related to the change
+issues: [13297]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.github/workflows/utils/cspell.json

@@ -464,6 +464,8 @@
       "subpackages",
       "swiatekm",
       "syft",
+      "systemcputime",
+      "systemdiskio",
       "tailsampling",
       "tchannel",
       "telemetrygen",
@@ -498,6 +500,7 @@
       "unmarshal",
       "unmarshalling",
       "unmarshalls",
+      "unreclaimable",
       "unredacted",
       "unshallow",
       "unstarted",

cmd/mdatagen/internal/loader.go

@@ -62,6 +62,10 @@ func LoadMetadata(filePath string) (Metadata, error) {
 		md.GeneratedPackageName = "metadata"
 	}
 
+	if err := md.expandSemConvRefs(); err != nil {
+		return md, err
+	}
+
 	if err := md.Validate(); err != nil {
 		return md, err
 	}

cmd/mdatagen/internal/loader_test.go

@@ -183,6 +183,14 @@ func TestLoadMetadata(t *testing.T) {
 				},
 
 				Attributes: map[AttributeName]Attribute{
+					"cpu": {
+						Description: "Logical CPU number starting at 0.",
+						Type: ValueType{
+							ValueType: pcommon.ValueTypeStr,
+						},
+						FullName:         "cpu",
+						RequirementLevel: AttributeRequirementLevelRecommended,
+					},
 					"enum_attr": {
 						Description:  "Attribute with a known set of string values.",
 						NameOverride: "",
@@ -275,6 +283,18 @@ func TestLoadMetadata(t *testing.T) {
 						FullName:         "required_string_attr",
 						RequirementLevel: AttributeRequirementLevelRequired,
 					},
+					"state": {
+						Description: "Breakdown of memory usage by type.",
+						Enum:        []string{"buffered", "cached", "inactive", "free", "slab_reclaimable", "slab_unreclaimable", "used"},
+						Type: ValueType{
+							ValueType: pcommon.ValueTypeStr,
+						},
+						FullName:         "state",
+						RequirementLevel: AttributeRequirementLevelRecommended,
+						SemanticConvention: &SemanticConvention{
+							SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/registry/attributes/system.md#system-memory-state",
+						},
+					},
 				},
 				Metrics: map[MetricName]Metric{
 					"default.metric": {
@@ -330,6 +350,7 @@ func TestLoadMetadata(t *testing.T) {
 							SemanticConvention:    &SemanticConvention{SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime"},
 							Description:           "Monotonic cumulative sum int metric enabled by default.",
 							ExtendedDocumentation: "The metric will be become optional soon.",
+							Attributes:            []AttributeName{"cpu"},
 						},
 						Unit: strPtr("s"),
 						Sum: &Sum{
@@ -338,6 +359,20 @@ func TestLoadMetadata(t *testing.T) {
 							Mono:                   Mono{Monotonic: true},
 						},
 					},
+					"system.memory.usage": {
+						Signal: Signal{
+							Enabled:     true,
+							Stability:   component.StabilityLevelDevelopment,
+							Description: "Bytes of memory in use.",
+							Attributes:  []AttributeName{"state"},
+						},
+						Unit: strPtr("By"),
+						Sum: &Sum{
+							MetricValueType:        MetricValueType{pmetric.NumberDataPointValueTypeInt},
+							AggregationTemporality: AggregationTemporality{Aggregation: pmetric.AggregationTemporalityCumulative},
+							Mono:                   Mono{Monotonic: false},
+						},
+					},
 					"optional.metric": {
 						Signal: Signal{
 							Enabled:     false,
@@ -629,7 +664,7 @@ func TestLoadMetadata(t *testing.T) {
 		{
 			name:    "testdata/invalid_metric_semconvref.yaml",
 			want:    Metadata{},
-			wantErr: "metric \"default.metric\": invalid semantic-conventions URL: want https://github.com/open-telemetry/semantic-conventions/blob/v1.37.2/*#metric-defaultmetric, got \"https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime\"",
+			wantErr: "metric \"default.metric\": invalid semantic-conventions URL: want https://github.com/open-telemetry/semantic-conventions/blob/v1.37.2/*#metric-defaultmetric, got \"https://github.com/open-telemetry/semantic-conventions/blob/v1.37.2/docs/system/system-metrics.md#metric-systemcputime\"",
 		},
 		{
 			name:    "testdata/no_metric_stability.yaml",
@@ -646,6 +681,16 @@ func TestLoadMetadata(t *testing.T) {
 			want:    Metadata{},
 			wantErr: "config type must be \"object\", got \"string\"",
 		},
+		{
+			name:    "testdata/invalid_metric_semconv_url_full.yaml",
+			want:    Metadata{},
+			wantErr: "metric \"default.metric\", use relative path for URL, not the full URL",
+		},
+		{
+			name:    "testdata/invalid_attribute_semconv_url_full.yaml",
+			want:    Metadata{},
+			wantErr: "attribute \"used_attr\", use relative path for URL, not the full URL",
+		},
 		{
 			name:    "testdata/~~this file doesn't exist~~.yaml",
 			wantErr: "unable to read the file file:testdata/~~this file doesn't exist~~.yaml",

cmd/mdatagen/internal/metadata.go

@@ -18,6 +18,8 @@ import (
 	"go.opentelemetry.io/collector/pdata/pcommon"
 )
 
+const semConvURL = "https://github.com/open-telemetry/semantic-conventions/blob"
+
 type Metadata struct {
 	// Type of the component.
 	Type string `mapstructure:"type"`
@@ -581,6 +583,8 @@ type Attribute struct {
 	Warnings Warnings `mapstructure:"warnings"`
 	// RequirementLevel defines the requirement level of the attribute.
 	RequirementLevel AttributeRequirementLevel `mapstructure:"requirement_level"`
+	// The semantic convention reference of the attribute.
+	SemanticConvention *SemanticConvention `mapstructure:"semantic_convention"`
 }
 
 // IsConditional returns true if the attribute is conditionally required.
@@ -785,3 +789,39 @@ type FeatureGate struct {
 	// ReferenceURL is the URL with contextual information about the feature gate.
 	ReferenceURL string `mapstructure:"reference_url"`
 }
+
+func (md *Metadata) expandSemConvRefs() error {
+	for k, v := range md.Attributes {
+		if v.SemanticConvention != nil {
+			if strings.HasPrefix(v.SemanticConvention.SemanticConventionRef, "http") {
+				return fmt.Errorf("attribute %q, use relative path for URL, not the full URL", k)
+			}
+			url := fmt.Sprintf(
+				"%s/v%s/docs/registry/attributes/%s",
+				semConvURL,
+				md.SemConvVersion,
+				v.SemanticConvention.SemanticConventionRef,
+			)
+			v.SemanticConvention.SemanticConventionRef = url
+		}
+		md.Attributes[k] = v
+	}
+
+	for k, v := range md.Metrics {
+		if v.SemanticConvention != nil {
+			if strings.HasPrefix(v.SemanticConvention.SemanticConventionRef, "http") {
+				return fmt.Errorf("metric %q, use relative path for URL, not the full URL", k)
+			}
+			url := fmt.Sprintf(
+				"%s/v%s/docs/%s",
+				semConvURL,
+				md.SemConvVersion,
+				v.SemanticConvention.SemanticConventionRef,
+			)
+			v.SemanticConvention.SemanticConventionRef = url
+		}
+		md.Metrics[k] = v
+	}
+
+	return nil
+}

cmd/mdatagen/internal/sampleconnector/documentation.md

@@ -24,13 +24,13 @@ The metric will be become optional soon.
 
 #### Attributes
 
-| Name | Description | Values | Requirement Level |
-| ---- | ----------- | ------ | -------- |
-| string_attr | Attribute with any string value. | Any Str | Recommended |
-| state | Integer attribute with overridden name. | Any Int | Recommended |
-| enum_attr | Attribute with a known set of string values. | Str: ``red``, ``green``, ``blue`` | Recommended |
-| slice_attr | Attribute with a slice value. | Any Slice | Recommended |
-| map_attr | Attribute with a map value. | Any Map | Recommended |
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| string_attr | Attribute with any string value. | Any Str | Recommended | - |
+| state | Integer attribute with overridden name. | Any Int | Recommended | - |
+| enum_attr | Attribute with a known set of string values. | Str: ``red``, ``green``, ``blue`` | Recommended | - |
+| slice_attr | Attribute with a slice value. | Any Slice | Recommended | - |
+| map_attr | Attribute with a map value. | Any Map | Recommended | - |
 
 ### default.metric.to_be_removed
 
@@ -54,13 +54,13 @@ Monotonic cumulative sum int metric with string input_type enabled by default.
 
 #### Attributes
 
-| Name | Description | Values | Requirement Level |
-| ---- | ----------- | ------ | -------- |
-| string_attr | Attribute with any string value. | Any Str | Recommended |
-| state | Integer attribute with overridden name. | Any Int | Recommended |
-| enum_attr | Attribute with a known set of string values. | Str: ``red``, ``green``, ``blue`` | Recommended |
-| slice_attr | Attribute with a slice value. | Any Slice | Recommended |
-| map_attr | Attribute with a map value. | Any Map | Recommended |
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| string_attr | Attribute with any string value. | Any Str | Recommended | - |
+| state | Integer attribute with overridden name. | Any Int | Recommended | - |
+| enum_attr | Attribute with a known set of string values. | Str: ``red``, ``green``, ``blue`` | Recommended | - |
+| slice_attr | Attribute with a slice value. | Any Slice | Recommended | - |
+| map_attr | Attribute with a map value. | Any Map | Recommended | - |
 
 ### reaggregate.metric
 
@@ -72,10 +72,10 @@ Metric for testing spatial reaggregation
 
 #### Attributes
 
-| Name | Description | Values | Requirement Level |
-| ---- | ----------- | ------ | -------- |
-| string_attr | Attribute with any string value. | Any Str | Recommended |
-| boolean_attr | Attribute with a boolean value. | Any Bool | Recommended |
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| string_attr | Attribute with any string value. | Any Str | Recommended | - |
+| boolean_attr | Attribute with a boolean value. | Any Bool | Recommended | - |
 
 ## Optional Metrics
 
@@ -99,11 +99,11 @@ metrics:
 
 #### Attributes
 
-| Name | Description | Values | Requirement Level |
-| ---- | ----------- | ------ | -------- |
-| string_attr | Attribute with any string value. | Any Str | Recommended |
-| boolean_attr | Attribute with a boolean value. | Any Bool | Recommended |
-| boolean_attr2 | Another attribute with a boolean value. | Any Bool | Recommended |
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| string_attr | Attribute with any string value. | Any Str | Recommended | - |
+| boolean_attr | Attribute with a boolean value. | Any Bool | Recommended | - |
+| boolean_attr2 | Another attribute with a boolean value. | Any Bool | Recommended | - |
 
 ### optional.metric.empty_unit
 
@@ -117,23 +117,23 @@ metrics:
 
 #### Attributes
 
-| Name | Description | Values | Requirement Level |
-| ---- | ----------- | ------ | -------- |
-| string_attr | Attribute with any string value. | Any Str | Recommended |
-| boolean_attr | Attribute with a boolean value. | Any Bool | Recommended |
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| string_attr | Attribute with any string value. | Any Str | Recommended | - |
+| boolean_attr | Attribute with a boolean value. | Any Bool | Recommended | - |
 
 ## Resource Attributes
 
-| Name | Description | Values | Enabled |
-| ---- | ----------- | ------ | ------- |
-| map.resource.attr | Resource attribute with a map value. | Any Map | true |
-| optional.resource.attr | Explicitly disabled ResourceAttribute. | Any Str | false |
-| slice.resource.attr | Resource attribute with a slice value. | Any Slice | true |
-| string.enum.resource.attr | Resource attribute with a known set of string values. | Str: ``one``, ``two`` | true |
-| string.resource.attr | Resource attribute with any string value. | Any Str | true |
-| string.resource.attr_disable_warning | Resource attribute with any string value. | Any Str | true |
-| string.resource.attr_remove_warning | Resource attribute with any string value. | Any Str | false |
-| string.resource.attr_to_be_removed | Resource attribute with any string value. | Any Str | true |
+| Name | Description | Values | Enabled | Semantic Convention |
+| ---- | ----------- | ------ | ------- | ------------------- |
+| map.resource.attr | Resource attribute with a map value. | Any Map | true | - |
+| optional.resource.attr | Explicitly disabled ResourceAttribute. | Any Str | false | - |
+| slice.resource.attr | Resource attribute with a slice value. | Any Slice | true | - |
+| string.enum.resource.attr | Resource attribute with a known set of string values. | Str: ``one``, ``two`` | true | - |
+| string.resource.attr | Resource attribute with any string value. | Any Str | true | - |
+| string.resource.attr_disable_warning | Resource attribute with any string value. | Any Str | true | - |
+| string.resource.attr_remove_warning | Resource attribute with any string value. | Any Str | false | - |
+| string.resource.attr_to_be_removed | Resource attribute with any string value. | Any Str | true | - |
 
 ## Entities
 

cmd/mdatagen/internal/sampleentityreceiver/documentation.md

@@ -30,9 +30,9 @@ Current phase of the pod
 
 #### Attributes
 
-| Name | Description | Values | Requirement Level |
-| ---- | ----------- | ------ | -------- |
-| phase | The phase of the pod (Pending, Running, Succeeded, Failed, Unknown) | Str: ``Pending``, ``Running``, ``Succeeded``, ``Failed``, ``Unknown`` | Recommended |
+| Name | Description | Values | Requirement Level | Semantic Convention |
+| ---- | ----------- | ------ | ----------------- | ------------------- |
+| phase | The phase of the pod (Pending, Running, Succeeded, Failed, Unknown) | Str: ``Pending``, ``Running``, ``Succeeded``, ``Failed``, ``Unknown`` | Recommended | - |
 
 ### k8s.replicaset.desired
 
@@ -44,13 +44,13 @@ Number of desired replicas
 
 ## Resource Attributes
 
-| Name | Description | Values | Enabled |
-| ---- | ----------- | ------ | ------- |
-| k8s.namespace.name | The name of the Kubernetes Namespace | Any Str | true |
-| k8s.pod.name | The name of the Kubernetes Pod | Any Str | true |
-| k8s.pod.uid | The UID of the Kubernetes Pod | Any Str | true |
-| k8s.replicaset.name | The name of the Kubernetes ReplicaSet | Any Str | true |
-| k8s.replicaset.uid | The UID of the Kubernetes ReplicaSet | Any Str | true |
+| Name | Description | Values | Enabled | Semantic Convention |
+| ---- | ----------- | ------ | ------- | ------------------- |
+| k8s.namespace.name | The name of the Kubernetes Namespace | Any Str | true | - |
+| k8s.pod.name | The name of the Kubernetes Pod | Any Str | true | - |
+| k8s.pod.uid | The UID of the Kubernetes Pod | Any Str | true | - |
+| k8s.replicaset.name | The name of the Kubernetes ReplicaSet | Any Str | true | - |
+| k8s.replicaset.uid | The UID of the Kubernetes ReplicaSet | Any Str | true | - |
 
 ## Entities
 

cmd/mdatagen/internal/sampleprocessor/documentation.md

@@ -4,13 +4,13 @@
 
 ## Resource Attributes
 
-| Name | Description | Values | Enabled |
-| ---- | ----------- | ------ | ------- |
-| map.resource.attr | Resource attribute with a map value. | Any Map | true |
-| optional.resource.attr | Explicitly disabled ResourceAttribute. | Any Str | false |
-| slice.resource.attr | Resource attribute with a slice value. | Any Slice | true |
-| string.enum.resource.attr | Resource attribute with a known set of string values. | Str: ``one``, ``two`` | true |
-| string.resource.attr | Resource attribute with any string value. | Any Str | true |
-| string.resource.attr_disable_warning | Resource attribute with any string value. | Any Str | true |
-| string.resource.attr_remove_warning | Resource attribute with any string value. | Any Str | false |
-| string.resource.attr_to_be_removed | Resource attribute with any string value. | Any Str | true |
+| Name | Description | Values | Enabled | Semantic Convention |
+| ---- | ----------- | ------ | ------- | ------------------- |
+| map.resource.attr | Resource attribute with a map value. | Any Map | true | - |
+| optional.resource.attr | Explicitly disabled ResourceAttribute. | Any Str | false | - |
+| slice.resource.attr | Resource attribute with a slice value. | Any Slice | true | - |
+| string.enum.resource.attr | Resource attribute with a known set of string values. | Str: ``one``, ``two`` | true | - |
+| string.resource.attr | Resource attribute with any string value. | Any Str | true | - |
+| string.resource.attr_disable_warning | Resource attribute with any string value. | Any Str | true | - |
+| string.resource.attr_remove_warning | Resource attribute with any string value. | Any Str | false | - |
+| string.resource.attr_to_be_removed | Resource attribute with any string value. | Any Str | true | - |