Skip to content

Commit 699e737

Browse files
triviajontriviajon
authored
Merge branch 'main' into triviajon/CONTINT-5388
2 parents 72b4cfc + 9847ec0 commit 699e737

7 files changed

Lines changed: 244 additions & 34 deletions

File tree

‎pkg/collector/corechecks/gpu/nvidia/stateless.go‎

Lines changed: 41 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,45 @@ func pcieLinkBytesPerSecond(gen int, width int) (float64, error) {
330330
return bps, nil
331331
}
332332

333+
func pcieLinkMetrics(device ddnvml.Device) ([]Metric, uint64, error) {
334+
var metricsOut []Metric
335+
336+
currentWidth, err := device.GetCurrPcieLinkWidth()
337+
if err != nil {
338+
return metricsOut, 0, fmt.Errorf("get current PCIe link width: %w", err)
339+
}
340+
metricsOut = append(metricsOut, Metric{Name: "pci.link.width.current", Value: float64(currentWidth), Type: metrics.GaugeType})
341+
342+
maxWidth, err := device.GetMaxPcieLinkWidth()
343+
if err != nil {
344+
return metricsOut, 0, fmt.Errorf("get max PCIe link width: %w", err)
345+
}
346+
metricsOut = append(metricsOut, Metric{Name: "pci.link.width.max", Value: float64(maxWidth), Type: metrics.GaugeType})
347+
metricsOut = append(metricsOut, Metric{Name: "pci.link.width.degraded", Value: boolToFloat(currentWidth < maxWidth), Type: metrics.GaugeType})
348+
349+
currentGeneration, err := device.GetCurrPcieLinkGeneration()
350+
if err != nil {
351+
return metricsOut, 0, fmt.Errorf("get current PCIe link generation: %w", err)
352+
}
353+
currentSpeed, err := pcieLinkBytesPerSecond(currentGeneration, currentWidth)
354+
if err != nil {
355+
return metricsOut, 0, fmt.Errorf("compute current PCIe link speed: %w", err)
356+
}
357+
metricsOut = append(metricsOut, Metric{Name: "pci.link.speed.current", Value: currentSpeed, Type: metrics.GaugeType})
358+
359+
maxGeneration, err := device.GetMaxPcieLinkGeneration()
360+
if err != nil {
361+
return metricsOut, 0, fmt.Errorf("get max PCIe link generation: %w", err)
362+
}
363+
maxSpeed, err := pcieLinkBytesPerSecond(maxGeneration, maxWidth)
364+
if err != nil {
365+
return metricsOut, 0, fmt.Errorf("compute max PCIe link speed: %w", err)
366+
}
367+
metricsOut = append(metricsOut, Metric{Name: "pci.link.speed.max", Value: maxSpeed, Type: metrics.GaugeType})
368+
metricsOut = append(metricsOut, Metric{Name: "pci.link.speed.degraded", Value: boolToFloat(currentSpeed < maxSpeed), Type: metrics.GaugeType})
369+
return metricsOut, 0, nil
370+
}
371+
333372
// createStatelessAPIs creates API call definitions for all stateless metrics on demand
334373
func createStatelessAPIs(deps *CollectorDependencies) []apiCallInfo {
335374
apis := []apiCallInfo{
@@ -401,39 +440,9 @@ func createStatelessAPIs(deps *CollectorDependencies) []apiCallInfo {
401440
},
402441
},
403442
{
404-
Name: "pci_link_speed_current",
443+
Name: "pci_link",
405444
Handler: func(device ddnvml.Device, _ uint64) ([]Metric, uint64, error) {
406-
gen, err := device.GetCurrPcieLinkGeneration()
407-
if err != nil {
408-
return nil, 0, err
409-
}
410-
width, err := device.GetCurrPcieLinkWidth()
411-
if err != nil {
412-
return nil, 0, err
413-
}
414-
speed, err := pcieLinkBytesPerSecond(gen, width)
415-
if err != nil {
416-
return nil, 0, err
417-
}
418-
return []Metric{{Name: "pci.link.speed.current", Value: speed, Type: metrics.GaugeType}}, 0, nil
419-
},
420-
},
421-
{
422-
Name: "pci_link_speed_max",
423-
Handler: func(device ddnvml.Device, _ uint64) ([]Metric, uint64, error) {
424-
gen, err := device.GetMaxPcieLinkGeneration()
425-
if err != nil {
426-
return nil, 0, err
427-
}
428-
width, err := device.GetMaxPcieLinkWidth()
429-
if err != nil {
430-
return nil, 0, err
431-
}
432-
speed, err := pcieLinkBytesPerSecond(gen, width)
433-
if err != nil {
434-
return nil, 0, err
435-
}
436-
return []Metric{{Name: "pci.link.speed.max", Value: speed, Type: metrics.GaugeType}}, 0, nil
445+
return pcieLinkMetrics(device)
437446
},
438447
},
439448
{

‎pkg/collector/corechecks/gpu/nvidia/stateless_test.go‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -727,6 +727,134 @@ func TestPCIELinkBytesPerSecond(t *testing.T) {
727727
}
728728
}
729729

730+
func TestPCIELinkMetrics(t *testing.T) {
731+
tests := map[string]struct {
732+
currentWidth int
733+
maxWidth int
734+
currentGeneration int
735+
maxGeneration int
736+
expectedMetricVals map[string]float64
737+
expectedErr string
738+
}{
739+
"matching link": {
740+
currentWidth: 16,
741+
maxWidth: 16,
742+
currentGeneration: 4,
743+
maxGeneration: 4,
744+
expectedMetricVals: map[string]float64{
745+
"pci.link.width.current": 16,
746+
"pci.link.width.max": 16,
747+
"pci.link.width.degraded": 0,
748+
"pci.link.speed.current": 31.50769230769231e9,
749+
"pci.link.speed.max": 31.50769230769231e9,
750+
"pci.link.speed.degraded": 0,
751+
},
752+
},
753+
"degraded link": {
754+
currentWidth: 8,
755+
maxWidth: 16,
756+
currentGeneration: 4,
757+
maxGeneration: 4,
758+
expectedMetricVals: map[string]float64{
759+
"pci.link.width.current": 8,
760+
"pci.link.width.max": 16,
761+
"pci.link.width.degraded": 1,
762+
"pci.link.speed.current": 15.753846153846155e9,
763+
"pci.link.speed.max": 31.50769230769231e9,
764+
"pci.link.speed.degraded": 1,
765+
},
766+
},
767+
"current width error": {
768+
currentWidth: -1,
769+
maxWidth: 16,
770+
currentGeneration: 4,
771+
maxGeneration: 4,
772+
expectedErr: "get current PCIe link width",
773+
},
774+
"max width error emits current width": {
775+
currentWidth: 16,
776+
maxWidth: -1,
777+
currentGeneration: 4,
778+
maxGeneration: 4,
779+
expectedMetricVals: map[string]float64{
780+
"pci.link.width.current": 16,
781+
},
782+
expectedErr: "get max PCIe link width",
783+
},
784+
"current generation error emits width metrics": {
785+
currentWidth: 8,
786+
maxWidth: 16,
787+
currentGeneration: -1,
788+
maxGeneration: 4,
789+
expectedMetricVals: map[string]float64{
790+
"pci.link.width.current": 8,
791+
"pci.link.width.max": 16,
792+
"pci.link.width.degraded": 1,
793+
},
794+
expectedErr: "get current PCIe link generation",
795+
},
796+
"max generation error emits current speed": {
797+
currentWidth: 8,
798+
maxWidth: 16,
799+
currentGeneration: 4,
800+
maxGeneration: -1,
801+
expectedMetricVals: map[string]float64{
802+
"pci.link.width.current": 8,
803+
"pci.link.width.max": 16,
804+
"pci.link.width.degraded": 1,
805+
"pci.link.speed.current": 15.753846153846155e9,
806+
},
807+
expectedErr: "get max PCIe link generation",
808+
},
809+
}
810+
811+
for name, test := range tests {
812+
t.Run(name, func(t *testing.T) {
813+
device := setupMockDevice(t, testutil.WithCustomHook(func(device *mock.Device) {
814+
device.GetCurrPcieLinkWidthFunc = func() (int, nvml.Return) {
815+
if test.currentWidth == -1 {
816+
return 0, nvml.ERROR_UNKNOWN
817+
}
818+
return test.currentWidth, nvml.SUCCESS
819+
}
820+
device.GetMaxPcieLinkWidthFunc = func() (int, nvml.Return) {
821+
if test.maxWidth == -1 {
822+
return 0, nvml.ERROR_UNKNOWN
823+
}
824+
return test.maxWidth, nvml.SUCCESS
825+
}
826+
device.GetCurrPcieLinkGenerationFunc = func() (int, nvml.Return) {
827+
if test.currentGeneration == -1 {
828+
return 0, nvml.ERROR_UNKNOWN
829+
}
830+
return test.currentGeneration, nvml.SUCCESS
831+
}
832+
device.GetMaxPcieLinkGenerationFunc = func() (int, nvml.Return) {
833+
if test.maxGeneration == -1 {
834+
return 0, nvml.ERROR_UNKNOWN
835+
}
836+
return test.maxGeneration, nvml.SUCCESS
837+
}
838+
}))
839+
840+
metricsOut, _, err := pcieLinkMetrics(device)
841+
if test.expectedErr == "" {
842+
require.NoError(t, err)
843+
} else {
844+
require.ErrorContains(t, err, test.expectedErr)
845+
}
846+
require.Len(t, metricsOut, len(test.expectedMetricVals))
847+
metricsByName := metricValuesToPointers(metricsOut)
848+
for metricName, expectedValue := range test.expectedMetricVals {
849+
metric := findMetric(metricsByName, metricName)
850+
require.NotNil(t, metric, "expected metric %s", metricName)
851+
require.InDelta(t, expectedValue, metric.Value, 1e6)
852+
require.Equal(t, metrics.GaugeType, metric.Type)
853+
}
854+
})
855+
}
856+
}
857+
730858
func findAPICallByName(t *testing.T, apis []apiCallInfo, name string) apiCallInfo {
731859
t.Helper()
732860
for _, api := range apis {

‎pkg/collector/corechecks/gpu/spec/gpu_metrics.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1695,6 +1695,62 @@ metrics:
16951695
mig: false
16961696
physical: true
16971697
vgpu: false
1698+
pci.link.speed.degraded:
1699+
metadata:
1700+
metric_type: gauge
1701+
description: Whether the current PCI link speed is lower than the max PCI link speed
1702+
tagsets:
1703+
- device
1704+
validator:
1705+
values: [0, 1]
1706+
support:
1707+
device_modes:
1708+
mig: false
1709+
physical: true
1710+
vgpu: false
1711+
pci.link.width.current:
1712+
metadata:
1713+
metric_type: gauge
1714+
description: Current lane width for the PCI link
1715+
tagsets:
1716+
- device
1717+
validator:
1718+
range:
1719+
min: 0
1720+
max: 32
1721+
support:
1722+
device_modes:
1723+
mig: false
1724+
physical: true
1725+
vgpu: false
1726+
pci.link.width.max:
1727+
metadata:
1728+
metric_type: gauge
1729+
description: Max lane width for the PCI link
1730+
tagsets:
1731+
- device
1732+
validator:
1733+
range:
1734+
min: 0
1735+
max: 32
1736+
support:
1737+
device_modes:
1738+
mig: false
1739+
physical: true
1740+
vgpu: false
1741+
pci.link.width.degraded:
1742+
metadata:
1743+
metric_type: gauge
1744+
description: Whether the current PCI link width is lower than the max PCI link width
1745+
tagsets:
1746+
- device
1747+
validator:
1748+
values: [0, 1]
1749+
support:
1750+
device_modes:
1751+
mig: false
1752+
physical: true
1753+
vgpu: false
16981754
performance_state:
16991755
metadata:
17001756
metric_type: gauge

‎pkg/config/schema/yaml/system-probe_schema.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@ properties:
511511
remote_config_btf_enabled:
512512
node_type: setting
513513
type: boolean
514-
default: false
514+
default: true
515515
env_vars:
516516
- DD_SYSTEM_PROBE_REMOTE_CONFIG_BTF_ENABLED
517517
tags: []

‎pkg/config/setup/system_probe.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Setup) {
134134
cfg.BindEnvAndSetDefault("system_probe_config.enable_co_re", true, "DD_ENABLE_CO_RE")
135135
cfg.BindEnvAndSetDefault("system_probe_config.btf_path", "", "DD_SYSTEM_PROBE_BTF_PATH")
136136
cfg.BindEnvAndSetDefault("system_probe_config.btf_output_dir", defaultBTFOutputDir, "DD_SYSTEM_PROBE_BTF_OUTPUT_DIR")
137-
cfg.BindEnvAndSetDefault("system_probe_config.remote_config_btf_enabled", false, "DD_SYSTEM_PROBE_REMOTE_CONFIG_BTF_ENABLED")
137+
cfg.BindEnvAndSetDefault("system_probe_config.remote_config_btf_enabled", true, "DD_SYSTEM_PROBE_REMOTE_CONFIG_BTF_ENABLED")
138138
cfg.BindEnvAndSetDefault("system_probe_config.enable_runtime_compiler", false, "DD_ENABLE_RUNTIME_COMPILER")
139139
// deprecated in favor of allow_prebuilt_fallback below
140140
cfg.BindEnvAndSetDefault("system_probe_config.allow_precompiled_fallback", false, "DD_ALLOW_PRECOMPILED_FALLBACK")

‎releasenotes/notes/gpu-pci-19836f335c56c50a.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Each section from every release note are combined when the
2+
# CHANGELOG.rst is rendered. So the text needs to be worded so that
3+
# it does not depend on any information only available in another
4+
# section. This may mean repeating some details, but each section
5+
# must be readable independently of the other.
6+
#
7+
# Each section note must be formatted as reStructuredText.
8+
---
9+
enhancements:
10+
- |
11+
gpu: add new PCI link width metrics ``gpu.pci.link.width.{current,max}`` and add degraded PCI link metrics ``gpu.pci.link.{width,speed}.degraded``.

‎releasenotes/notes/sysprobe-rc-btf-default-652a26ca81ee6bfd.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
enhancements:
3+
- |
4+
System Probe will now download BTF (BPF Type Format) data, if needed, to support eBPF-based features.
5+
This behavior will only take effect in environments where the Linux kernel is newer than the Agent release
6+
and BTF is not available directly from the kernel.

0 commit comments

Comments
 (0)