Skip to content

[SDTEST-3776] Wire telemetry manager through the SDK to gather request/upload metrics #272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ypopovych merged 2 commits into from
Jun 3, 2026
Merged

[SDTEST-3776] Wire telemetry manager through the SDK to gather request/upload metrics #272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4a9a545
[SDTEST-3776] Wire telemetry manager through the SDK to gather reques…
ypopovych Jun 3, 2026
693d860
Add telemetry architecture doc for SDTEST-3777 handoff
ypopovych Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions DatadogSDKTesting.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
objects = {

/* Begin PBXBuildFile section */
1F26679E6F497C3AEF3A4845 /* MetricObservers.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5687C35EB7CCAF30113528FC /* MetricObservers.swift */; };
1FB416017E14C140B15E1D97 /* Async.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6F3EB9ACD93007FC43F039EE /* Async.swift */; };
300F42B595C3462AB9656B0C /* EarlyFlakeDetectionSwiftTestingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC931D86ADB841B6969C7A7A /* EarlyFlakeDetectionSwiftTestingTests.swift */; };
3592750B292103668D7668F7 /* MultipartFormURLRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EE92A3947C2E5BD2183BB4E /* MultipartFormURLRequest.swift */; };
Expand All @@ -16,6 +17,7 @@
4CCD47B30A3B513F26C8936E /* TelemetryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C8BF04956AA1E2FADBB80244 /* TelemetryTests.swift */; };
56B58A2FC91F21D1ABB0155E /* TelemetryApi.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01EB970146040DD4967C7411 /* TelemetryApi.swift */; };
7972B04FB1DFA64DC20032AE /* Telemetry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2E11F924F40C94461C2B6B27 /* Telemetry.swift */; };
7C6CBEE99C831DBEBB28BD67 /* TelemetryObservers.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26CA2B1C720A94291E83285B /* TelemetryObservers.swift */; };
821FA0CC8B62245A4949FDBB /* APITypes.swift in Sources */ = {isa = PBXBuildFile; fileRef = E85241CF97F1B6DAA49ECA23 /* APITypes.swift */; };
86137D23E1704D17A7B6D486 /* TelemetryExporterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 04DFE52D135C457D9160920A /* TelemetryExporterTests.swift */; };
8EF6F88255506A4CEE6D36B6 /* GitUploadApi.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5F3AA8F51FDDEF0C319080E8 /* GitUploadApi.swift */; };
Expand Down Expand Up @@ -430,8 +432,10 @@
12536CCE14EC1D3DAAC8DB81 /* Synced.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = Synced.swift; sourceTree = "<group>"; };
1495758A45B516A2C584B845 /* TestImpactAnalysisApi.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = TestImpactAnalysisApi.swift; sourceTree = "<group>"; };
1531A96019132E3E9E96A3A7 /* KnownTestsApi.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = KnownTestsApi.swift; sourceTree = "<group>"; };
26CA2B1C720A94291E83285B /* TelemetryObservers.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = TelemetryObservers.swift; sourceTree = "<group>"; };
2E11F924F40C94461C2B6B27 /* Telemetry.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = Telemetry.swift; sourceTree = "<group>"; };
31F8747157604DB48FE692EB /* AutoTestRetriesSwiftTestingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AutoTestRetriesSwiftTestingTests.swift; sourceTree = "<group>"; };
5687C35EB7CCAF30113528FC /* MetricObservers.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MetricObservers.swift; sourceTree = "<group>"; };
5F3AA8F51FDDEF0C319080E8 /* GitUploadApi.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = GitUploadApi.swift; sourceTree = "<group>"; };
6F3EB9ACD93007FC43F039EE /* Async.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = Async.swift; sourceTree = "<group>"; };
81894A4E29B0A28B00108C51 /* Retrying.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Retrying.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -803,6 +807,7 @@
2E11F924F40C94461C2B6B27 /* Telemetry.swift */,
843CE17ADB4681EE3DC4FF9B /* TelemetryMetrics.swift */,
E714A2777988AA4DA1AACB5C /* TelemetryTags.swift */,
26CA2B1C720A94291E83285B /* TelemetryObservers.swift */,
);
path = Telemetry;
sourceTree = "<group>";
Expand Down Expand Up @@ -1008,6 +1013,7 @@
BC161A86B91745F4BDC88FF1 /* TelemetryExporter.swift */,
A7926AEE2FCDCBFE007834F2 /* TelemetryLogExporter.swift */,
A7926AF02FCDD411007834F2 /* TelemetryMetricExporter.swift */,
5687C35EB7CCAF30113528FC /* MetricObservers.swift */,
);
path = Telemetry;
sourceTree = "<group>";
Expand Down Expand Up @@ -2085,6 +2091,7 @@
7972B04FB1DFA64DC20032AE /* Telemetry.swift in Sources */,
A0EC035461740BCF6DFB8171 /* TelemetryMetrics.swift in Sources */,
E42959C85F3B9BAB313984F7 /* TelemetryTags.swift in Sources */,
7C6CBEE99C831DBEBB28BD67 /* TelemetryObservers.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -2198,6 +2205,7 @@
CD1E9CDE6FE677BC07E66905 /* SpansApi.swift in Sources */,
56B58A2FC91F21D1ABB0155E /* TelemetryApi.swift in Sources */,
98D8D496CC4210A0D1D07F93 /* Synced.swift in Sources */,
1F26679E6F497C3AEF3A4845 /* MetricObservers.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
15 changes: 10 additions & 5 deletions Sources/DatadogSDKTesting/DDTestMonitor.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,8 @@ internal class DDTestMonitor {
DDTestMonitor.instance?.gitUploader = GitUploader(
log: Log.instance, api: DDTestMonitor.tracer.api.git, gitDirectory: gitDirectory,
commitFolder: try? DDTestMonitor.cacheManager?.commit(feature: "git"),
unshallowEnabled: DDTestMonitor.config.gitUnshallowEnabled
unshallowEnabled: DDTestMonitor.config.gitUnshallowEnabled,
telemetry: DDTestMonitor.tracer.telemetry
)
} else {
Log.debug("Git Upload Disabled")
Expand Down Expand Up @@ -291,7 +292,8 @@ internal class DDTestMonitor {
sha: commit,
testLevel: .test,
configurations: baseConfigurations,
customConfigurations: customConfigurations
customConfigurations: customConfigurations,
observer: DDTestMonitor.tracer.telemetry?.gitSettingsRequestObserver
)
}
}
Expand Down Expand Up @@ -396,7 +398,8 @@ internal class DDTestMonitor {
environment: DDTestMonitor.env.environment,
configurations: DDTestMonitor.env.baseConfigurations,
custom: DDTestMonitor.config.customConfigurations,
api: DDTestMonitor.tracer.api.knownTests, cache: cache)
api: DDTestMonitor.tracer.api.knownTests, cache: cache,
telemetry: DDTestMonitor.tracer.telemetry)
self.knownTests = runFactory(factory, errorKind: .knownTests)
}
knownTestsSetup.addDependency(updateTracerConfig)
Expand Down Expand Up @@ -454,7 +457,8 @@ internal class DDTestMonitor {
module: module,
attemptToFixRetries: attemptToFixRetryCount,
api: DDTestMonitor.tracer.api.testManagement,
cache: cache)
cache: cache,
telemetry: DDTestMonitor.tracer.telemetry)
self.testManagement = runFactory(factory, errorKind: .testManagementTests)
}
testManagementSetup.addDependency(updateTracerConfig)
Expand Down Expand Up @@ -485,7 +489,8 @@ internal class DDTestMonitor {
repository: repository,
cache: cache,
skippingEnabled: remote.itr.testsSkipping,
swiftTestingEnabled: DDTestMonitor.env.tiaSwiftTestingEnabled)
swiftTestingEnabled: DDTestMonitor.env.tiaSwiftTestingEnabled,
telemetry: DDTestMonitor.tracer.telemetry)
self.tia = runFactory(factory, errorKind: .skippableTests)
}
tiaSetup.addDependency(updateTracerConfig)
Expand Down
105 changes: 92 additions & 13 deletions Sources/DatadogSDKTesting/DDTracer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ internal class DDTracer {
/// feature factories don't need to reach through `eventsExporter` to talk
/// to the backend.
var api: TestOptimizationApi
/// Common telemetry manager. Created alongside the tracer so the API /
/// exporter layers and every feature can record SDK self-metrics. `nil`
/// when instrumentation telemetry is disabled or storage is unavailable.
let telemetry: Telemetry?

/// Logger used to emit `print()`/stderr captures and test-error context as
/// first-class OTel `LogRecord`s through the registered LoggerProvider.
Expand All @@ -53,9 +57,11 @@ internal class DDTracer {
init(id: String, version: String, exporter: ExporterProtocol?,
api: TestOptimizationApi, enabled: Bool, launchContext: SpanContext?,
resource: Resource = Resource(),
logRecordExporter: LogRecordExporter? = nil)
logRecordExporter: LogRecordExporter? = nil,
telemetry: Telemetry? = nil)
{
self.launchSpanContext = launchContext
self.telemetry = telemetry
self.eventsExporter = exporter
self.api = api
self.resource = resource
Expand Down Expand Up @@ -157,7 +163,6 @@ internal class DDTracer {
let exporterConfiguration = ExporterConfiguration(
environment: env.environment,
metadata: metadata,
performancePreset: .instantDataDelivery,
logger: Log.instance
)
let api = TestOptimizationApiService(
Expand All @@ -174,31 +179,105 @@ internal class DDTracer {
dateProvider: DDTestMonitor.clock,
debugNetworkRequests: conf.extraDebugNetwork
)
var resource = Resource()
resource.applicationName = identifier
resource.applicationVersion = version
resource.environment = env.environment
resource.service = env.service
resource.sdkLanguage = "swift"
resource.sdkName = identifier
resource.sdkVersion = DDTestMonitor.tracerVersion

// Build the telemetry manager before the exporter so its observers can
// be wired into the exporter's upload/serialization pipeline.
let telemetry: Telemetry? = conf.instrumentationTelemetryEnabled
? DDTracer.makeTelemetry(api: api, configuration: exporterConfiguration, resource: resource,
exportInterval: conf.telemetryHeartbeatInterval)
: nil

// Exporter files live under the cache manager's session directory so
// they stay scoped to this test run and get cleaned up alongside the
// rest of the per-session state.
let eventsExporter: Exporter?
if let storage = try? DDTestMonitor.cacheManager?.session(feature: "exporter") {
eventsExporter = try? Exporter(config: exporterConfiguration, api: api, storage: storage)
eventsExporter = try? Exporter(config: exporterConfiguration, api: api, storage: storage,
observers: DDTracer.exporterObservers(telemetry: telemetry))
} else {
Log.print("Exporter init skipped: cache manager unavailable")
eventsExporter = nil
}

var resource = Resource()
resource.applicationName = identifier
resource.applicationVersion = version
resource.environment = env.environment
resource.service = env.service
resource.sdkLanguage = "swift"
resource.sdkName = identifier
resource.sdkVersion = DDTestMonitor.tracerVersion

self.init(id: identifier, version: version, exporter: eventsExporter,
api: api,
enabled: !conf.disableTracesExporting, launchContext: launchSpanContext,
resource: resource,
logRecordExporter: logRecordExporter)
logRecordExporter: logRecordExporter,
telemetry: telemetry)
}

/// Build the common telemetry manager wired to the SDK's telemetry intake,
/// reusing the exporter's configuration. Returns `nil` when the backing
/// storage or exporter can't be created, in which case telemetry is simply
/// not gathered.
private static func makeTelemetry(api: TestOptimizationApi, configuration: ExporterConfiguration,
resource: Resource, exportInterval: TimeInterval) -> Telemetry?
{
guard let cacheManager = DDTestMonitor.cacheManager,
let storage = try? cacheManager.session(feature: "telemetry")
else {
Log.print("Telemetry init skipped: cache manager unavailable")
return nil
}

guard let telemetryExporter = try? TelemetryExporter(config: configuration,
storage: storage,
api: api.telemetry)
else {
Log.print("Telemetry init skipped: telemetry exporter unavailable")
return nil
}

let metricExporter = TelemetryMetricExporter(telemetryExporter: telemetryExporter,
namespace: .civisibility,
distributionNamespace: .civisibility)
return Telemetry(exporter: metricExporter, resource: resource, exportInterval: exportInterval)
}

/// Build the exporter's telemetry observers, mapping each upload feature to
/// its `endpoint_payload.*` metrics tagged by endpoint (spans → `test_cycle`,
/// coverage → `code_coverage`). Empty when telemetry is disabled.
private static func exporterObservers(telemetry: Telemetry?) -> ExporterObservers {
guard let telemetry else { return .init() }
return ExporterObservers(
spans: endpointPayloadObservers(telemetry: telemetry, endpoint: .testCycle),
coverage: endpointPayloadObservers(telemetry: telemetry, endpoint: .codeCoverage)
)
}

private static func endpointPayloadObservers(telemetry: Telemetry,
endpoint: Telemetry.Endpoint) -> ExporterObservers.Feature
{
ExporterObservers.Feature(
// Transport facts (size sent, duration, status) come from the upload
// request itself.
request: Telemetry.RequestMetricsObserver(
onRequest: { telemetry.metrics.endpointPayload.requests.add(endpoint: endpoint) },
onDurationMs: { telemetry.metrics.endpointPayload.requestsMs.record($0, endpoint: endpoint) },
onRequestBytes: { telemetry.metrics.endpointPayload.bytes.record(Double($0), endpoint: endpoint) },
onError: { telemetry.metrics.endpointPayload.requestsErrors.add(errorType: $0, endpoint: endpoint) }
),
// The worker owns the batch lifecycle; only `dropped` is unique to it.
upload: Telemetry.UploadMetricsObserver(
onDropped: { _ in telemetry.metrics.endpointPayload.dropped.add(endpoint: endpoint) }
),
// Serialization happens at the storage layer.
payload: Telemetry.PayloadMetricsObserver(
onFinalized: { count, serializationMs in
telemetry.metrics.endpointPayload.eventsCount.record(Double(count), endpoint: endpoint)
telemetry.metrics.endpointPayload.eventsSerializationMs.record(serializationMs, endpoint: endpoint)
}
)
)
}

private func createSpanBuilder(name: String, attributes: [String: AttributeValue], startTime: Date? = nil) -> SpanBuilder {
Expand Down
7 changes: 5 additions & 2 deletions Sources/DatadogSDKTesting/KnownTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,12 @@ struct KnownTestsFactory: FeatureFactory {
let customConfigurations: [String: String]
let cacheFolder: Directory
let api: KnownTestsApi
let telemetry: Telemetry?
let cacheFileName = "known_tests.json"

init(repository: String, service: String, environment: String,
configurations: [String: String], custom: [String: String],
api: KnownTestsApi, cache: Directory)
api: KnownTestsApi, cache: Directory, telemetry: Telemetry? = nil)
{
self.configurations = configurations
self.customConfigurations = custom
Expand All @@ -89,6 +90,7 @@ struct KnownTestsFactory: FeatureFactory {
self.service = service
self.environment = environment
self.api = api
self.telemetry = telemetry
}

static func isEnabled(config: Config, env: Environment, remote: TracerSettings) -> Bool {
Expand Down Expand Up @@ -128,7 +130,8 @@ struct KnownTestsFactory: FeatureFactory {
tests = try await api.tests(service: service, env: environment,
repositoryURL: repository,
configurations: configurations,
customConfigurations: customConfigurations).tests
customConfigurations: customConfigurations,
observer: telemetry?.knownTestsRequestObserver).tests
} catch {
throw LibraryConfigurationCommunicationError(
requestName: "Known Tests Request",
Expand Down
Loading
Loading