[SDTEST-3777] Gather remaining CI Visibility instrumentation telemetry metrics

DatadogSDKTesting.xcodeproj/project.pbxproj

@@ -18,6 +18,7 @@
 		56B58A2FC91F21D1ABB0155E /* TelemetryApi.swift in Sources */ = {isa = PBXBuildFile; fileRef = 01EB970146040DD4967C7411 /* TelemetryApi.swift */; };
 		7972B04FB1DFA64DC20032AE /* Telemetry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2E11F924F40C94461C2B6B27 /* Telemetry.swift */; };
 		7C6CBEE99C831DBEBB28BD67 /* TelemetryObservers.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26CA2B1C720A94291E83285B /* TelemetryObservers.swift */; };
+		02DF858038394B24B4534C49 /* TelemetryEventsFeature.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6BC8A25D6F0A4B818594FE62 /* TelemetryEventsFeature.swift */; };
 		821FA0CC8B62245A4949FDBB /* APITypes.swift in Sources */ = {isa = PBXBuildFile; fileRef = E85241CF97F1B6DAA49ECA23 /* APITypes.swift */; };
 		86137D23E1704D17A7B6D486 /* TelemetryExporterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 04DFE52D135C457D9160920A /* TelemetryExporterTests.swift */; };
 		8EF6F88255506A4CEE6D36B6 /* GitUploadApi.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5F3AA8F51FDDEF0C319080E8 /* GitUploadApi.swift */; };
@@ -433,6 +434,7 @@
 		1495758A45B516A2C584B845 /* TestImpactAnalysisApi.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = TestImpactAnalysisApi.swift; sourceTree = "<group>"; };
 		1531A96019132E3E9E96A3A7 /* KnownTestsApi.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = KnownTestsApi.swift; sourceTree = "<group>"; };
 		26CA2B1C720A94291E83285B /* TelemetryObservers.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = TelemetryObservers.swift; sourceTree = "<group>"; };
+		6BC8A25D6F0A4B818594FE62 /* TelemetryEventsFeature.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = TelemetryEventsFeature.swift; sourceTree = "<group>"; };
 		2E11F924F40C94461C2B6B27 /* Telemetry.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = Telemetry.swift; sourceTree = "<group>"; };
 		31F8747157604DB48FE692EB /* AutoTestRetriesSwiftTestingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AutoTestRetriesSwiftTestingTests.swift; sourceTree = "<group>"; };
 		5687C35EB7CCAF30113528FC /* MetricObservers.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = MetricObservers.swift; sourceTree = "<group>"; };
@@ -808,6 +810,7 @@
 				843CE17ADB4681EE3DC4FF9B /* TelemetryMetrics.swift */,
 				E714A2777988AA4DA1AACB5C /* TelemetryTags.swift */,
 				26CA2B1C720A94291E83285B /* TelemetryObservers.swift */,
+				6BC8A25D6F0A4B818594FE62 /* TelemetryEventsFeature.swift */,
 			);
 			path = Telemetry;
 			sourceTree = "<group>";
@@ -2092,6 +2095,7 @@
 				A0EC035461740BCF6DFB8171 /* TelemetryMetrics.swift in Sources */,
 				E42959C85F3B9BAB313984F7 /* TelemetryTags.swift in Sources */,
 				7C6CBEE99C831DBEBB28BD67 /* TelemetryObservers.swift in Sources */,
+				02DF858038394B24B4534C49 /* TelemetryEventsFeature.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

Sources/DatadogSDKTesting/Coverage/BackgroundCoverageProcessor.swift

@@ -28,15 +28,18 @@ final class BackgroundCoverageProcessor: CoverageProcessor {
     private let parser: CoverageParser
     private let workQueue: OperationQueue
     private let cleanupCoverageFiles: Bool
+    private let telemetry: Telemetry?
 
     init(exporter: any CoverageExporterType,
          parser: CoverageParser,
          priority: CodeCoveragePriority,
-         cleanupCoverageFiles: Bool = true)
+         cleanupCoverageFiles: Bool = true,
+         telemetry: Telemetry? = nil)
     {
         self.exporter = exporter
         self.parser = parser
         self.cleanupCoverageFiles = cleanupCoverageFiles
+        self.telemetry = telemetry
         let queue = OperationQueue()
         queue.qualityOfService = priority.qos
         queue.maxConcurrentOperationCount = max(ProcessInfo.processInfo.activeProcessorCount - 1, 1)
@@ -47,6 +50,7 @@ final class BackgroundCoverageProcessor: CoverageProcessor {
         let parser = self.parser
         let exporter = self.exporter
         let cleanup = self.cleanupCoverageFiles
+        let telemetry = self.telemetry
         workQueue.addOperation {
             guard FileManager.default.fileExists(atPath: record.coverageFileURL.path) else {
                 Log.debug("Coverage file is missing at: \(record.coverageFileURL.path)")
@@ -64,9 +68,18 @@ final class BackgroundCoverageProcessor: CoverageProcessor {
                 info = try parser.filesCovered(in: record.coverageFileURL)
             } catch {
                 Log.print("Coverage parsing failed for \(record.name): \(error)")
+                telemetry?.metrics.codeCoverage.errors.add()
                 return
             }
             let files = info.files.values.map { CoverageFile(file: $0) }
+
+            telemetry?.metrics.codeCoverage.finished.add()
+            if files.isEmpty {
+                telemetry?.metrics.codeCoverage.isEmpty.add()
+            } else {
+                telemetry?.metrics.codeCoverage.files.record(Double(files.count))
+            }
+
             let data = CoverageData(name: record.name,
                                     files: files,
                                     workspacePath: record.workspacePath,

Sources/DatadogSDKTesting/Coverage/CodeCoverage.swift

@@ -17,6 +17,7 @@ final class CodeCoverage: TestHooksFeature {
     static var id: FeatureId = "Code Coverage"
 
     let swiftTestingEnabled: Bool
+    let telemetry: Telemetry?
 
     var isEnabled: Bool { _state.use { $0.collector != nil } }
 
@@ -32,8 +33,9 @@ final class CodeCoverage: TestHooksFeature {
     }
     private let _state: Synced<State>
 
-    init(collector: TestCoverageCollector, swiftTestingEnabled: Bool) {
+    init(collector: TestCoverageCollector, swiftTestingEnabled: Bool, telemetry: Telemetry? = nil) {
         self.swiftTestingEnabled = swiftTestingEnabled
+        self.telemetry = telemetry
         self._state = .init(.init(collector: collector, active: nil))
     }
 
@@ -64,8 +66,8 @@ final class CodeCoverage: TestHooksFeature {
         let context = CoverageContext.test(testSpanId: test.id,
                                            suiteId: test.suite.id,
                                            sessionId: test.session.id)
-        _state.update { state in
-            guard let collector = state.collector else { return }
+        let started: Bool = _state.update { state in
+            guard let collector = state.collector else { return false }
             if let prior = state.active {
                 Log.print("""
                     Code coverage error: a coverage gathering session is already \
@@ -78,9 +80,13 @@ final class CodeCoverage: TestHooksFeature {
                 prior.end()
                 state.active = nil
                 state.collector = nil
-                return
+                return false
             }
             state.active = collector.startCoverage(context: context)
+            return state.active != nil
+        }
+        if started {
+            telemetry?.metrics.codeCoverage.started.add()
         }
     }
 
@@ -110,6 +116,20 @@ struct CodeCoverageFactory: FeatureFactory {
     let debug: Bool
     let exporter: ExporterProtocol
     let swiftTestingEnabled: Bool
+    let telemetry: Telemetry?
+
+    init(workspacePath: String?, priority: CodeCoveragePriority, tempFolder: Directory,
+         debug: Bool, exporter: ExporterProtocol, swiftTestingEnabled: Bool,
+         telemetry: Telemetry? = nil)
+    {
+        self.workspacePath = workspacePath
+        self.priority = priority
+        self.tempFolder = tempFolder
+        self.debug = debug
+        self.exporter = exporter
+        self.swiftTestingEnabled = swiftTestingEnabled
+        self.telemetry = telemetry
+    }
 
     static func isEnabled(config: Config, env: Environment, remote: TracerSettings) -> Bool {
         config.codeCoverageEnabled && remote.itr.codeCoverage
@@ -120,8 +140,10 @@ struct CodeCoverageFactory: FeatureFactory {
                                                 exporter: exporter,
                                                 workspacePath: workspacePath,
                                                 priority: priority,
-                                                debug: debug)
+                                                debug: debug,
+                                                telemetry: telemetry)
         log.debug("Code Coverage Enabled")
-        return CodeCoverage(collector: provider, swiftTestingEnabled: swiftTestingEnabled)
+        return CodeCoverage(collector: provider, swiftTestingEnabled: swiftTestingEnabled,
+                            telemetry: telemetry)
     }
 }

Sources/DatadogSDKTesting/Coverage/CodeCoverageProvider.swift

@@ -51,15 +51,17 @@ final class CodeCoverageProvider: TestCoverageCollector {
     let debug: Bool
 
     init(storagePath: Directory, exporter: ExporterProtocol,
-         workspacePath: String?, priority: CodeCoveragePriority, debug: Bool) throws
+         workspacePath: String?, priority: CodeCoveragePriority, debug: Bool,
+         telemetry: Telemetry? = nil) throws
     {
         let llvm = try LLVMCoverageProcessor(for: PlatformUtils.xcodeVersion,
                                              temp: storagePath.url)
         self.llvmProcessor = llvm
         self.processor = BackgroundCoverageProcessor(exporter: exporter,
                                                      parser: llvm.parser,
                                                      priority: priority,
-                                                     cleanupCoverageFiles: !debug)
+                                                     cleanupCoverageFiles: !debug,
+                                                     telemetry: telemetry)
         self.debug = debug
         self.storagePath = storagePath
         self.workspacePath = workspacePath

Sources/DatadogSDKTesting/DDModule.swift

@@ -27,7 +27,6 @@ public final class DDModule: NSObject {
     var id: SpanId { span.context.spanId }
     let span: SpanSdk
     var session: TestSession { _session }
-    var configuration: SessionConfig { _session.configuration }
     var startTime: Date { span.startTime }
 
     private let _session: DDSession
@@ -62,7 +61,7 @@ public final class DDModule: NSObject {
         attributes.testSessionId = session.id
         attributes.testModuleId = id
 
-        for (key, value) in session.configuration.metrics {
+        for (key, value) in session.configuration.env.baseMetrics {
             attributes[key] = .double(value)
         }
 
@@ -137,7 +136,8 @@ public extension DDModule {
     ///   - name: name of the suite
     ///   - startTime: Optional, the time where the suite started
     @objc func suiteStart(name: String, startTime: Date? = nil) -> DDSuite {
-        startSuite(named: name, at: startTime, framework: .init(name: "SwiftManual", version: "0.0.0")) as! DDSuite
+        configuration.telemetry?.metrics.events.manualApiEvents.add(eventType: .suite)
+        return startSuite(named: name, at: startTime, framework: .init(name: "SwiftManual", version: "0.0.0")) as! DDSuite
     }
 
     @objc func suiteStart(name: String) -> DDSuite {

Sources/DatadogSDKTesting/DDSession.swift

@@ -52,8 +52,8 @@ public final class DDSession: NSObject {
         }
 
         var attributes: [String: AttributeValue] = [
-            DDTestSessionTags.testToolchain: .string(config.platform.runtimeName.lowercased()
-                                                     + "-" + config.platform.runtimeVersion),
+            DDTestSessionTags.testToolchain: .string(config.env.platform.runtimeName.lowercased()
+                                                     + "-" + config.env.platform.runtimeVersion),
         ]
 
         attributes.type = DDTagValues.typeSessionEnd
@@ -63,7 +63,7 @@ public final class DDSession: NSObject {
         if let command = config.command {
             attributes[DDTestTags.testCommand] = .string(command)
         }
-        for (key, value) in config.metrics {
+        for (key, value) in config.env.baseMetrics {
             attributes[key] = .double(value)
         }
 
@@ -160,24 +160,23 @@ public extension DDSession {
             let _ = DDTestMonitor.installTestMonitor()
         }
         let config = SessionConfig(activeFeatures: DDTestMonitor.instance?.activeFeatures ?? [],
-                                   platform: DDTestMonitor.env.platform,
+                                   env: DDTestMonitor.env,
+                                   config: DDTestMonitor.config,
                                    clock: DDTestMonitor.clock,
                                    crash: DDTestMonitor.instance?.crashInfo,
                                    command: command,
-                                   service: DDTestMonitor.env.service,
-                                   metrics: DDTestMonitor.env.baseMetrics,
-                                   log: Log.instance)
-        waitForAsync {
-            do {
-                try await DDTestMonitor.clock.sync()
-            } catch {
-                DDTestMonitor.clock = DateClock()
-            }
+                                   log: Log.instance,
+                                   telemetry: DDTestMonitor.tracer.telemetry)
+        waitForAsync { await DDTestMonitor.clock.sync() }
+        let session = DDSession(name: name, config: config,
+                                modules: DDModule.StatelessManager(observer: SessionAndModuleObserver()),
+                                startTime: startTime)
+        if let telemetry = DDTestMonitor.tracer.telemetry {
+            telemetry.metrics.session.started.add(provider: config.env.ci?.provider, autoInjected: false)
+            telemetry.metrics.events.manualApiEvents.add(eventType: .session)
+            session.emitGitShaCheck(to: telemetry)
         }
-        return DDSession(name: name, config: config,
-                         modules: DDModule.StatelessManager(config: config,
-                                                            observer: SessionAndModuleObserver()),
-                         startTime: startTime)
+        return session
     }
 
     @objc static func start(name: String) -> DDSession {
@@ -209,6 +208,7 @@ public extension DDSession {
     ///   - name: name of the module
     ///   - startTime: Optional, the time where the module started
     @objc func moduleStart(name: String, startTime: Date? = nil) -> DDModule {
+        configuration.telemetry?.metrics.events.manualApiEvents.add(eventType: .module)
         return _moduleManager.module(named: name, at: startTime, provider: self) as! DDModule
     }
 

Sources/DatadogSDKTesting/DDSuite.swift

@@ -23,7 +23,6 @@ public final class DDSuite: NSObject {
         span.endTime?.timeIntervalSince(span.startTime).toNanoseconds ?? 0
     }
     var status: TestStatus { span.testStatus }
-    var configuration: SessionConfig { _module.configuration }
 
     var id: SpanId { span.context.spanId }
     let span: SpanSdk
@@ -68,7 +67,7 @@ public final class DDSuite: NSObject {
         attributes.testModuleId = module.id
         attributes.testSuiteId = id
 
-        for (key, value) in module.configuration.metrics {
+        for (key, value) in module.configuration.env.baseMetrics {
             attributes[key] = .double(value)
         }
 
@@ -134,7 +133,8 @@ public final class DDSuite: NSObject {
     ///   - action: callback with test. Test will be ended automatically after call end
     @discardableResult
     @objc public func testStart(name: String, _ action: (DDTest) -> Any) -> Any {
-        testStart(named: name, action)
+        configuration.telemetry?.metrics.events.manualApiEvents.add(eventType: .test)
+        return testStart(named: name, action)
     }
 
     /// Starts a test in this suite
@@ -143,7 +143,8 @@ public final class DDSuite: NSObject {
     ///   - startTime: start time for the test
     ///   - action: callback with test. Test will be ended automatically after call end
     @objc public func testStart(name: String, startTime: Date, _ action: (DDTest) -> Any) -> Any {
-        testStart(named: name, at: startTime, action)
+        configuration.telemetry?.metrics.events.manualApiEvents.add(eventType: .test)
+        return testStart(named: name, at: startTime, action)
     }
 
     public func testStart<T>(named name: String, at start: Date? = nil,

Sources/DatadogSDKTesting/DDTest.swift

@@ -183,7 +183,7 @@ extension DDTest {
         attributes.resource = "\(suite.name).\(name)"
 
         // TODO: Move to common medatada when we will have common metrics
-        for metric in suite.configuration.metrics {
+        for metric in suite.configuration.env.baseMetrics {
             attributes[metric.key] = .double(metric.value)
         }
 

Sources/DatadogSDKTesting/DDTestMonitor.swift

@@ -26,11 +26,11 @@ internal import OpenTelemetrySdk
 
 internal class DDTestMonitor {
     static var instance: DDTestMonitor?
-    static var clock: Clock = {
+    static var clock: any Clock = {
         #if os(watchOS)
             return DateClock()
         #else
-            return DDTestMonitor.config.disableNTPClock ? DateClock() as Clock : NTPClock()
+            return config.disableNTPClock ? DateClock() as Clock : FallbackClock(NTPClock()) { DateClock() }
         #endif
     }()
 
@@ -282,7 +282,7 @@ internal class DDTestMonitor {
             let baseConfigurations = DDTestMonitor.env.baseConfigurations
             let customConfigurations = DDTestMonitor.config.customConfigurations
             do {
-                return try Log.measure(name: log) { () throws(APICallError) -> TracerSettings in
+                let config = try Log.measure(name: log) { () throws(APICallError) -> TracerSettings in
                     try waitForAsync { () async throws(APICallError) -> TracerSettings in
                         try await settingsApi.tracerSettings(
                             service: service,
@@ -297,6 +297,8 @@ internal class DDTestMonitor {
                         )
                     }
                 }
+                DDTestMonitor.tracer.telemetry?.metrics.gitRequests.settingsResponse.add(config: config)
+                return config
             } catch {
                 let err = LibraryConfigurationCommunicationError(
                     requestName: "SettingsRequest",
@@ -524,7 +526,8 @@ internal class DDTestMonitor {
                                               tempFolder: temp,
                                               debug: DDTestMonitor.config.extraDebugCodeCoverage,
                                               exporter: eventsExporter,
-                                              swiftTestingEnabled: DDTestMonitor.env.tiaSwiftTestingEnabled)
+                                              swiftTestingEnabled: DDTestMonitor.env.tiaSwiftTestingEnabled,
+                                              telemetry: DDTestMonitor.tracer.telemetry)
             self.coverage = runFactory(factory)
         }
         coverageSetup.addDependency(updateTracerConfig)
@@ -702,13 +705,15 @@ internal class DDTestMonitor {
 
     var activeFeatures: any TestHooksFeatures {
         testOptimizationSetupQueue.waitUntilAllOperationsAreFinished()
+        let telemetryEvents = DDTestMonitor.tracer.telemetry.map { TelemetryEventsFeature(telemetry: $0) }
         let features: [(any TestHooksFeature)?] = [
             testManagement, tia, coverage, efd, atr, knownTests,
             AdditionalTags(codeCoverage: coverage == nil,
                            bundleFunctions: bundleFunctionInfo,
                            codeOwners: codeOwners,
                            workspacePath: DDTestMonitor.env.workspacePath),
-            LibraryConfigurationErrorTags(errors: libraryConfigurationErrors)
+            LibraryConfigurationErrorTags(errors: libraryConfigurationErrors),
+            telemetryEvents
         ]
         return features.compactMap { $0 }
     }