Export agent connection env vars to library init containers for telemetry

Swap mutator ordering so the APM mutator runs before the config webhook.
This ensures library init containers already exist when the config webhook
injects agent connection env vars (DD_AGENT_HOST, DD_TRACE_AGENT_URL,
socket volumes). Removes the old timestamp file mechanism (c-init-time.*)
and manual DD_AGENT_HOST injection in favor of the existing config mutator.
Restores DD_INJECT_START_TIME. Replaces OnLibraryInjection callback with
direct logging in InjectAPMLibraries.

Author: natitsechanski

pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation.go

@@ -40,11 +40,15 @@ func NewAutoInstrumentation(datadogConfig config.Component, wmeta workloadmeta.C
 
 	// For auto instrumentation, we need all the mutators to be applied for SSI to function. Specifically, we need
 	// things like the Datadog socket to be mounted from the config webhook and the DD_ENV, DD_SERVICE, and DD_VERSION
-	// env vars to be set from labels if they are available..
+	// env vars to be set from labels if they are available.
+	//
+	// The APM mutator runs before the config webhook so that init containers created by library injection
+	// already exist when the config webhook injects agent connection env vars (DD_AGENT_HOST, DD_TRACE_AGENT_URL,
+	// socket volumes, etc.). This lets copy-lib.sh send telemetry to the trace agent without manual env var injection.
 	mutator := mutatecommon.NewMutators(
 		tagsfromlabels.NewMutator(tagsfromlabels.NewMutatorConfig(datadogConfig), apm),
-		configWebhook.NewMutator(configWebhook.NewMutatorConfig(datadogConfig), apm),
 		apm,
+		configWebhook.NewMutator(configWebhook.NewMutatorConfig(datadogConfig), apm),
 	)
 	labelSelectors := NewLabelSelectors(NewLabelSelectorsConfig(datadogConfig))
 	return NewWebhook(config.Webhook, mutator, labelSelectors)

pkg/clusteragent/admission/mutate/autoinstrumentation/libraryinjection/init_container.go

@@ -69,23 +69,19 @@ func (p *InitContainerProvider) InjectInjector(pod *corev1.Pod, cfg InjectorConf
 		ReadOnly:  true,
 	})
 
-	// Timestamp file path for tracking init container completion
-	tsFilePath := injectorMount.MountPath + "/c-init-time." + InjectorInitContainerName
-
 	// Init container that copies injector files
 	initContainer := corev1.Container{
 		Name:    InjectorInitContainerName,
 		Image:   cfg.Package.FullRef(),
 		Command: []string{"/bin/sh", "-c", "--"},
 		Args: []string{
 			fmt.Sprintf(
-				`cp -r /%s/* %s && echo %s > %s/%s && echo $(date +%%s) >> %s`,
+				`cp -r /%s/* %s && echo %s > %s/%s`,
 				injectorMount.SubPath,
 				injectorMount.MountPath,
 				asAbsPath(injectorFilePath("launcher.preload.so")),
 				etcMountPath,
 				ldSoPreloadFileName,
-				tsFilePath,
 			),
 		},
 		VolumeMounts: []corev1.VolumeMount{
@@ -125,34 +121,19 @@ func (p *InitContainerProvider) InjectLibrary(pod *corev1.Pod, cfg LibraryConfig
 		SubPath:   libraryPackagesDir + "/" + cfg.Language,
 	}
 
-	// Volume mount for injector (to write timestamp)
-	injectorMount := corev1.VolumeMount{
-		Name:      InstrumentationVolumeName,
-		MountPath: asAbsPath(injectPackageDir),
-		SubPath:   injectPackageDir,
-	}
-
 	// Init container name
 	containerName := fmt.Sprintf(LibraryInitContainerNameTemplate, cfg.Language)
 
-	// Timestamp file path
-	tsFilePath := injectorMount.MountPath + "/c-init-time." + containerName
-
 	// Init container that copies library files
 	initContainer := corev1.Container{
 		Name:    containerName,
 		Image:   cfg.Package.FullRef(),
 		Command: []string{"/bin/sh", "-c", "--"},
 		Args: []string{
-			fmt.Sprintf(
-				`sh copy-lib.sh %s && echo $(date +%%s) >> %s`,
-				initContainerMount.MountPath,
-				tsFilePath,
-			),
+			"sh copy-lib.sh " + initContainerMount.MountPath,
 		},
 		VolumeMounts: []corev1.VolumeMount{
 			initContainerMount,
-			injectorMount,
 		},
 		Resources: cfg.Context.ResourceRequirements,
 	}

pkg/clusteragent/admission/mutate/autoinstrumentation/libraryinjection/init_container_test.go

@@ -49,6 +49,9 @@ func TestInjectInjector(t *testing.T) {
 
 	// Check volume mounts were added to app container
 	require.Len(t, pod.Spec.Containers[0].VolumeMounts, 2)
+
+	// Check injector command does not contain timestamp writing
+	assert.NotContains(t, pod.Spec.InitContainers[0].Args[0], "c-init-time")
 }
 
 func TestInjectLibrary(t *testing.T) {
@@ -71,8 +74,20 @@ func TestInjectLibrary(t *testing.T) {
 
 	// Check init container was added
 	require.Len(t, pod.Spec.InitContainers, 1)
-	assert.Equal(t, "datadog-lib-java-init", pod.Spec.InitContainers[0].Name)
-	assert.Equal(t, "gcr.io/datadoghq/dd-lib-java-init:latest", pod.Spec.InitContainers[0].Image)
+	initContainer := pod.Spec.InitContainers[0]
+	assert.Equal(t, "datadog-lib-java-init", initContainer.Name)
+	assert.Equal(t, "gcr.io/datadoghq/dd-lib-java-init:latest", initContainer.Image)
+
+	// Check only the library volume mount is present (no injector mount)
+	require.Len(t, initContainer.VolumeMounts, 1)
+	assert.Equal(t, libraryinjection.InstrumentationVolumeName, initContainer.VolumeMounts[0].Name)
+
+	// Check no env vars are set directly (agent connection vars come from config mutator)
+	assert.Empty(t, initContainer.Env)
+
+	// Check command does not contain timestamp writing
+	assert.NotContains(t, initContainer.Args[0], "date")
+	assert.NotContains(t, initContainer.Args[0], "c-init-time")
 }
 
 func TestInjectInjector_SkipsWhenInsufficientResources(t *testing.T) {

pkg/clusteragent/admission/mutate/autoinstrumentation/libraryinjection/mutator.go

@@ -72,6 +72,9 @@ func InjectAPMLibraries(pod *corev1.Pod, cfg LibraryInjectionConfig) error {
 
 		metrics.LibInjectionAttempts.Inc(lib.Language, strconv.FormatBool(injected), strconv.FormatBool(cfg.AutoDetected), cfg.InjectionType)
 
+		log.Infof("APM library injection: pod=%s language=%s status=%s type=%s",
+			mutatecommon.PodString(pod), lib.Language, libResult.Status, cfg.InjectionType)
+
 		if libResult.Status == MutationStatusInjected && lib.Package.CanonicalVersion != "" {
 			annotation.Set(pod, annotation.LibraryCanonicalVersion.Format(lib.Language), lib.Package.CanonicalVersion)
 		}

pkg/ssi/testutils/pod_init_container.go

@@ -99,7 +99,7 @@ func (v *initContainerInjectionValidator) RequireInjection(t *testing.T) {
 		},
 	}
 	validator.RequireVolumeMounts(t, expectedVolumeMounts)
-	validator.RequireCommand(t, "/bin/sh -c -- cp -r /opt/datadog-packages/datadog-apm-inject/* /datadog-inject && echo /opt/datadog-packages/datadog-apm-inject/stable/inject/launcher.preload.so > /datadog-etc/ld.so.preload && echo $(date +%s) >> /datadog-inject/c-init-time.datadog-init-apm-inject")
+	validator.RequireCommand(t, "/bin/sh -c -- cp -r /opt/datadog-packages/datadog-apm-inject/* /datadog-inject && echo /opt/datadog-packages/datadog-apm-inject/stable/inject/launcher.preload.so > /datadog-etc/ld.so.preload")
 
 	// Validate library init containers.
 	for lang := range v.libraryVersions {
@@ -111,14 +111,9 @@ func (v *initContainerInjectionValidator) RequireInjection(t *testing.T) {
 				MountPath: "/datadog-lib",
 				SubPath:   "opt/datadog/apm/library/" + lang,
 			},
-			{
-				Name:      "datadog-auto-instrumentation",
-				MountPath: "/opt/datadog-packages/datadog-apm-inject",
-				SubPath:   "opt/datadog-packages/datadog-apm-inject",
-			},
 		}
 		validator.RequireVolumeMounts(t, expectedVolumeMounts)
-		validator.RequireCommand(t, "/bin/sh -c -- sh copy-lib.sh /datadog-lib && echo $(date +%s) >> /opt/datadog-packages/datadog-apm-inject/c-init-time.datadog-lib-"+lang+"-init")
+		validator.RequireCommand(t, "/bin/sh -c -- sh copy-lib.sh /datadog-lib")
 	}
 }