Non root container cni root nginx with "stdout/stderr piping"

keymanager/workload_service/server.go

@@ -211,6 +211,10 @@ func NewServer(keyProtectionService KeyProtectionService, workloadService Worklo
 	if err != nil {
 		return nil, fmt.Errorf("failed to listen on unix socket %s: %w", socketPath, err)
 	}
+	// Change the permission so that non-root containers can access the socket
+	if err := os.Chmod(socketPath, 0666); err != nil {
+		return nil, fmt.Errorf("failed to chmod unix socket %s: %w", socketPath, err)
+	}
 	s.listener = ln
 
 	go s.processClaims()

launcher/container_runner.go

@@ -20,6 +20,8 @@
 	attestationpb "github.com/GoogleCloudPlatform/confidential-space/server/proto/gen/attestation"
 	"github.com/cenkalti/backoff/v4"
 	"github.com/containerd/containerd"
+	gocni "github.com/containerd/go-cni"
+
 	"github.com/containerd/containerd/cio"
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/content"
@@ -57,6 +59,7 @@
 	logger        logging.Logger
 	gpuAttester   gpu.Attester
 	serialConsole *os.File
+	cni           gocni.CNI
 }
 
 const tokenFileTmp = ".token.tmp"
@@ -88,6 +91,15 @@
 // Default OOM score for a CS container.
 const defaultOOMScore = 1000
 
+// Constants for a non-root container.
+const (
+	hostUIDBegin = 10000 // Starting (outside container) uid for the root user inside the container
+	hostGIDBegin = 10000 // Starting (outside container) gid for the root group inside the container
+	userNSSize   = 65536 // 16-bit range of uid/gid inside the container
+
+	stdoutStderrPipePath = "/tmp/workload.log"
+)
+
 // NewRunner returns a runner.
 func NewRunner(ctx context.Context, cdClient *containerd.Client, token oauth2.Token, launchSpec spec.LaunchSpec, mdsClient *metadata.Client, tpm io.ReadWriteCloser, logger logging.Logger, serialConsole *os.File) (*ContainerRunner, error) {
 	image, err := initImage(ctx, cdClient, launchSpec, token)
@@ -133,9 +145,6 @@
 	}
 
 	logger.Info(fmt.Sprintf("Exposed Ports:             : %v\n", imageConfig.ExposedPorts))
-	if err := openPorts(imageConfig.ExposedPorts); err != nil {
-		return nil, err
-	}
 
 	logger.Info(fmt.Sprintf("Image Labels               : %v\n", imageConfig.Labels))
 	launchPolicy, err := spec.GetLaunchPolicy(imageConfig.Labels, logger)
@@ -187,11 +196,17 @@
 		// the host network (same effect as --net-host in ctr command)
 		oci.WithHostHostsFile,
 		oci.WithHostResolvconf,
-		oci.WithHostNamespace(specs.NetworkNamespace),
+
 		oci.WithEnv([]string{fmt.Sprintf("HOSTNAME=%s", hostname)}),
 		oci.WithAddedCapabilities(launchSpec.AddedCapabilities),
 		withRlimits(rlimits),
 		withOOMScoreAdj(defaultOOMScore),
+		oci.WithUserNamespace(
+			[]specs.LinuxIDMapping{{ContainerID: 0, HostID: hostUIDBegin, Size: userNSSize}},
+			[]specs.LinuxIDMapping{{ContainerID: 0, HostID: hostGIDBegin, Size: userNSSize}},
+		),
+		withSysBindMount(), // mount /sys as "bind" instead of "sysfs" for a non-root container
+		// withStdoutStderrPipeMounts(stdoutStderrPipePath),
 	}
 	if launchSpec.DevShmSize != 0 {
 		specOpts = append(specOpts, oci.WithDevShmSize(launchSpec.DevShmSize))
@@ -237,11 +252,18 @@
 		deviceROTs = append(deviceROTs, nvidiaAttester)
 	}
 
+	os.Remove(stdoutStderrPipePath)
+	f, err := os.OpenFile(stdoutStderrPipePath, os.O_CREATE|os.O_RDWR, 0666)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create log file: %w", err)
+	}
+	f.Close()
+
 	container, err = cdClient.NewContainer(
 		ctx,
 		containerID,
 		containerd.WithImage(image),
-		containerd.WithNewSnapshot(snapshotID, image),
+		containerd.WithRemappedSnapshot(snapshotID, image, hostUIDBegin, hostGIDBegin),
 		containerd.WithNewSpec(specOpts...),
 	)
 	if err != nil {
@@ -309,13 +331,25 @@
 	if err != nil {
 		return nil, err
 	}
+	cni, err := gocni.New(
+		gocni.WithPluginConfDir("/etc/cni/net.d"),
+		gocni.WithPluginDir([]string{"/opt/cni/bin"}),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize CNI: %w", err)
+	}
+	if err := cni.Load(gocni.WithDefaultConf); err != nil {
+		return nil, fmt.Errorf("failed to load CNI configurations: %w", err)
+	}
+
 	return &ContainerRunner{
-		container,
-		launchSpec,
-		attestAgent,
-		logger,
-		nvidiaAttester,
-		serialConsole,
+		container:     container,
+		launchSpec:    launchSpec,
+		attestAgent:   attestAgent,
+		logger:        logger,
+		gpuAttester:   nvidiaAttester,
+		serialConsole: serialConsole,
+		cni:           cni,
 	}, nil
 }
 
@@ -744,31 +778,82 @@
 		r.logger.Info("MemoryMonitoring is disabled by the VM operator")
 	}
 
-	var streamOpt cio.Opt
+	var logWriter io.Writer
 	switch r.launchSpec.LogRedirect {
 	case spec.Nowhere:
-		streamOpt = cio.WithStreams(nil, nil, nil)
+		logWriter = io.Discard
 		r.logger.Info("Container stdout/stderr will not be redirected.")
 	case spec.Everywhere:
-		w := io.MultiWriter(os.Stdout, r.serialConsole)
-		streamOpt = cio.WithStreams(nil, w, w)
+		logWriter = io.MultiWriter(os.Stdout, r.serialConsole)
 		r.logger.Info("Container stdout/stderr will be redirected to serial and Cloud Logging. This may result in performance issues due to slow serial console writes.")
 	case spec.CloudLogging:
-		streamOpt = cio.WithStreams(nil, os.Stdout, os.Stdout)
+		logWriter = os.Stdout
 		r.logger.Info("Container stdout/stderr will be redirected to Cloud Logging.")
 	case spec.Serial:
-		streamOpt = cio.WithStreams(nil, r.serialConsole, r.serialConsole)
+		logWriter = r.serialConsole
 		r.logger.Info("Container stdout/stderr will be redirected to serial logging. This may result in performance issues due to slow serial console writes.")
 	default:
 		return fmt.Errorf("unknown logging redirect location: %v", r.launchSpec.LogRedirect)
 	}
 
+	f, err := os.OpenFile(stdoutStderrPipePath, os.O_RDWR, 0)
+	if err != nil {
+		return fmt.Errorf("failed to open named pipe: %w", err)
+	}
+
+	streamOpt := cio.WithStreams(nil, f, f)
+
+	go func() {
+		defer f.Close()
+		io.Copy(logWriter, f)
+	}()
+
 	task, err := r.container.NewTask(ctx, cio.NewCreator(streamOpt))
 	if err != nil {
 		return &RetryableError{err}
 	}
 	defer task.Delete(ctx)
 
+	pid := task.Pid()
+	netnsPath := fmt.Sprintf("/proc/%d/ns/net", pid)
+
+	// Debug: Check where FDs 0, 1, 2 point to on the host
+	for i := 0; i <= 2; i++ {
+		fdPath := fmt.Sprintf("/proc/%d/fd/%d", pid, i)
+		linkTarget, err := os.Readlink(fdPath)
+		if err != nil {
+			r.logger.Error(fmt.Sprintf("Failed to read container FD %d link", i), "error", err)
+		} else {
+			r.logger.Info(fmt.Sprintf("Container FD %d points to", i), "target", linkTarget)
+		}
+	}
+
+	cniResult, err := r.cni.Setup(ctx, containerID, netnsPath)
+	if err != nil {
+		return fmt.Errorf("failed to setup network via CNI: %w", err)
+	}
+	r.logger.Info(fmt.Sprintf("CNI network setup completed: %v", cniResult))
+
+	image, err := r.container.Image(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get image from container: %w", err)
+	}
+	imageConfig, err := getImageConfig(ctx, image)
+	if err != nil {
+		return fmt.Errorf("failed to get image config: %w", err)
+	}
+
+	containerIP := ""
+	rawResults := cniResult.Raw()
+	// Currently, we have only single network interface defined with a single IP address by `10-workload.conf`.
+	if len(rawResults) > 0 && len(rawResults[0].IPs) > 0 {
+		containerIP = rawResults[0].IPs[0].Address.IP.String()
+	}
+
+	if err := openPorts(imageConfig.ExposedPorts, containerIP); err != nil {
+		return fmt.Errorf("failed to open and forward ports: %w", err)
+	}
+
 	setupDuration := time.Since(start)
 	r.logger.Info("Workload setup completed",
 		"setup_sec", setupDuration.Seconds(),
@@ -845,7 +930,7 @@
 }
 
 // openPorts writes firewall rules to accept all traffic into that port and protocol using iptables.
-func openPorts(ports map[string]struct{}) error {
+func openPorts(ports map[string]struct{}, containerIP string) error {
 	for k := range ports {
 		portAndProtocol := strings.Split(k, "/")
 		if len(portAndProtocol) != 2 {
@@ -875,6 +960,30 @@
 		if err != nil {
 			return fmt.Errorf("failed to open port on IPv6 %s %s: %v %s", port, protocol, err, out)
 		}
+
+		// Forward traffic from host port to container port with the same number!
+		if containerIP != "" {
+			forwardCmd := exec.Command("iptables", "-t", "nat", "-A", "PREROUTING",
+				"-p", protocol, "--dport", port,
+				"-j", "DNAT", "--to-destination", fmt.Sprintf("%s:%s", containerIP, port))
+
+			out, err = forwardCmd.CombinedOutput()
+			if err != nil {
+				return fmt.Errorf("failed to forward port %s to container %s: %v %s", port, containerIP, err, out)
+			}
+
+			// Allow traffic in FORWARD chain to the container IP on this port
+			forwardInCmd := exec.Command("iptables", "-A", "FORWARD", "-d", containerIP, "-p", protocol, "--dport", port, "-j", "ACCEPT")
+			if out, err := forwardInCmd.CombinedOutput(); err != nil {
+				return fmt.Errorf("failed to add FORWARD rule for container %s: %v %s", containerIP, err, out)
+			}
+
+			// Allow replies from the container to go out
+			forwardOutCmd := exec.Command("iptables", "-A", "FORWARD", "-s", containerIP, "-j", "ACCEPT")
+			if out, err := forwardOutCmd.CombinedOutput(); err != nil {
+				return fmt.Errorf("failed to add FORWARD reply rule for container %s: %v %s", containerIP, err, out)
+			}
+		}
 	}
 
 	return nil
@@ -905,6 +1014,16 @@
 	// close the agent
 	r.attestAgent.Close()
 
+	// Cleanup network using go-cni
+	task, err := r.container.Task(ctx, nil)
+	if err == nil {
+		pid := task.Pid()
+		netnsPath := fmt.Sprintf("/proc/%d/ns/net", pid)
+		if err := r.cni.Remove(ctx, containerID, netnsPath); err != nil {
+			r.logger.Error("failed to cleanup network via CNI", "error", err)
+		}
+	}
+
 	// Exit gracefully:
 	// Delete container and close connection to attestation service.
 	r.container.Delete(ctx, containerd.WithSnapshotCleanup)
@@ -926,6 +1045,43 @@
 	}
 }
 
+// withSysBindMount overrides the default /sys mount with a read-only bind mount.
+func withSysBindMount() oci.SpecOpts {
+	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
+		for i, m := range s.Mounts {
+			if m.Destination == "/sys" {
+				s.Mounts[i] = specs.Mount{
+					Destination: "/sys",
+					Type:        "bind",
+					Source:      "/sys",
+					Options:     []string{"rbind", "ro", "nosuid", "noexec", "nodev"},
+				}
+			}
+		}
+		return nil
+	}
+}
+
+func withStdoutStderrPipeMounts(pipePath string) oci.SpecOpts {
+	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
+		s.Mounts = append(s.Mounts,
+			specs.Mount{
+				Destination: "/dev/stdout",
+				Type:        "bind",
+				Source:      pipePath,
+				Options:     []string{"bind", "rw"},
+			},
+			specs.Mount{
+				Destination: "/dev/stderr",
+				Type:        "bind",
+				Source:      pipePath,
+				Options:     []string{"bind", "rw"},
+			},
+		)
+		return nil
+	}
+}
+
 // appendCgroupRw mount maps a cgroup as read-write.
 func appendCgroupRw(mounts []specs.Mount) []specs.Mount {
 	m := specs.Mount{

launcher/go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/confidentsecurity/go-nvtrust v0.2.2
 	github.com/containerd/containerd v1.7.23
 	github.com/containerd/containerd/v2 v2.0.1
+	github.com/containerd/go-cni v1.1.13
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/golang-jwt/jwt/v4 v4.5.2
 	github.com/google/go-cmp v0.7.0
@@ -58,6 +59,7 @@ require (
 	github.com/containerd/platforms v1.0.0-rc.0 // indirect
 	github.com/containerd/ttrpc v1.2.6 // indirect
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
+	github.com/containernetworking/cni v1.3.0 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -86,7 +88,9 @@ require (
 	github.com/moby/sys/user v0.3.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/opencontainers/selinux v1.11.1 // indirect
+	github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sasha-s/go-deadlock v0.3.5 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect