fix(policies): normalize downloaded assets to Unix format (#1427)

Scripts and AppArmor profiles are downloaded from the Windows AD server,
where administrators author them with Windows tooling. Those editors
routinely save files with CRLF line endings and a leading UTF-8 BOM.

On Linux both are silent foot-guns: a CRLF shebang makes the kernel look
for an interpreter path with a trailing carriage return, and a BOM sits
in front of the shebang entirely, so the script fails to execute even
though it looks correct. The same byte sequences trip up AppArmor
profile parsing. Administrators should not have to remember to re-save
every asset in Unix format for it to work.

Sanitizing at the asset compression funnel cleans every asset exactly
once per refresh and transparently covers all current and future asset
consumers (scripts, apparmor) from a single place, since they all read
from the compressed database. Files that look binary (they contain a NUL
byte) are left byte-for-byte intact so non-text assets are never
corrupted.

Closes #1425 
UDENG-10844

Author: denisonbarbosa

internal/policies/export_test.go

@@ -9,6 +9,11 @@ const (
 	PoliciesFileName       = policiesFileName
 )
 
+// SanitizeAssetContent exposes sanitizeAssetContent for testing.
+func SanitizeAssetContent(content []byte) []byte {
+	return sanitizeAssetContent(content)
+}
+
 // WithGDM specifies a personalized gdm manager.
 func WithGDM(m *gdm.Manager) Option {
 	return func(o *options) error {

internal/policies/policies.go

@@ -6,6 +6,7 @@ package policies
 
 import (
 	"archive/zip"
+	"bytes"
 	"context"
 	"errors"
 	"io"
@@ -347,13 +348,37 @@ func CompressAssets(ctx context.Context, p string) (err error) {
 		}
 
 		// #nosec G122 -- This is a path controlled by us
-		// Copy file content
+		// Assets are authored on and downloaded from the Windows AD server, so
+		// text files routinely carry CRLF line endings or a leading UTF-8 BOM
+		// that would otherwise break script execution and policy parsing on
+		// Linux. Sanitize text assets, but binary assets (which can be large)
+		// are stream-copied unchanged to avoid buffering them entirely.
 		srcF, err := os.Open(path)
 		if err != nil {
 			return err
 		}
 		defer srcF.Close()
-		if _, err = io.Copy(fZip, srcF); err != nil {
+
+		binary, err := isBinary(srcF)
+		if err != nil {
+			return err
+		}
+		if _, err := srcF.Seek(0, io.SeekStart); err != nil {
+			return err
+		}
+
+		if binary {
+			if _, err := io.Copy(fZip, srcF); err != nil {
+				return err
+			}
+			return nil
+		}
+
+		content, err := io.ReadAll(srcF)
+		if err != nil {
+			return err
+		}
+		if _, err = fZip.Write(sanitizeAssetContent(content)); err != nil {
 			return err
 		}
 
@@ -363,6 +388,59 @@ func CompressAssets(ctx context.Context, p string) (err error) {
 	return err
 }
 
+// utf8BOM is the UTF-8 byte order mark that Windows editors (e.g. Notepad,
+// PowerShell ISE) frequently prepend to files. When present at the start of a
+// script it sits before the shebang and prevents the kernel from recognizing
+// the interpreter.
+var utf8BOM = []byte{0xEF, 0xBB, 0xBF}
+
+// isBinary streams through r and reports whether it contains a NUL byte, the
+// conventional signal that content is binary and must be preserved
+// byte-for-byte. It reads in chunks so large binary assets are not buffered in
+// memory entirely.
+func isBinary(r io.Reader) (bool, error) {
+	buf := make([]byte, 32*1024)
+	for {
+		n, err := r.Read(buf)
+		if bytes.IndexByte(buf[:n], 0) != -1 {
+			return true, nil
+		}
+		if err == io.EOF {
+			return false, nil
+		}
+		if err != nil {
+			return false, err
+		}
+	}
+}
+
+// sanitizeAssetContent normalizes Windows-specific byte sequences in a
+// downloaded asset so it works out of the box on Linux:
+//   - a leading UTF-8 BOM is removed;
+//   - CRLF (\r\n) line endings are converted to LF (\n).
+//
+// Assets are authored on and downloaded from the Windows AD server, where these
+// sequences are routinely introduced by editors. They break script execution
+// (a CRLF shebang resolves to an interpreter path with a trailing \r) and
+// policy parsing, forcing administrators to remember to save assets in Unix
+// format.
+//
+// Files that look binary (i.e. contain a NUL byte) are returned unchanged to
+// avoid corrupting non-text assets an administrator may have placed alongside
+// scripts and profiles.
+func sanitizeAssetContent(content []byte) []byte {
+	// A NUL byte is the conventional signal that content is binary and must be
+	// preserved byte-for-byte.
+	if bytes.IndexByte(content, 0) != -1 {
+		return content
+	}
+
+	content = bytes.TrimPrefix(content, utf8BOM)
+	content = bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n"))
+
+	return content
+}
+
 // GetUniqueRules return order rules, with one entry per key for a given type.
 // Returned file is a map of type to its entries.
 func (pols Policies) GetUniqueRules() map[string][]entry.Entry {

internal/policies/policies_test.go

@@ -518,6 +518,140 @@ func TestCompressAssets(t *testing.T) {
 	}
 }
 
+func TestSanitizeAssetContent(t *testing.T) {
+	t.Parallel()
+
+	bom := []byte{0xEF, 0xBB, 0xBF}
+	withBOM := func(b []byte) []byte {
+		return append(append([]byte{}, bom...), b...)
+	}
+
+	tests := map[string]struct {
+		content []byte
+
+		want []byte
+	}{
+		// Normalized cases
+		"Plain LF content is left untouched": {
+			content: []byte("#!/bin/sh\necho hello\n"),
+			want:    []byte("#!/bin/sh\necho hello\n"),
+		},
+		"CRLF is converted to LF": {
+			content: []byte("#!/bin/sh\r\necho hello\r\n"),
+			want:    []byte("#!/bin/sh\necho hello\n"),
+		},
+		"Multiple CRLF are all converted": {
+			content: []byte("a\r\nb\r\nc\r\n"),
+			want:    []byte("a\nb\nc\n"),
+		},
+		"Leading BOM is stripped": {
+			content: withBOM([]byte("#!/bin/sh\n")),
+			want:    []byte("#!/bin/sh\n"),
+		},
+		"Leading BOM and CRLF are both normalized": {
+			content: withBOM([]byte("#!/bin/sh\r\necho hi\r\n")),
+			want:    []byte("#!/bin/sh\necho hi\n"),
+		},
+		"Content consisting only of a BOM is emptied": {
+			content: withBOM(nil),
+			want:    nil,
+		},
+		"Empty content is left untouched": {
+			content: []byte{},
+			want:    nil,
+		},
+
+		// Preservation cases
+		"Lone CR is preserved": {
+			content: []byte("progress\rdone\r\n"),
+			want:    []byte("progress\rdone\n"),
+		},
+		"BOM not at the start is preserved": {
+			content: append([]byte("prefix"), withBOM([]byte("\n"))...),
+			want:    append([]byte("prefix"), withBOM([]byte("\n"))...),
+		},
+
+		// Binary cases: a NUL byte means the file must be preserved verbatim.
+		"Binary content with CRLF is preserved": {
+			content: []byte("\x00\x01\x02\r\n"),
+			want:    []byte("\x00\x01\x02\r\n"),
+		},
+		"Binary content with a leading BOM is preserved": {
+			content: withBOM([]byte("\x00binary\r\n")),
+			want:    withBOM([]byte("\x00binary\r\n")),
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			got := policies.SanitizeAssetContent(tc.content)
+
+			// Compare as strings to gracefully handle nil vs empty slices.
+			require.Equal(t, string(tc.want), string(got), "SanitizeAssetContent returned unexpected content")
+		})
+	}
+}
+
+func TestCompressAssetsSanitizesContent(t *testing.T) {
+	t.Parallel()
+
+	bom := []byte{0xEF, 0xBB, 0xBF}
+	withBOM := func(b []byte) []byte {
+		return append(append([]byte{}, bom...), b...)
+	}
+
+	crlfScript := []byte("#!/bin/sh\r\necho crlf\r\n")
+	wantCrlfScript := []byte("#!/bin/sh\necho crlf\n")
+	bomScript := withBOM([]byte("#!/bin/sh\r\necho bom\r\n"))
+	wantBomScript := []byte("#!/bin/sh\necho bom\n")
+	lfScript := []byte("#!/bin/sh\necho lf\n")
+	// A binary asset carrying both a BOM and CRLF must survive untouched.
+	binaryAsset := withBOM([]byte("\x00\x01\r\n\x02"))
+
+	// content authored on Windows, mapped to the expected sanitized result.
+	assets := map[string]struct {
+		content []byte
+		want    []byte
+	}{
+		filepath.Join("scripts", "crlf.sh"):  {content: crlfScript, want: wantCrlfScript},
+		filepath.Join("scripts", "bom.sh"):   {content: bomScript, want: wantBomScript},
+		filepath.Join("scripts", "lf.sh"):    {content: lfScript, want: lfScript},
+		filepath.Join("scripts", "binary"):   {content: binaryAsset, want: binaryAsset},
+		filepath.Join("apparmor", "usr.bin"): {content: bomScript, want: wantBomScript},
+	}
+
+	// Lay out the assets directory the same way it exists after a download.
+	parentDir := t.TempDir()
+	assetsDir := filepath.Join(parentDir, "assets")
+	for relPath, asset := range assets {
+		dest := filepath.Join(assetsDir, relPath)
+		require.NoError(t, os.MkdirAll(filepath.Dir(dest), 0700), "Setup: can’t create asset subdirectory")
+		require.NoError(t, os.WriteFile(dest, asset.content, 0600), "Setup: can’t write asset file")
+	}
+
+	require.NoError(t, policies.CompressAssets(context.Background(), assetsDir), "CompressAssets should return no error but got one")
+
+	// Drop the uncompressed assets so the policies are loaded purely from the db,
+	// then extract them again to inspect what was actually stored.
+	require.NoError(t, os.RemoveAll(assetsDir), "Teardown: can’t remove uncompressed assets directory")
+	require.NoError(t, os.WriteFile(filepath.Join(parentDir, policies.PoliciesFileName), nil, 0600), "Setup: can’t create empty policy cache file")
+
+	pols, err := policies.NewFromCache(context.Background(), parentDir)
+	require.NoError(t, err, "NewFromCache should return no error but got one")
+	defer pols.Close()
+
+	extractDir := filepath.Join(t.TempDir(), "extracted")
+	require.NoError(t, pols.SaveAssetsTo(context.Background(), ".", extractDir, -1, -1), "SaveAssetsTo should return no error but got one")
+
+	for relPath, asset := range assets {
+		got, err := os.ReadFile(filepath.Join(extractDir, relPath))
+		require.NoError(t, err, "should be able to read extracted asset %q", relPath)
+		require.Equal(t, string(asset.want), string(got), "unexpected stored content for asset %q", relPath)
+	}
+}
+
 func TestGetUniqueRules(t *testing.T) {
 	t.Parallel()