scans_test.go

package main

import (
	"encoding/json"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"testing"

	"github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/ocicontainer"
	"github.com/jfrog/jfrog-cli-artifactory/utils/tests"

	"github.com/stretchr/testify/assert"

	"github.com/jfrog/jfrog-cli-security/commands/scan"
	securityTests "github.com/jfrog/jfrog-cli-security/tests"
	securityTestUtils "github.com/jfrog/jfrog-cli-security/tests/utils"
	"github.com/jfrog/jfrog-cli-security/tests/utils/integration"
	"github.com/jfrog/jfrog-cli-security/tests/validations"
	"github.com/jfrog/jfrog-cli-security/utils"
	"github.com/jfrog/jfrog-cli-security/utils/formats"
	"github.com/jfrog/jfrog-cli-security/utils/jasutils"

	"github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/container"

	"github.com/jfrog/jfrog-cli-core/v2/common/build"
	"github.com/jfrog/jfrog-cli-core/v2/common/format"
	commonTests "github.com/jfrog/jfrog-cli-core/v2/common/tests"
	coreTests "github.com/jfrog/jfrog-cli-core/v2/utils/tests"

	"github.com/jfrog/jfrog-cli-security/utils/xray/scangraph"
	"github.com/jfrog/jfrog-client-go/utils/log"
	clientTestUtils "github.com/jfrog/jfrog-client-go/utils/tests"
	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
)

type binaryScanParams struct {
	// The argument to scan, e.g., a binary file or a directory containing binaries
	BinaryPattern string
	// --format flag value if provided
	Format format.OutputFormat
	// --licenses flag value if provided
	WithLicense bool
	// --sbom flag value if provided
	WithSbom bool
	// selective scans to perform
	WithSubScans []utils.SubScanType
	// -- without-contextual-analysis flag value if provided
	WithoutContextualAnalysis bool
	// Will combined with "," if provided and be used as --watches flag value
	Watches []string
	// -- vuln flag 'True' value must be provided with 'createWatchesFuncs' to create watches for the test
	WithVuln bool
	// Number of threads to use for scanning
	Threads int
}

func subScansToFlags(subScans []utils.SubScanType) (flags []string) {
	for _, scanType := range subScans {
		switch scanType {
		case utils.ScaScan:
			flags = append(flags, "--sca")
		case utils.SecretsScan:
			flags = append(flags, "--secrets")
		}
	}
	return flags
}

func getBinaryScanCmdArgs(params binaryScanParams) (args []string) {
	if params.WithLicense {
		args = append(args, "--licenses")
	}
	if params.WithSbom {
		args = append(args, "--sbom")
	}
	if params.Format != "" {
		args = append(args, "--format="+string(params.Format))
	}
	if len(params.WithSubScans) > 0 {
		args = append(args, subScansToFlags(params.WithSubScans)...)
	}
	if params.WithoutContextualAnalysis {
		args = append(args, "--without-contextual-analysis")
	}
	if params.WithVuln {
		args = append(args, "--vuln")
	}
	if len(params.Watches) > 0 {
		args = append(args, "--watches="+strings.Join(params.Watches, ","))
	}
	if params.Threads > 0 {
		args = append(args, "--threads="+strconv.Itoa(params.Threads))
	}
	args = append(args, params.BinaryPattern)
	return args
}

func testXrayBinaryScan(t *testing.T, params binaryScanParams, errorExpected bool) string {
	output, err := runXrayBinaryScan(t, params)
	if errorExpected {
		if err != nil {
			log.Error("Xray binary scan failed with err: " + err.Error())
		}
		assert.Error(t, err)
	} else {
		assert.NoError(t, err)
	}
	return output
}

func runXrayBinaryScan(t *testing.T, params binaryScanParams) (string, error) {
	cleanUp := integration.UseTestHomeWithDefaultXrayConfig(t)
	defer cleanUp()
	return securityTests.PlatformCli.RunCliCmdWithOutputs(t, append([]string{"scan"}, getBinaryScanCmdArgs(params)...)...)
}

// Binary scan tests
func TestXrayBinaryScanJson(t *testing.T) {
	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
	output := testXrayMultipleBinariesScan(t, binaryScanParams{Format: format.Json, WithLicense: true}, false)
	validations.VerifyJsonResults(t, output, validations.ValidationParams{
		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1},
	})
}

func TestXrayBinaryScanSimpleJson(t *testing.T) {
	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
	output := testXrayBinaryScanWithWatch(t, format.SimpleJson, "xray-scan-binary-policy", "scan-binary-watch", true)
	validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1, Violations: 1},
	})
}

func TestXrayBinaryScanCycloneDx(t *testing.T) {
	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
	output := testXrayBinaryScanJASArtifact(t, "backupfriend-client.tar.gz", false, binaryScanParams{Format: format.CycloneDx})
	validations.VerifyCycloneDxResults(t, output, validations.ValidationParams{
		Total: &validations.TotalCount{Vulnerabilities: 4},
		Vulnerabilities: &validations.VulnerabilityCount{
			ValidateScan:                &validations.ScanCount{Sca: 3, Secrets: 1},
			ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 2, Undetermined: 1},
		},
	})
}

func TestXrayBinaryScanJsonDocker(t *testing.T) {
	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
	// In Windows, indexer fails to index the tar, Caused by: failed to rename path for layer so we run in clean copy in temp dir
	output := testXrayBinaryScanJASArtifact(t, "xmas.tar", true, binaryScanParams{Format: format.SimpleJson})
	validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
		Total: &validations.TotalCount{Vulnerabilities: 6},
		Vulnerabilities: &validations.VulnerabilityCount{
			ValidateScan:                &validations.ScanCount{Sca: 4, Secrets: 2},
			ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 2, NotApplicable: 1, NotCovered: 1},
		},
	})
}

func TestXrayBinaryScanJsonGeneric(t *testing.T) {
	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
	output := testXrayBinaryScanJASArtifact(t, "backupfriend-client.tar.gz", false, binaryScanParams{Format: format.SimpleJson})
	validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
		Total: &validations.TotalCount{Vulnerabilities: 4},
		Vulnerabilities: &validations.VulnerabilityCount{
			ValidateScan:                &validations.ScanCount{Sca: 3, Secrets: 1},
			ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 2, Undetermined: 1},
		},
	})
}

func TestXrayBinaryScanJsonJar(t *testing.T) {
	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
	output := testXrayBinaryScanJASArtifact(t, "student-services-security-0.0.1.jar", false, binaryScanParams{Format: format.SimpleJson})
	validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
		Total: &validations.TotalCount{Vulnerabilities: 41},
		Vulnerabilities: &validations.VulnerabilityCount{
			ValidateScan:                &validations.ScanCount{Sca: 40, Secrets: 1},
			ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 17, NotCovered: 3, NotApplicable: 20},
		},
	})
}

func TestXrayBinaryScanSelectiveScan(t *testing.T) {
	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
	testCases := []struct {
		name      string
		subScans  []utils.SubScanType
		withoutCa bool
		validate  func(t *testing.T, issueCount validations.ValidationCountActualValues)
	}{
		{
			name:      "SCA only scan",
			subScans:  []utils.SubScanType{utils.ScaScan},
			withoutCa: true,
			validate: func(t *testing.T, issueCount validations.ValidationCountActualValues) {
				// Expect only SCA vulnerabilities
				assert.GreaterOrEqual(t, issueCount.ScaVulnerabilities, 3, "SCA vulnerabilities count mismatch - should be 3 or more")
				// All other vulnerability types should be 0
				assert.Equal(t, 0, issueCount.SecretsVulnerabilities, "Secrets vulnerabilities count mismatch - should be 0")
				assert.Equal(t, 0, issueCount.ApplicableVulnerabilities, "Applicable vulnerabilities count mismatch - should be 0")
				assert.Equal(t, 0, issueCount.UndeterminedVulnerabilities, "Undetermined vulnerabilities count mismatch - should be 0")
			},
		},
		{
			name:     "Secrets only scan",
			subScans: []utils.SubScanType{utils.SecretsScan},
			validate: func(t *testing.T, issueCount validations.ValidationCountActualValues) {
				// Expect only Secrets vulnerabilities
				assert.GreaterOrEqual(t, issueCount.SecretsVulnerabilities, 1, "Secrets vulnerabilities count mismatch - should be 1 or more")
				// All other vulnerability types should be 0
				assert.Equal(t, 0, issueCount.ScaVulnerabilities, "SCA vulnerabilities count mismatch - should be 0")
				assert.Equal(t, 0, issueCount.ApplicableVulnerabilities, "Applicable vulnerabilities count mismatch - should be 0")
				assert.Equal(t, 0, issueCount.UndeterminedVulnerabilities, "Undetermined vulnerabilities count mismatch - should be 0")
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			output := testXrayBinaryScanJASArtifact(t, "backupfriend-client.tar.gz", false, binaryScanParams{Format: format.CycloneDx, WithSubScans: tc.subScans, WithoutContextualAnalysis: tc.withoutCa})
			tc.validate(t, validations.GetCycloneDxActualValues(t, output))
		})
	}
}

func TestXrayBinaryScanJsonWithProgress(t *testing.T) {
	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
	callback := commonTests.MockProgressInitialization()
	defer callback()
	output := testXrayMultipleBinariesScan(t, binaryScanParams{Format: format.Json, WithLicense: true}, false)
	validations.VerifyJsonResults(t, output, validations.ValidationParams{
		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1},
	})
}

func TestXrayBinaryScanSimpleJsonWithProgress(t *testing.T) {
	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
	callback := commonTests.MockProgressInitialization()
	defer callback()
	output := testXrayBinaryScanWithWatch(t, format.SimpleJson, "xray-scan-binary-progress-policy", "scan-binary-progress-watch", true)
	validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1, Violations: 1},
	})
}

func testXrayBinaryScanWithWatch(t *testing.T, format format.OutputFormat, policyName, watchName string, errorExpected bool) string {
	params := binaryScanParams{
		WithLicense: true,
		Format:      format,
	}
	if policyName != "" && watchName != "" {
		watchName, deleteWatch := securityTestUtils.CreateTestPolicyAndWatch(t, "xray-scan-binary-policy", "scan-binary-watch", xrayUtils.High)
		defer deleteWatch()
		// Include violations and vulnerabilities
		params.Watches = []string{watchName}
		params.WithVuln = true
	}
	return testXrayMultipleBinariesScan(t, params, errorExpected)
}

func testXrayMultipleBinariesScan(t *testing.T, params binaryScanParams, errorExpected bool) string {
	params.BinaryPattern = filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "binaries", "*")
	return testXrayBinaryScan(t, params, errorExpected)
}

func testXrayBinaryScanJASArtifact(t *testing.T, artifact string, inTempDir bool, params binaryScanParams) string {
	pathToScan := filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "jas-scan")
	if inTempDir {
		var cleanUp func()
		pathToScan, cleanUp = securityTestUtils.CreateTestProjectEnvAndChdir(t, pathToScan)
		defer cleanUp()
	}
	params.BinaryPattern = filepath.Join(pathToScan, artifact)
	if params.Threads == 0 {
		params.Threads = 5
	}
	return testXrayBinaryScan(t,
		params,
		false,
	)
}

func TestXrayBinaryScanWithBypassArchiveLimits(t *testing.T) {
	integration.InitScanTest(t, scan.BypassArchiveLimitsMinXrayVersion)
	unsetEnv := clientTestUtils.SetEnvWithCallbackAndAssert(t, "JF_INDEXER_COMPRESS_MAXENTITIES", "10")
	defer unsetEnv()
	cleanUp := integration.UseTestHomeWithDefaultXrayConfig(t)
	defer cleanUp()
	binariesPath := filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "binaries", "*")
	scanArgs := []string{"scan", binariesPath, "--format=json", "--licenses"}
	// Run without bypass flag and expect scan to fail
	err := securityTests.PlatformCli.Exec(scanArgs...)
	// Expect error
	assert.Error(t, err)

	// Run with bypass flag and expect it to find vulnerabilities
	scanArgs = append(scanArgs, "--bypass-archive-limits")
	output := securityTests.PlatformCli.RunCliCmdWithOutput(t, scanArgs...)
	validations.VerifyJsonResults(t, output, validations.ValidationParams{
		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1},
	})
}

// Docker scan tests

func TestDockerScanWithProgressBar(t *testing.T) {
	callback := commonTests.MockProgressInitialization()
	defer callback()
	TestDockerScan(t)
}

func TestDockerScanWithTokenValidation(t *testing.T) {
	securityTestUtils.GetAndValidateXrayVersion(t, jasutils.DynamicTokenValidationMinXrayVersion)
	testCli, cleanup := integration.InitNativeDockerTest(t)
	defer cleanup()
	// #nosec G101 -- Image with dummy token for tests
	tokensImageToScan := "srmishj/inactive_tokens:latest"
	runDockerScan(t, testCli, tokensImageToScan, "", 0, 0, 0, 5, true)
}

func TestDockerScan(t *testing.T) {
	testCli, cleanup := integration.InitNativeDockerTest(t)
	defer cleanup()

	watchName, deleteWatch := securityTestUtils.CreateTestPolicyAndWatch(t, "docker-policy", "docker-watch", xrayUtils.Low)
	defer deleteWatch()

	imagesToScan := []string{
		// Image with RPM with vulnerabilities
		"alpine:3.17",
	}
	for _, imageName := range imagesToScan {
		runDockerScan(t, testCli, imageName, watchName, 3, 3, 3, 0, false)
	}

	// Image with 0 vulnerabilities
	runDockerScan(t, testCli, "busybox:1.35", "", 0, 0, 0, 0, false)
}

func runDockerScan(t *testing.T, testCli *coreTests.JfrogCli, imageName, watchName string, minViolations, minVulnerabilities, minLicenses int, minInactives int, validateSecrets bool) {
	// Pull image from docker repo
	imageTag := path.Join(*securityTests.ContainerRegistry, securityTests.DockerVirtualRepo, imageName)
	dockerPullCommand := container.NewPullCommand(ocicontainer.DockerClient)
	dockerPullCommand.SetCmdParams([]string{"pull", imageTag}).SetImageTag(imageTag).SetRepo(securityTests.DockerVirtualRepo).SetServerDetails(securityTests.XrDetails).SetBuildConfiguration(new(build.BuildConfiguration))
	if !assert.NoError(t, dockerPullCommand.Run()) {
		return
	}
	defer tests.DeleteTestImage(t, imageTag, ocicontainer.DockerClient)
	cleanUp := integration.UseTestHomeWithDefaultXrayConfig(t)
	defer cleanUp()
	// Run docker scan on image
	cmdArgs := []string{"docker", "scan", imageTag, "--server-id=default", "--licenses", "--fail=false", "--min-severity=low", "--fixable-only"}
	if validateSecrets {
		cmdArgs = append(cmdArgs, "--secrets", "--validate-secrets", "--format=simple-json")
	} else {
		cmdArgs = append(cmdArgs, "--format=json")
	}
	if watchName != "" {
		cmdArgs = append(cmdArgs, "--watches="+watchName, "--vuln")
	}
	output := testCli.WithoutCredentials().RunCliCmdWithOutput(t, cmdArgs...)
	if assert.NotEmpty(t, output) {
		if validateSecrets {
			validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
				Vulnerabilities: &validations.VulnerabilityCount{ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Inactive: minInactives}},
			})
		} else {
			validationParams := validations.ValidationParams{
				Total: &validations.TotalCount{
					Vulnerabilities: minVulnerabilities,
					Licenses:        minLicenses,
					Violations:      minViolations,
				},
			}
			validations.VerifyJsonResults(t, output, validationParams)
		}
	}
}

// JAS docker scan tests

func TestAdvancedSecurityDockerScan(t *testing.T) {
	testCli, cleanup := integration.InitNativeDockerTest(t)
	defer cleanup()
	runAdvancedSecurityDockerScan(t, testCli, "jfrog/demo-security:latest")
}

func runAdvancedSecurityDockerScan(t *testing.T, testCli *coreTests.JfrogCli, imageName string) {
	// Pull image from docker repo
	imageTag := path.Join(*securityTests.ContainerRegistry, securityTests.DockerVirtualRepo, imageName)
	dockerPullCommand := container.NewPullCommand(ocicontainer.DockerClient)
	dockerPullCommand.SetCmdParams([]string{"pull", imageTag}).SetImageTag(imageTag).SetRepo(securityTests.DockerVirtualRepo).SetServerDetails(securityTests.XrDetails).SetBuildConfiguration(new(build.BuildConfiguration))
	if assert.NoError(t, dockerPullCommand.Run()) {
		defer tests.DeleteTestImage(t, imageTag, ocicontainer.DockerClient)
		args := []string{"docker", "scan", imageTag, "--server-id=default", "--format=simple-json", "--fail=false", "--min-severity=low", "--fixable-only"}
		cleanUp := integration.UseTestHomeWithDefaultXrayConfig(t)
		defer cleanUp()
		// Run docker scan on image
		output := testCli.WithoutCredentials().RunCliCmdWithOutput(t, args...)
		if assert.NotEmpty(t, output) {
			verifyAdvancedSecurityScanResults(t, output, true, true)
		}
	}
}

func verifyAdvancedSecurityScanResults(t *testing.T, content string, isApplicable bool, isSecret bool) {
	var results formats.SimpleJsonResults
	err := json.Unmarshal([]byte(content), &results)
	assert.NoError(t, err)
	// Verify that the scan succeeded, and that at least one "Applicable" status was received.
	applicableStatusExists := false
	for _, vulnerability := range results.Vulnerabilities {
		if vulnerability.Applicable == string(jasutils.Applicable) {
			applicableStatusExists = true
			break
		}
	}

	if isApplicable {
		assert.True(t, applicableStatusExists)
	}

	if isSecret {
		// Verify that secretes detection succeeded.
		assert.NotEqual(t, 0, len(results.SecretsVulnerabilities))
	} else {
		// Verify that secretes detection succeeded.
		assert.Equal(t, 0, len(results.SecretsVulnerabilities))
	}
}