fix(spdx): use the hasExtractedLicensingInfos field for licenses that are not listed in the SPDX (#8077)

Author: DmitriyLewen

.github/workflows/spdx-cron.yaml

@@ -0,0 +1,33 @@
+name: SPDX licenses cron
+on:
+  schedule:
+    - cron: '0 0 * * 0' # every Sunday at 00:00
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Check if SPDX exceptions
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4.1.6
+
+      - name: Check if SPDX exceptions are up-to-date
+        run: |
+          mage spdx:updateLicenseExceptions
+          if [ -n "$(git status --porcelain)" ]; then
+            echo "Run 'mage spdx:updateLicenseExceptions' and push it"
+            exit 1
+          fi        
+
+      - name: Microsoft Teams Notification
+        ## Until the PR with the fix for the AdaptivCard version is merged yet
+        ## https://github.com/Skitionek/notify-microsoft-teams/pull/96
+        ## Use the aquasecurity fork
+        uses: aquasecurity/notify-microsoft-teams@master
+        if: failure()
+        with:
+          webhook_url: ${{ secrets.TRIVY_MSTEAMS_WEBHOOK }}
+          needs: ${{ toJson(needs) }}
+          job: ${{ toJson(job) }}
+          steps: ${{ toJson(steps) }}

magefiles/magefile.go

@@ -533,3 +533,10 @@ type Helm mg.Namespace
 func (Helm) UpdateVersion() error {
 	return sh.RunWith(ENV, "go", "run", "-tags=mage_helm", "./magefiles")
 }
+
+type SPDX mg.Namespace
+
+// UpdateLicenseExceptions updates 'exception.json' with SPDX license exceptions
+func (SPDX) UpdateLicenseExceptions() error {
+	return sh.RunWith(ENV, "go", "run", "-tags=mage_spdx", "./magefiles/spdx.go")
+}

magefiles/spdx.go

@@ -0,0 +1,78 @@
+//go:build mage_spdx
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/samber/lo"
+	"golang.org/x/xerrors"
+
+	"github.com/aquasecurity/trivy/pkg/downloader"
+	"github.com/aquasecurity/trivy/pkg/log"
+)
+
+const (
+	exceptionFileName = "exceptions.json"
+	exceptionDir      = "./pkg/licensing/expression"
+	exceptionURL      = "https://spdx.org/licenses/exceptions.json"
+)
+
+type Exceptions struct {
+	Exceptions []Exception `json:"exceptions"`
+}
+
+type Exception struct {
+	ID string `json:"licenseExceptionId"`
+}
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatal("Fatal error", log.Err(err))
+	}
+
+}
+
+// run downloads exceptions.json file, takes only IDs and saves into `expression` package.
+func run() error {
+	tmpDir, err := downloader.DownloadToTempDir(context.Background(), exceptionURL, downloader.Options{})
+	if err != nil {
+		return xerrors.Errorf("unable to download exceptions.json file: %w", err)
+	}
+	tmpFile, err := os.ReadFile(filepath.Join(tmpDir, exceptionFileName))
+	if err != nil {
+		return xerrors.Errorf("unable to read exceptions.json file: %w", err)
+	}
+
+	exceptions := Exceptions{}
+	if err = json.Unmarshal(tmpFile, &exceptions); err != nil {
+		return xerrors.Errorf("unable to unmarshal exceptions.json file: %w", err)
+	}
+
+	exs := lo.Map(exceptions.Exceptions, func(ex Exception, _ int) string {
+		return ex.ID
+	})
+	sort.Strings(exs)
+
+	exceptionFile := filepath.Join(exceptionDir, exceptionFileName)
+	f, err := os.Create(exceptionFile)
+	if err != nil {
+		return xerrors.Errorf("unable to create file %s: %w", exceptionFile, err)
+	}
+	defer f.Close()
+
+	e, err := json.Marshal(exs)
+	if err != nil {
+		return xerrors.Errorf("unable to marshal exceptions list: %w", err)
+	}
+
+	if _, err = f.Write(e); err != nil {
+		return xerrors.Errorf("unable to write exceptions list: %w", err)
+	}
+
+	return nil
+}

pkg/licensing/expression/category.go

@@ -1,5 +1,18 @@
 package expression
 
+import (
+	"encoding/json"
+	"strings"
+	"sync"
+
+	"github.com/samber/lo"
+
+	"github.com/aquasecurity/trivy/pkg/log"
+	"github.com/aquasecurity/trivy/pkg/set"
+
+	_ "embed"
+)
+
 // Canonical names of the licenses.
 // ported from https://github.com/google/licenseclassifier/blob/7c62d6fe8d3aa2f39c4affb58c9781d9dc951a2d/license_type.go#L24-L177
 const (
@@ -359,3 +372,70 @@ var (
 		ZeroBSD,
 	}
 )
+
+var spdxLicenses = set.New[string]()
+
+var initSpdxLicenses = sync.OnceFunc(func() {
+	if spdxLicenses.Size() > 0 {
+		return
+	}
+
+	licenseSlices := [][]string{
+		ForbiddenLicenses,
+		RestrictedLicenses,
+		ReciprocalLicenses,
+		NoticeLicenses,
+		PermissiveLicenses,
+		UnencumberedLicenses,
+	}
+
+	for _, licenseSlice := range licenseSlices {
+		spdxLicenses.Append(licenseSlice...)
+	}
+
+	// Save GNU licenses with "-or-later" and `"-only" suffixes
+	for _, l := range GnuLicenses {
+		license := SimpleExpr{
+			License: l,
+		}
+		spdxLicenses.Append(license.String())
+
+		license.HasPlus = true
+		spdxLicenses.Append(license.String())
+	}
+})
+
+//go:embed exceptions.json
+var exceptions []byte
+
+var spdxExceptions map[string]SimpleExpr
+
+var initSpdxExceptions = sync.OnceFunc(func() {
+	if len(spdxExceptions) > 0 {
+		return
+	}
+
+	var exs []string
+	if err := json.Unmarshal(exceptions, &exs); err != nil {
+		log.WithPrefix(log.PrefixSPDX).Warn("Unable to parse SPDX exception file", log.Err(err))
+		return
+	}
+	spdxExceptions = lo.SliceToMap(exs, func(exception string) (string, SimpleExpr) {
+		return strings.ToUpper(exception), SimpleExpr{License: exception}
+	})
+})
+
+// ValidateSPDXLicense returns true if SPDX license list contain licenseID
+func ValidateSPDXLicense(license string) bool {
+	initSpdxLicenses()
+
+	return spdxLicenses.Contains(license)
+}
+
+// ValidateSPDXException returns true if SPDX exception list contain exceptionID
+func ValidateSPDXException(exception string) bool {
+	initSpdxExceptions()
+
+	_, ok := spdxExceptions[strings.ToUpper(exception)]
+	return ok
+}

pkg/licensing/expression/exceptions.json

@@ -0,0 +1 @@
+["389-exception","Asterisk-exception","Asterisk-linking-protocols-exception","Autoconf-exception-2.0","Autoconf-exception-3.0","Autoconf-exception-generic","Autoconf-exception-generic-3.0","Autoconf-exception-macro","Bison-exception-1.24","Bison-exception-2.2","Bootloader-exception","CGAL-linking-exception","CLISP-exception-2.0","Classpath-exception-2.0","DigiRule-FOSS-exception","FLTK-exception","Fawkes-Runtime-exception","Font-exception-2.0","GCC-exception-2.0","GCC-exception-2.0-note","GCC-exception-3.1","GNAT-exception","GNOME-examples-exception","GNU-compiler-exception","GPL-3.0-389-ds-base-exception","GPL-3.0-interface-exception","GPL-3.0-linking-exception","GPL-3.0-linking-source-exception","GPL-CC-1.0","GStreamer-exception-2005","GStreamer-exception-2008","Gmsh-exception","Independent-modules-exception","KiCad-libraries-exception","LGPL-3.0-linking-exception","LLGPL","LLVM-exception","LZMA-exception","Libtool-exception","Linux-syscall-note","Nokia-Qt-exception-1.1","OCCT-exception-1.0","OCaml-LGPL-linking-exception","OpenJDK-assembly-exception-1.0","PCRE2-exception","PS-or-PDF-font-exception-20170817","QPL-1.0-INRIA-2004-exception","Qt-GPL-exception-1.0","Qt-LGPL-exception-1.1","Qwt-exception-1.0","RRDtool-FLOSS-exception-2.0","SANE-exception","SHL-2.0","SHL-2.1","SWI-exception","Swift-exception","Texinfo-exception","UBDL-exception","Universal-FOSS-exception-1.0","WxWindows-exception-3.1","cryptsetup-OpenSSL-exception","eCos-exception-2.0","erlang-otp-linking-exception","fmt-exception","freertos-exception-2.0","gnu-javamail-exception","harbour-exception","i2p-gpl-java-exception","libpri-OpenH323-exception","mif-exception","mxml-exception","openvpn-openssl-exception","romic-exception","stunnel-exception","u-boot-exception-2.0","vsftpd-openssl-exception","x11vnc-openssl-exception"]

pkg/licensing/expression/expression.go

@@ -59,26 +59,33 @@ func normalize(expr Expression, fn NormalizeFunc) Expression {
 // There MUST be white space on either side of the operator "WITH".
 // ref: https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions
 func NormalizeForSPDX(expr Expression) Expression {
-	e, ok := expr.(SimpleExpr)
-	if !ok {
-		return expr // do not normalize compound expressions
-	}
-
-	var b strings.Builder
-	for _, c := range e.License {
-		switch {
-		// spec: idstring = 1*(ALPHA / DIGIT / "-" / "." )
-		case isAlphabet(c) || unicode.IsNumber(c) || c == '-' || c == '.':
-			_, _ = b.WriteRune(c)
-		case c == ':':
-			// TODO: Support DocumentRef
-			_, _ = b.WriteRune(c)
-		default:
-			// Replace invalid characters with '-'
-			_, _ = b.WriteRune('-')
+	switch e := expr.(type) {
+	case SimpleExpr:
+		var b strings.Builder
+		for _, c := range e.License {
+			switch {
+			// spec: idstring = 1*(ALPHA / DIGIT / "-" / "." )
+			case isAlphabet(c) || unicode.IsNumber(c) || c == '-' || c == '.':
+				_, _ = b.WriteRune(c)
+			case c == ':':
+				// TODO: Support DocumentRef
+				_, _ = b.WriteRune(c)
+			default:
+				// Replace invalid characters with '-'
+				_, _ = b.WriteRune('-')
+			}
+		}
+		return SimpleExpr{License: b.String(), HasPlus: e.HasPlus}
+	case CompoundExpr:
+		if e.Conjunction() == TokenWith {
+			initSpdxExceptions()
+			// Use correct SPDX exceptionID
+			if exc, ok := spdxExceptions[strings.ToUpper(e.Right().String())]; ok {
+				return NewCompoundExpr(e.Left(), e.Conjunction(), exc)
+			}
 		}
 	}
-	return SimpleExpr{License: b.String(), HasPlus: e.HasPlus}
+	return expr
 }
 
 func isAlphabet(r rune) bool {

pkg/licensing/expression/types.go

@@ -56,6 +56,14 @@ func (c CompoundExpr) Conjunction() Token {
 	return c.conjunction
 }
 
+func (c CompoundExpr) Left() Expression {
+	return c.left
+}
+
+func (c CompoundExpr) Right() Expression {
+	return c.right
+}
+
 func (c CompoundExpr) String() string {
 	left := c.left.String()
 	if l, ok := c.left.(CompoundExpr); ok {

pkg/log/logger.go

@@ -25,6 +25,7 @@ const (
 	PrefixLicense          = "license"
 	PrefixVulnerabilityDB  = "vulndb"
 	PrefixJavaDB           = "javadb"
+	PrefixSPDX             = "spdx"
 )
 
 // Logger is an alias of slog.Logger