perf: parallelize trust verification and default to mozilla root store (#183)

* perf: parallelize trust verification and default to mozilla root store

On macOS, x509.Certificate.Verify with the system cert pool calls
SecTrustEvaluateWithError — a blocking syscall that can take seconds
per certificate for OCSP/CRL checks. With large certificate stores
(2500+ certs), scan+export operations hung indefinitely because every
certificate was verified sequentially against the system trust store.

Three changes fix this:

1. Default TrustStore from "system" to "mozilla". The embedded Mozilla
   root pool uses pure-Go verification — no syscalls, no network I/O.
   This alone eliminates the hang for the common case.

2. Parallelize trust verification in ScanSummary, dump-certs, and
   countAIAUnresolvedIssuers. All mozilla checks fire concurrently
   via goroutines, then only certs that mozilla didn't trust fall
   through to the (slower) system trust check.

3. Add TrustStore label to VerifyChainTrustInput and debug-log every
   trust verification call with subject, store name, and result.
   This makes future performance diagnosis trivial with -l debug.

Before: scan of ~2500 certs hung indefinitely (>10 minutes, killed)
After:  same scan completes in ~45 seconds

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* perf: bound system trust concurrency, fix nil panic, lint fixes

- Add semaphore (runtime.NumCPU) for system trust goroutines to avoid
  overwhelming macOS SecTrust with unbounded concurrent syscalls
- Guard VerifyChainTrust against nil cert before debug logging
- Retry bundle export with system trust store when mozilla fails, so
  certs trusted only by the host OS (corporate roots) export without
  requiring --force
- Fix golangci-lint modernize: use atomic.Int32 in verify_test.go

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: preserve export system trust retry errors

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: danielewood

CHANGELOG.md

@@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- **Breaking:** Default `TrustStore` in `DefaultOptions()` changed from `"system"` to `"mozilla"` — pure-Go Mozilla root verification is used by default instead of macOS `SecTrustEvaluateWithError` syscalls, eliminating multi-minute hangs on large certificate stores
+- Parallelize trust verification in scan summary, dump-certs, and AIA resolution — mozilla checks run concurrently, system checks only run for certs mozilla didn't trust
+- Add `TrustStore` label to `VerifyChainTrustInput` and debug-log every trust verification call with subject, store, and result
 - Normalize all exported private key PEM output (`.key`, K8s `tls.key`, YAML `key`) to PKCS#8 (`PRIVATE KEY`) regardless of input format ([#167])
 - Bundle export warns when Kubernetes TLS secret contains an unencrypted private key alongside encrypted outputs ([#167])
 - Use browser Web Crypto API for PBKDF2 key derivation in WASM builds to avoid blocking the main thread during encrypted key export ([#167])

README.md

@@ -403,9 +403,9 @@ if certkit.CertExpiresWithin(cert, 30*24*time.Hour) {
     // cert expires within 30 days
 }
 
-// Build verified chains (library defaults to system trust store)
+// Build verified chains (library defaults to mozilla trust store)
 opts := certkit.DefaultOptions()
-opts.TrustStore = "mozilla" // override the default system trust store if needed
+opts.TrustStore = "system" // override the default mozilla trust store if needed
 bundle, _ := certkit.Bundle(ctx, certkit.BundleInput{Leaf: leaf, Options: opts})
 
 // Generate keys

bundle.go

@@ -13,6 +13,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"runtime"
 	"slices"
 	"strings"
 	"sync"
@@ -337,6 +338,8 @@ type VerifyChainTrustInput struct {
 	Cert          *x509.Certificate
 	Roots         *x509.CertPool
 	Intermediates *x509.CertPool
+	// TrustStore is an optional label for debug logging (e.g. "mozilla", "system").
+	TrustStore string
 }
 
 // CheckTrustAnchorsInput holds parameters for CheckTrustAnchors.
@@ -366,8 +369,18 @@ type CheckTrustAnchorsResult struct {
 // invalid at the leaf's issuance time. This is an uncommon edge case in
 // practice (intermediates outlive the leaves they sign).
 func VerifyChainTrust(input VerifyChainTrustInput) bool {
+	if input.Cert == nil {
+		return false
+	}
+	store := input.TrustStore
+	if store == "" {
+		store = "unknown"
+	}
+	slog.Debug("verifying chain trust", "subject", input.Cert.Subject.CommonName, "store", store)
 	chains, err := verifyChainTrustChains(input)
-	return err == nil && len(chains) > 0
+	trusted := err == nil && len(chains) > 0
+	slog.Debug("chain trust result", "subject", input.Cert.Subject.CommonName, "store", store, "trusted", trusted)
+	return trusted
 }
 
 func verifyChainTrustChains(input VerifyChainTrustInput) ([][]*x509.Certificate, error) {
@@ -413,6 +426,7 @@ func CheckTrustAnchors(input CheckTrustAnchorsInput) CheckTrustAnchorsResult {
 		Cert:          input.Cert,
 		Roots:         mozillaPool,
 		Intermediates: input.Intermediates,
+		TrustStore:    "mozilla",
 	}) {
 		result.Anchors = append(result.Anchors, "mozilla")
 	}
@@ -422,13 +436,15 @@ func CheckTrustAnchors(input CheckTrustAnchorsInput) CheckTrustAnchorsResult {
 		Cert:          input.Cert,
 		Roots:         systemPool,
 		Intermediates: input.Intermediates,
+		TrustStore:    "system",
 	}) {
 		result.Anchors = append(result.Anchors, "system")
 	}
 	if input.FileRoots != nil && VerifyChainTrust(VerifyChainTrustInput{
 		Cert:          input.Cert,
 		Roots:         input.FileRoots,
 		Intermediates: input.Intermediates,
+		TrustStore:    "file",
 	}) {
 		result.Anchors = append(result.Anchors, "file")
 	}
@@ -496,7 +512,7 @@ func DefaultOptions() BundleOptions {
 		AIATimeout:       2 * time.Second,
 		AIAMaxDepth:      5,
 		AIAMaxTotalCerts: defaultAIAMaxTotalCerts,
-		TrustStore:       "system",
+		TrustStore:       "mozilla",
 		Verify:           true,
 		MaxIntermediates: defaultBundleMaxIntermediates,
 	}
@@ -705,25 +721,64 @@ func countAIAUnresolvedIssuers(certs []*x509.Certificate, roots *x509.CertPool)
 		}
 	}
 
-	unresolved := 0
-	for _, cert := range certs {
+	// Identify candidates that need trust verification.
+	type candidate struct {
+		idx  int
+		cert *x509.Certificate
+	}
+	var candidates []candidate
+	skipFlags := make([]bool, len(certs))
+	for i, cert := range certs {
 		if cert == nil {
+			skipFlags[i] = true
 			continue
 		}
 		if len(cert.IssuingCertificateURL) == 0 {
+			skipFlags[i] = true
 			continue
 		}
 		if IsMozillaRoot(cert) {
+			skipFlags[i] = true
 			continue
 		}
 		if bytes.Equal(cert.RawSubject, cert.RawIssuer) {
+			skipFlags[i] = true
+			continue
+		}
+		if roots != nil {
+			candidates = append(candidates, candidate{idx: i, cert: cert})
+		}
+	}
+
+	// Verify trust for all candidates concurrently. Bounded to NumCPU
+	// because system trust checks on macOS block in SecTrustEvaluateWithError.
+	trusted := make([]bool, len(certs))
+	if len(candidates) > 0 {
+		var wg sync.WaitGroup
+		sem := make(chan struct{}, runtime.NumCPU())
+		for _, c := range candidates {
+			wg.Add(1)
+			sem <- struct{}{}
+			go func(idx int, cert *x509.Certificate) {
+				defer wg.Done()
+				defer func() { <-sem }()
+				trusted[idx] = VerifyChainTrust(VerifyChainTrustInput{
+					Cert:          cert,
+					Roots:         roots,
+					Intermediates: intermediates,
+					TrustStore:    "aia-resolve",
+				})
+			}(c.idx, c.cert)
+		}
+		wg.Wait()
+	}
+
+	unresolved := 0
+	for i, cert := range certs {
+		if skipFlags[i] {
 			continue
 		}
-		if roots != nil && VerifyChainTrust(VerifyChainTrustInput{
-			Cert:          cert,
-			Roots:         roots,
-			Intermediates: intermediates,
-		}) {
+		if trusted[i] {
 			continue
 		}
 		if hasIssuerInSet(cert, certs) {

bundle_test.go

@@ -159,8 +159,8 @@ func TestDefaultOptions(t *testing.T) {
 	if opts.AIAMaxTotalCerts != defaultAIAMaxTotalCerts {
 		t.Fatalf("AIAMaxTotalCerts = %d, want %d", opts.AIAMaxTotalCerts, defaultAIAMaxTotalCerts)
 	}
-	if opts.TrustStore != "system" {
-		t.Fatalf("TrustStore = %q, want system", opts.TrustStore)
+	if opts.TrustStore != "mozilla" {
+		t.Fatalf("TrustStore = %q, want mozilla", opts.TrustStore)
 	}
 	if !opts.Verify {
 		t.Fatal("Verify = false, want true")

cmd/certkit/scan.go

@@ -12,8 +12,10 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
+	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/sensiblebit/certkit"
@@ -279,10 +281,61 @@ func runScan(cmd *cobra.Command, args []string) error {
 			}
 			intermediatePool := store.IntermediatePool()
 
+			// Pre-compute trust status concurrently. Mozilla checks are pure Go
+			// and fast; system checks hit macOS SecTrust which is slow. Run
+			// mozilla first, then only check system for untrusted remainders.
+			type dumpTrust struct {
+				mozilla bool
+				system  bool
+			}
+			now := time.Now()
+			trustStatus := make([]dumpTrust, len(certs))
+			if !scanForceExport && (trustPools.Mozilla != nil || trustPools.System != nil) {
+				var wg sync.WaitGroup
+				if trustPools.Mozilla != nil {
+					for i, c := range certs {
+						if !allowExpired && now.After(c.Cert.NotAfter) {
+							continue
+						}
+						wg.Add(1)
+						go func(idx int, cert *x509.Certificate) {
+							defer wg.Done()
+							trustStatus[idx].mozilla = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
+								Cert:          cert,
+								Roots:         trustPools.Mozilla,
+								Intermediates: intermediatePool,
+								TrustStore:    "mozilla",
+							})
+						}(i, c.Cert)
+					}
+					wg.Wait()
+				}
+				if trustPools.System != nil {
+					sem := make(chan struct{}, runtime.NumCPU())
+					for i, c := range certs {
+						if (!allowExpired && now.After(c.Cert.NotAfter)) || trustStatus[i].mozilla {
+							continue
+						}
+						wg.Add(1)
+						sem <- struct{}{}
+						go func(idx int, cert *x509.Certificate) {
+							defer wg.Done()
+							defer func() { <-sem }()
+							trustStatus[idx].system = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
+								Cert:          cert,
+								Roots:         trustPools.System,
+								Intermediates: intermediatePool,
+								TrustStore:    "system",
+							})
+						}(i, c.Cert)
+					}
+					wg.Wait()
+				}
+			}
+
 			var data []byte
 			var count, skipped int
-			now := time.Now()
-			for _, c := range certs {
+			for i, c := range certs {
 				cert := c.Cert
 
 				// Skip expired certificates unless --allow-expired is set
@@ -294,17 +347,7 @@ func runScan(cmd *cobra.Command, args []string) error {
 
 				// Validate chain unless --force is set (uses same logic as summary)
 				if !scanForceExport {
-					mozillaTrusted := trustPools.Mozilla != nil && certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
-						Cert:          cert,
-						Roots:         trustPools.Mozilla,
-						Intermediates: intermediatePool,
-					})
-					systemTrusted := trustPools.System != nil && certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
-						Cert:          cert,
-						Roots:         trustPools.System,
-						Intermediates: intermediatePool,
-					})
-					if !mozillaTrusted && !systemTrusted && (trustPools.Mozilla != nil || trustPools.System != nil) {
+					if !trustStatus[i].mozilla && !trustStatus[i].system && (trustPools.Mozilla != nil || trustPools.System != nil) {
 						slog.Debug("skipping untrusted certificate", "subject", cert.Subject)
 						skipped++
 						continue

cmd/wasm/main.go

@@ -327,7 +327,7 @@ func getState(_ js.Value, _ []js.Value) any {
 		expired := now.After(rec.NotAfter)
 		trusted := false
 		if roots != nil {
-			trusted = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{Cert: rec.Cert, Roots: roots, Intermediates: intermediatePool})
+			trusted = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{Cert: rec.Cert, Roots: roots, Intermediates: intermediatePool, TrustStore: "mozilla"})
 		}
 
 		serial := ""

internal/certstore/memstore.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"log/slog"
 	"maps"
+	"runtime"
 	"slices"
 	"sort"
 	"sync"
@@ -398,19 +399,61 @@ func (s *MemStore) ScanSummary(input ScanSummaryInput) ScanSummary {
 		}
 	}
 
+	// Pre-compute trust status for all non-expired certs concurrently.
+	// Mozilla checks are pure Go and fast; system checks hit the macOS
+	// Security framework (SecTrustEvaluateWithError) which is slow.
+	// Run mozilla first, then only check system for certs mozilla didn't trust.
+	type trustStatus struct {
+		mozilla bool
+		system  bool
+	}
 	now := time.Now()
-	for _, rec := range certs {
-		expired := now.After(rec.NotAfter)
-		mozillaTrusted := false
-		systemTrusted := false
-		if !expired {
-			if input.MozillaPool != nil {
-				mozillaTrusted = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{Cert: rec.Cert, Roots: input.MozillaPool, Intermediates: intermediatePool})
+	trustResults := make([]trustStatus, len(certs))
+	var wg sync.WaitGroup
+	if input.MozillaPool != nil {
+		for i, rec := range certs {
+			if now.After(rec.NotAfter) {
+				continue
 			}
-			if input.SystemPool != nil {
-				systemTrusted = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{Cert: rec.Cert, Roots: input.SystemPool, Intermediates: intermediatePool})
+			wg.Add(1)
+			go func(idx int, cert *x509.Certificate) {
+				defer wg.Done()
+				trustResults[idx].mozilla = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
+					Cert:          cert,
+					Roots:         input.MozillaPool,
+					Intermediates: intermediatePool,
+					TrustStore:    "mozilla",
+				})
+			}(i, rec.Cert)
+		}
+		wg.Wait()
+	}
+	if input.SystemPool != nil {
+		sem := make(chan struct{}, runtime.NumCPU())
+		for i, rec := range certs {
+			if now.After(rec.NotAfter) || trustResults[i].mozilla {
+				continue
 			}
+			wg.Add(1)
+			sem <- struct{}{}
+			go func(idx int, cert *x509.Certificate) {
+				defer wg.Done()
+				defer func() { <-sem }()
+				trustResults[idx].system = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
+					Cert:          cert,
+					Roots:         input.SystemPool,
+					Intermediates: intermediatePool,
+					TrustStore:    "system",
+				})
+			}(i, rec.Cert)
 		}
+		wg.Wait()
+	}
+
+	for i, rec := range certs {
+		expired := now.After(rec.NotAfter)
+		mozillaTrusted := trustResults[i].mozilla
+		systemTrusted := trustResults[i].system
 
 		switch rec.CertType {
 		case "root":