feat(helm): add hash cache persistence support

.github/workflows/validate-helm-examples.yml

@@ -115,7 +115,7 @@ jobs:
       matrix:
         example-group:
           - secret_manager_group_1: ["akeyless", "cyberarksaas", "hashicorpvault", "azurekeyvault"]
-          - secret_manager_group_2: ["delinea", "gcpsecretmanager", "awssecretsmanager"]  # TODO: add fetch-only
+          - secret_manager_group_2: ["gcpsecretmanager", "awssecretsmanager"]  # TODO: add fetch-only, delinea (credentials expired)
           # - consumers: ["k8s_incluster", "k8s_kubeconfigfile", "gitlabci"]
 
     steps:
@@ -171,6 +171,35 @@ jobs:
           kubectl create secret generic gcp-key-secret --namespace ggscout-test --from-file=gcp_key.json
           echo "✅ Secret gcp-key-secret created with GCP credentials"
 
+      # Verify GitGuardian API credentials before deploying
+      - name: Verify GitGuardian API credentials
+        env:
+          GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
+          GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+        run: |
+          echo "🔍 Testing GitGuardian API connectivity..."
+          echo "Endpoint: ${GITGUARDIAN_API_URL}"
+
+          # Test the ping endpoint (POST) which requires the same auth
+          HTTP_STATUS=$(curl -s -o /tmp/gg_response.json -w "%{http_code}" \
+            -X POST \
+            -H "Authorization: Token ${GITGUARDIAN_API_KEY}" \
+            -H "Content-Type: application/json" \
+            -d '[]' \
+            "${GITGUARDIAN_API_URL%/}/nhi/ping")
+
+          echo "HTTP Status: $HTTP_STATUS"
+          echo "Response body:"
+          cat /tmp/gg_response.json
+          echo ""
+
+          if [ "$HTTP_STATUS" -ge 400 ]; then
+            echo "❌ GitGuardian API authentication failed (HTTP $HTTP_STATUS)"
+            echo "Check that GITGUARDIAN_API_KEY and GITGUARDIAN_API_URL secrets are correct"
+            exit 1
+          fi
+          echo "✅ GitGuardian API credentials verified"
+
       # Deploy and validate each example in the cluster
       - name: Deploy and validate examples
         env:

README.md

@@ -86,6 +86,41 @@ To apply the secrets to your cluster/namespace, run the following command: `kube
 
 Other examples can be found in [charts/ggscout/examples](charts/ggscout/examples).
 
+## Hash Cache Persistence
+
+ggscout uses scrypt to hash secrets before sending them to GitGuardian. By default, hashes are recomputed from scratch on every CronJob run. For large vaults this can be slow.
+
+Enabling persistence creates a PVC that stores the hash cache across runs, so only new or changed secrets need to be hashed:
+
+```yaml
+persistence:
+  enabled: true
+```
+
+On the first run the cache is empty (all misses). On subsequent runs, unchanged secrets are served from cache (hits), skipping expensive scrypt computation.
+
+### Configuration
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `persistence.enabled` | Enable persistent hash cache | `false` |
+| `persistence.storageClassName` | StorageClass name (empty = cluster default) | `""` |
+| `persistence.size` | PVC size (64 bytes/entry, 100Mi handles ~1.6M secrets) | `100Mi` |
+| `persistence.existingClaim` | Use an existing PVC instead of creating one | `""` |
+
+### Using an existing PVC
+
+If you already have a PVC you want to reuse:
+
+```yaml
+persistence:
+  enabled: true
+  existingClaim: my-existing-pvc
+```
+
+> [!NOTE]
+> The cache is only mounted on the inventory CronJob (fetch/fetch-and-send). The ping and sync CronJobs do not perform hashing and are unaffected.
+
 > [!IMPORTANT]
 > If you want to only fetch the identities without sending them, please see this [example](charts/ggscout/examples/fetch-only)
 

charts/ggscout/templates/_cronjob.tpl

@@ -25,8 +25,11 @@ spec:
             {{- end }}
         spec:
           {{- include "ggscout.securityContext" $ | indent 10 }}
-          {{- if or $.Values.caBundle.certs $.Values.caBundle.existingSecret }}
+          {{- $needsCaInit := or $.Values.caBundle.certs $.Values.caBundle.existingSecret }}
+          {{- $needsCacheTTL := and .use_cache_pvc .Values.persistence.enabled (gt (int .Values.persistence.cacheTTLSeconds) 0) }}
+          {{- if or $needsCaInit $needsCacheTTL }}
           initContainers:
+            {{- if $needsCaInit }}
             - name: init-ca
               {{- include "ggscout.containerSecurityContext" . | indent 14 }}
               image: {{ include "ggscout.caBundle.image" . }}
@@ -47,6 +50,32 @@ spec:
                 - name: ssl-ca-bundle
                   mountPath: /etc/ssl/ca-bundle
                   readOnly: true
+            {{- end }}
+            {{- if $needsCacheTTL }}
+            - name: cache-ttl-check
+              {{- include "ggscout.containerSecurityContext" . | indent 14 }}
+              image: {{ include "ggscout.caBundle.image" . }}
+              imagePullPolicy: {{ .Values.imagePullPolicy }}
+              command: ["/bin/sh", "-c"]
+              args:
+                - |
+                  CACHE="/var/cache/ggscout/hashcache"
+                  TTL={{ .Values.persistence.cacheTTLSeconds }}
+                  if [ -f "$CACHE" ]; then
+                    AGE=$(( $(date +%s) - $(stat -c %Y "$CACHE") ))
+                    if [ "$AGE" -gt "$TTL" ]; then
+                      echo "Cache is ${AGE}s old (TTL=${TTL}s), flushing stale cache"
+                      rm -f "$CACHE"
+                    else
+                      echo "Cache is ${AGE}s old (TTL=${TTL}s), keeping"
+                    fi
+                  else
+                    echo "No cache file found, cold run"
+                  fi
+              volumeMounts:
+                - name: hashcache
+                  mountPath: /var/cache/ggscout
+            {{- end }}
           {{- end }}
           containers:
             - name: {{ .Chart.Name }}
@@ -67,6 +96,10 @@ spec:
                 - name: SSL_CERT_FILE
                   value: /etc/ssl/custom-certs/ca-bundle.crt
               {{- end }}
+              {{- if and .use_cache_pvc .Values.persistence.enabled }}
+                - name: HASH_CACHE_PATH
+                  value: /var/cache/ggscout/hashcache
+              {{- end }}
               {{- range .Values.env }}
                 - {{ toJson . }}
               {{- end }}
@@ -76,6 +109,10 @@ spec:
                 - name: ssl-custom-certs
                   mountPath: /etc/ssl/custom-certs
                   readOnly: true
+                {{- if and .use_cache_pvc .Values.persistence.enabled }}
+                - name: hashcache
+                  mountPath: /var/cache/ggscout
+                {{- end }}
                 {{- range .Values.volumeMounts }}
                 - {{ toJson . }}
                 {{- end }}
@@ -113,6 +150,11 @@ spec:
                     path: ca-bundle.crt
                 {{- end }}
             {{- end }}
+            {{- if and .use_cache_pvc .Values.persistence.enabled }}
+            - name: hashcache
+              persistentVolumeClaim:
+                claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{ else }}{{ include "ggscout.fullname" . }}-hashcache{{ end }}
+            {{- end }}
             {{- range .Values.volumes }}
             - {{ toJson . }}
             {{- end }}

charts/ggscout/templates/cronjob_inventory.yaml

@@ -6,5 +6,5 @@
 {{ $command = "fetch-and-send" -}}
 {{- end }}
 
-{{ include "ggscout.cronjob" (merge (dict "cronjob_name" "inventory" "command" $command "schedule" .Values.inventory.jobs.fetch.schedule "backOffLimit" .Values.inventory.jobs.fetch.backOffLimit "ttlSecondsAfterFinished" .Values.inventory.jobs.fetch.ttlSecondsAfterFinished ) .) -}}
+{{ include "ggscout.cronjob" (merge (dict "cronjob_name" "inventory" "command" $command "schedule" .Values.inventory.jobs.fetch.schedule "backOffLimit" .Values.inventory.jobs.fetch.backOffLimit "ttlSecondsAfterFinished" .Values.inventory.jobs.fetch.ttlSecondsAfterFinished "use_cache_pvc" true ) .) -}}
 {{- end }}

charts/ggscout/templates/persistentvolumeclaim.yaml

@@ -0,0 +1,17 @@
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) .Values.inventory.jobs.fetch.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "ggscout.fullname" . }}-hashcache
+  labels:
+    {{- include "ggscout.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+  {{- if .Values.persistence.storageClassName }}
+  storageClassName: {{ .Values.persistence.storageClassName }}
+  {{- end }}
+{{- end }}

charts/ggscout/tests/persistence_test.yaml

@@ -0,0 +1,172 @@
+---
+# Test docs: https://github.com/helm-unittest/helm-unittest/blob/main/DOCUMENT.md
+
+# --- PVC tests ---
+suite: test persistence
+templates:
+- persistentvolumeclaim.yaml
+set:
+  inventory.jobs.fetch.enabled: true
+  inventory.config.gitguardian.api_token: "foobar"
+  inventory.config.gitguardian.endpoint: "https://some-url.com"
+tests:
+- it: should not create PVC when persistence is disabled (default)
+  asserts:
+  - hasDocuments:
+      count: 0
+
+- it: should create PVC when persistence is enabled
+  set:
+    persistence.enabled: true
+  asserts:
+  - isKind:
+      of: PersistentVolumeClaim
+  - matchRegex:
+      path: metadata.name
+      pattern: .*-hashcache$
+  - equal:
+      path: spec.accessModes[0]
+      value: ReadWriteOnce
+  - equal:
+      path: spec.resources.requests.storage
+      value: 100Mi
+
+- it: should apply custom storageClassName
+  set:
+    persistence.enabled: true
+    persistence.storageClassName: fast-ssd
+  asserts:
+  - equal:
+      path: spec.storageClassName
+      value: fast-ssd
+
+- it: should not create PVC when existingClaim is set
+  set:
+    persistence.enabled: true
+    persistence.existingClaim: my-existing-pvc
+  asserts:
+  - hasDocuments:
+      count: 0
+
+- it: should not create PVC when inventory fetch is disabled
+  set:
+    persistence.enabled: true
+    inventory.jobs.fetch.enabled: false
+  asserts:
+  - hasDocuments:
+      count: 0
+---
+# --- Inventory CronJob persistence tests ---
+suite: test persistence in inventory cronjob
+templates:
+- cronjob_inventory.yaml
+set:
+  inventory.jobs.fetch.enabled: true
+  inventory.config.gitguardian.api_token: "foobar"
+  inventory.config.gitguardian.endpoint: "https://some-url.com"
+tests:
+- it: should inject HASH_CACHE_PATH env var when persistence is enabled
+  set:
+    persistence.enabled: true
+  asserts:
+  - contains:
+      path: spec.jobTemplate.spec.template.spec.containers[0].env
+      content:
+        name: HASH_CACHE_PATH
+        value: /var/cache/ggscout/hashcache
+
+- it: should not inject HASH_CACHE_PATH env var when persistence is disabled
+  asserts:
+  - notContains:
+      path: spec.jobTemplate.spec.template.spec.containers[0].env
+      content:
+        name: HASH_CACHE_PATH
+        value: /var/cache/ggscout/hashcache
+
+- it: should add hashcache volumeMount when persistence is enabled
+  set:
+    persistence.enabled: true
+  asserts:
+  - contains:
+      path: spec.jobTemplate.spec.template.spec.containers[0].volumeMounts
+      content:
+        name: hashcache
+        mountPath: /var/cache/ggscout
+
+- it: should add hashcache volume with generated claimName when persistence is enabled
+  set:
+    persistence.enabled: true
+  asserts:
+  - contains:
+      path: spec.jobTemplate.spec.template.spec.volumes
+      content:
+        name: hashcache
+        persistentVolumeClaim:
+          claimName: RELEASE-NAME-ggscout-hashcache
+
+- it: should use existingClaim as claimName when specified
+  set:
+    persistence.enabled: true
+    persistence.existingClaim: my-existing-pvc
+  asserts:
+  - contains:
+      path: spec.jobTemplate.spec.template.spec.volumes
+      content:
+        name: hashcache
+        persistentVolumeClaim:
+          claimName: my-existing-pvc
+
+- it: should not add hashcache volume or mount when persistence is disabled
+  asserts:
+  - notContains:
+      path: spec.jobTemplate.spec.template.spec.containers[0].volumeMounts
+      content:
+        name: hashcache
+        mountPath: /var/cache/ggscout
+  - notContains:
+      path: spec.jobTemplate.spec.template.spec.volumes
+      content:
+        name: hashcache
+      any: true
+
+- it: should render cache-ttl-check initContainer when persistence is enabled with default TTL
+  set:
+    persistence.enabled: true
+  asserts:
+  - contains:
+      path: spec.jobTemplate.spec.template.spec.initContainers
+      content:
+        name: cache-ttl-check
+      any: true
+
+- it: should mount hashcache volume in cache-ttl-check initContainer
+  set:
+    persistence.enabled: true
+  asserts:
+  - contains:
+      path: spec.jobTemplate.spec.template.spec.initContainers[0].volumeMounts
+      content:
+        name: hashcache
+        mountPath: /var/cache/ggscout
+
+- it: should include TTL value in cache-ttl-check args
+  set:
+    persistence.enabled: true
+    persistence.cacheTTLSeconds: 3600
+  asserts:
+  - matchRegex:
+      path: spec.jobTemplate.spec.template.spec.initContainers[0].args[0]
+      pattern: "TTL=3600"
+
+- it: should not render cache-ttl-check initContainer when cacheTTLSeconds is 0
+  set:
+    persistence.enabled: true
+    persistence.cacheTTLSeconds: 0
+  asserts:
+  - isNull:
+      path: spec.jobTemplate.spec.template.spec.initContainers
+
+- it: should not render cache-ttl-check initContainer when persistence is disabled
+  asserts:
+  - isNull:
+      path: spec.jobTemplate.spec.template.spec.initContainers