Secret Scanner Comparison Benchmark #33
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Secret Scanner Comparison Benchmark | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # Run weekly on Sundays at 02:00 UTC | |
| - cron: '0 2 * * 0' | |
| permissions: | |
| contents: read | |
| jobs: | |
| trufflehog: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| count: ${{ steps.count.outputs.findings }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Run TruffleHog OSS | |
| run: | | |
| # Use TruffleHog directly to capture JSON output properly | |
| docker run --rm -v "$(pwd):/pwd" \ | |
| trufflesecurity/trufflehog:latest filesystem /pwd \ | |
| --json --only-verified > trufflehog_output.json || true | |
| continue-on-error: true | |
| id: trufflehog | |
| - name: Count TruffleHog findings | |
| id: count | |
| run: | | |
| # Count findings from TruffleHog output (it outputs JSON lines) | |
| count=0 | |
| if [ -f trufflehog_output.json ]; then | |
| count=$(cat trufflehog_output.json | \ | |
| grep -c "\"verified\":" || echo "0") | |
| fi | |
| echo "findings=$count" >> $GITHUB_OUTPUT | |
| echo "TruffleHog found $count verified secrets" | |
| git-secrets: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| count: ${{ steps.count.outputs.findings }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Install git-secrets | |
| run: | | |
| git clone https://github.com/awslabs/git-secrets.git | |
| cd git-secrets | |
| sudo make install | |
| - name: Initialize git-secrets | |
| run: | | |
| git secrets --register-aws | |
| git secrets --install | |
| - name: Run git-secrets scan | |
| id: scan | |
| run: | | |
| set +e | |
| git secrets --scan > git_secrets_output.txt 2>&1 | |
| exit_code=$? | |
| echo "exit_code=$exit_code" >> $GITHUB_OUTPUT | |
| cat git_secrets_output.txt | |
| continue-on-error: true | |
| - name: Count git-secrets findings | |
| id: count | |
| run: | | |
| count=0 | |
| if [ -f git_secrets_output.txt ]; then | |
| # Count lines that indicate findings (exclude headers/empty lines) | |
| count=$(grep -c ".*:.*:.*" git_secrets_output.txt || echo "0") | |
| fi | |
| echo "findings=$count" >> $GITHUB_OUTPUT | |
| echo "git-secrets found $count secrets" | |
| detect-secrets: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| count: ${{ steps.count.outputs.findings }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.14' | |
| - name: Install detect-secrets | |
| run: | | |
| pip install detect-secrets | |
| - name: Run detect-secrets scan | |
| run: | | |
| detect-secrets scan --all-files > detect_secrets_output.json | |
| continue-on-error: true | |
| - name: Count detect-secrets findings | |
| id: count | |
| run: | | |
| count=0 | |
| if [ -f detect_secrets_output.json ]; then | |
| # Count the number of potential secrets found | |
| count=$(jq '.results | to_entries | map(.value | length) | \ | |
| add // 0' detect_secrets_output.json) | |
| fi | |
| echo "findings=$count" >> $GITHUB_OUTPUT | |
| echo "detect-secrets found $count potential secrets" | |
| gitleaks: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| count: ${{ steps.count.outputs.findings }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install gitleaks | |
| run: | | |
| wget -O gitleaks.tar.gz \ | |
| https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz | |
| tar -xf gitleaks.tar.gz | |
| sudo mv gitleaks /usr/local/bin/ | |
| gitleaks version | |
| - name: Run gitleaks scan | |
| run: | | |
| gitleaks detect --source . --report-format json \ | |
| --report-path gitleaks_output.json --no-git || \ | |
| echo "Gitleaks scan completed" | |
| continue-on-error: true | |
| - name: Count gitleaks findings | |
| id: count | |
| run: | | |
| count=0 | |
| if [ -f gitleaks_output.json ]; then | |
| # Count findings in JSON output | |
| count=$(jq 'length' gitleaks_output.json 2>/dev/null || echo "0") | |
| fi | |
| echo "findings=$count" >> $GITHUB_OUTPUT | |
| echo "gitleaks found $count secrets" | |
| gittyleaks: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| count: ${{ steps.count.outputs.findings }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.14' | |
| - name: Install gittyleaks | |
| run: | | |
| pip install gittyleaks | |
| - name: Run gittyleaks scan | |
| run: | | |
| gittyleaks --find-anything > gittyleaks_output.txt 2>&1 | |
| continue-on-error: true | |
| - name: Count gittyleaks findings | |
| id: count | |
| run: | | |
| count=0 | |
| if [ -f gittyleaks_output.txt ]; then | |
| # Count lines that contain findings (exclude header/footer lines) | |
| count=$(grep ":" gittyleaks_output.txt | \ | |
| grep -v "Bot Detective" | grep -v "^---" | \ | |
| wc -l || echo "0") | |
| fi | |
| echo "findings=$count" >> $GITHUB_OUTPUT | |
| echo "gittyleaks found $count secrets" | |
| whispers: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| count: ${{ steps.count.outputs.findings }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.14' | |
| - name: Install whispers (with timeout handling) | |
| run: | | |
| # Try to install whispers with a timeout and fallback | |
| timeout 300 pip install whispers || echo "Failed to install whispers" | |
| continue-on-error: true | |
| - name: Run whispers scan | |
| run: | | |
| if command -v whispers >/dev/null 2>&1; then | |
| whispers . --output whispers_output.json --format json || \ | |
| echo "Whispers scan failed" | |
| else | |
| echo "Whispers not available, skipping scan" | |
| echo "[]" > whispers_output.json | |
| fi | |
| continue-on-error: true | |
| - name: Count whispers findings | |
| id: count | |
| run: | | |
| count=0 | |
| if [ -f whispers_output.json ]; then | |
| # Count findings in JSON output | |
| count=$(jq 'length' whispers_output.json 2>/dev/null || echo "0") | |
| fi | |
| echo "findings=$count" >> $GITHUB_OUTPUT | |
| echo "whispers found $count secrets" | |
| trufflehog3: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| count: ${{ steps.count.outputs.findings }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.14' | |
| - name: Install trufflehog3 | |
| run: | | |
| pip install trufflehog3 | |
| - name: Run trufflehog3 scan | |
| run: | | |
| trufflehog3 . --format json > trufflehog3_output.json 2>&1 || \ | |
| echo "TruffleHog3 scan completed with warnings" | |
| continue-on-error: true | |
| - name: Count trufflehog3 findings | |
| id: count | |
| run: | | |
| count=0 | |
| if [ -f trufflehog3_output.json ]; then | |
| # Count findings - each line that starts with '{' is a finding | |
| count=$(grep -c '^{' trufflehog3_output.json || echo "0") | |
| fi | |
| echo "findings=$count" >> $GITHUB_OUTPUT | |
| echo "trufflehog3 found $count secrets" | |
| summary: | |
| needs: [trufflehog, git-secrets, gitleaks, detect-secrets, gittyleaks, | |
| whispers, trufflehog3] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Create Summary Report | |
| run: | | |
| echo "# Secret Scanner Comparison Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Scanner | Secrets Found |" >> $GITHUB_STEP_SUMMARY | |
| echo "|---------|---------------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| TruffleHog | ${{ needs.trufflehog.outputs.count }} |" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| echo "| git-secrets | ${{ needs.git-secrets.outputs.count }} |" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| echo "| gitleaks | ${{ needs.gitleaks.outputs.count }} |" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| echo "| detect-secrets |" \ | |
| "${{ needs.detect-secrets.outputs.count }} |" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| echo "| gittyleaks | ${{ needs.gittyleaks.outputs.count }} |" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| echo "| whispers | ${{ needs.whispers.outputs.count }} |" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| echo "| trufflehog3 | ${{ needs.trufflehog3.outputs.count }} |" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Total unique scanning tools tested:** 7" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "_This benchmark helps understand the relative effectiveness" \ | |
| "of different secret scanning tools on the OWASP WrongSecrets" \ | |
| "repository._" >> $GITHUB_STEP_SUMMARY | |
| # Also output to console | |
| echo "=== Secret Scanner Comparison Results ===" | |
| echo "TruffleHog: ${{ needs.trufflehog.outputs.count }} secrets" | |
| echo "git-secrets: ${{ needs.git-secrets.outputs.count }} secrets" | |
| echo "gitleaks: ${{ needs.gitleaks.outputs.count }} secrets" | |
| echo "detect-secrets: ${{ needs.detect-secrets.outputs.count }} \ | |
| secrets" | |
| echo "gittyleaks: ${{ needs.gittyleaks.outputs.count }} secrets" | |
| echo "whispers: ${{ needs.whispers.outputs.count }} secrets" | |
| echo "trufflehog3: ${{ needs.trufflehog3.outputs.count }} secrets" |