Merge pull request #2643 from blacklanternsecurity/3.0-update

Dev -> 3.0 Sync

Author: TheTechromancer

.github/workflows/benchmark.yml

@@ -0,0 +1,166 @@
+name: Performance Benchmarks
+
+on:
+  pull_request:
+    paths:
+      - 'bbot/**/*.py'
+      - 'pyproject.toml'
+      - '.github/workflows/benchmark.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v5
+      with:
+        fetch-depth: 0  # Need full history for branch comparison
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.11"
+
+    - name: Install dependencies
+      run: |
+        pip install poetry
+        poetry install --with dev
+
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libmagic1
+
+    # Generate benchmark comparison report using our branch-based script
+    - name: Generate benchmark comparison report
+      run: |
+        poetry run python bbot/scripts/benchmark_report.py \
+          --base ${{ github.base_ref }} \
+          --current ${{ github.head_ref }} \
+          --output benchmark_report.md \
+          --keep-results
+      continue-on-error: true
+
+    # Upload benchmark results as artifacts
+    - name: Upload benchmark results
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-results
+        path: |
+          benchmark_report.md
+          base_benchmark_results.json
+          current_benchmark_results.json
+        retention-days: 30
+
+    # Comment on PR with benchmark results
+    - name: Comment benchmark results on PR
+      uses: actions/github-script@v8
+      with:
+        script: |
+          const fs = require('fs');
+          
+          try {
+            const report = fs.readFileSync('benchmark_report.md', 'utf8');
+            
+            // Find existing benchmark comment (with pagination)
+            const comments = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              per_page: 100, // Get more comments per page
+            });
+            
+            // Debug: log all comments to see what we're working with
+            console.log(`Found ${comments.data.length} comments on this PR`);
+            comments.data.forEach((comment, index) => {
+              console.log(`Comment ${index}: user=${comment.user.login}, body preview="${comment.body.substring(0, 100)}..."`);
+            });
+            
+            const existingComments = comments.data.filter(comment => 
+              comment.body.toLowerCase().includes('performance benchmark') &&
+              comment.user.login === 'github-actions[bot]'
+            );
+            
+            console.log(`Found ${existingComments.length} existing benchmark comments`);
+            
+            if (existingComments.length > 0) {
+              // Sort comments by creation date to find the most recent
+              const sortedComments = existingComments.sort((a, b) => 
+                new Date(b.created_at) - new Date(a.created_at)
+              );
+              
+              const mostRecentComment = sortedComments[0];
+              console.log(`Updating most recent benchmark comment: ${mostRecentComment.id} (created: ${mostRecentComment.created_at})`);
+              
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: mostRecentComment.id,
+                body: report
+              });
+              console.log('Updated existing benchmark comment');
+              
+              // Delete any older duplicate comments
+              if (existingComments.length > 1) {
+                console.log(`Deleting ${existingComments.length - 1} older duplicate comments`);
+                for (let i = 1; i < sortedComments.length; i++) {
+                  const commentToDelete = sortedComments[i];
+                  console.log(`Attempting to delete comment ${commentToDelete.id} (created: ${commentToDelete.created_at})`);
+                  
+                  try {
+                    await github.rest.issues.deleteComment({
+                      owner: context.repo.owner,
+                      repo: context.repo.repo,
+                      comment_id: commentToDelete.id
+                    });
+                    console.log(`Successfully deleted duplicate comment: ${commentToDelete.id}`);
+                  } catch (error) {
+                    console.error(`Failed to delete comment ${commentToDelete.id}: ${error.message}`);
+                    console.error(`Error details:`, error);
+                  }
+                }
+              }
+            } else {
+              // Create new comment
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: report
+              });
+              console.log('Created new benchmark comment');
+            }
+          } catch (error) {
+            console.error('Failed to post benchmark results:', error);
+            
+            // Post a fallback comment
+            const fallbackMessage = [
+              '## Performance Benchmark Report',
+              '',
+              '> ⚠️ **Failed to generate detailed benchmark comparison**',
+              '> ',
+              '> The benchmark comparison failed to run. This might be because:',
+              '> - Benchmark tests don\'t exist on the base branch yet',
+              '> - Dependencies are missing', 
+              '> - Test execution failed',
+              '> ',
+              '> Please check the [workflow logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.',
+              '> ',
+              '> 📁 Benchmark artifacts may be available for download from the workflow run.'
+            ].join('\\n');
+            
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: fallbackMessage
+            });
+          } 

.github/workflows/codeql.yml

@@ -55,7 +55,7 @@ jobs:
         # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
 
     # Add any setup steps before running the `github/codeql-action/init` action.
     # This includes steps like installing compilers or runtimes (`actions/setup-node`

.github/workflows/distro_tests.yml

@@ -16,7 +16,7 @@ jobs:
       matrix:
         os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "kalilinux/kali-rolling", "parrotsec/security"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Install Python and Poetry
         run: |
           if [ -f /etc/os-release ]; then

.github/workflows/docs_updater.yml

@@ -9,12 +9,12 @@ jobs:
   update_docs:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           ref: dev  # Checkout the dev branch
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.x"
       - name: Install dependencies

.github/workflows/tests.yml

@@ -19,9 +19,9 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
       - name: Set Python Version Environment Variable
@@ -55,11 +55,11 @@ jobs:
     if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
     continue-on-error: true
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.x"
       - name: Install dependencies
@@ -72,7 +72,7 @@ jobs:
         run: python -m build
       - name: Publish Pypi package
         if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@release/v1.13
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
       - name: Get BBOT version
@@ -107,10 +107,10 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev')
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.11"
       - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
@@ -151,7 +151,7 @@ jobs:
   #   runs-on: ubuntu-latest
   #   if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
   #   steps:
-  #     - uses: actions/checkout@v4
+  #     - uses: actions/checkout@v5
   #       with:
   #         ref: ${{ github.head_ref }}
   #         fetch-depth: 0 # Fetch all history for all tags and branches

.github/workflows/version_updater.yml

@@ -9,13 +9,13 @@ jobs:
   update-nuclei-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: dev
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.x'
       - name: Install dependencies
@@ -36,12 +36,12 @@ jobs:
       - name: Get current version
         id: get-current-version
         run: |
-          version=$(grep -m 1 -oP '(?<=version": ")[^"]*' bbot/modules/deadly/nuclei.py)
+          version=$(grep -m 1 -oP '(?<=version": ")[^"]*' bbot/modules/nuclei.py)
           echo "current_version=$version" >> $GITHUB_ENV
       - name: Update version
         id: update-version
         if: env.latest_version != env.current_version
-        run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/deadly/nuclei.py"
+        run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/nuclei.py"
       - name: Create pull request to update the version
         if: steps.update-version.outcome == 'success'
         uses: peter-evans/create-pull-request@v7
@@ -50,7 +50,7 @@ jobs:
           commit-message: "Update nuclei"
           title: "Update nuclei to ${{ env.latest_version }}"
           body: |
-            This PR uses https://api.github.com/repos/projectdiscovery/nuclei/releases/latest to obtain the latest version of nuclei and update the version in bbot/modules/deadly/nuclei.py."
+            This PR uses https://api.github.com/repos/projectdiscovery/nuclei/releases/latest to obtain the latest version of nuclei and update the version in bbot/modules/nuclei.py."
 
             # Release notes:
             ${{ env.release_notes }}
@@ -61,13 +61,13 @@ jobs:
   update-trufflehog-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: dev
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.x'
       - name: Install dependencies

bbot/core/engine.py

@@ -636,7 +636,7 @@ async def finished_tasks(self, tasks, timeout=None):
         """
         if tasks:
             try:
-                done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
+                done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)
                 return done
             except BaseException as e:
                 if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):

bbot/core/helpers/bloom.py

@@ -1,6 +1,7 @@
 import os
 import mmh3
 import mmap
+import xxhash
 
 
 class BloomFilter:
@@ -55,14 +56,12 @@ def _hashes(self, item):
             if not isinstance(item, str):
                 item = str(item)
             item = item.encode("utf-8")
-        return [abs(hash(item)) % self.size, abs(mmh3.hash(item)) % self.size, abs(self._fnv1a_hash(item)) % self.size]
 
-    def _fnv1a_hash(self, data):
-        hash = 0x811C9DC5  # 2166136261
-        for byte in data:
-            hash ^= byte
-            hash = (hash * 0x01000193) % 2**32  # 16777619
-        return hash
+        return [
+            abs(hash(item)) % self.size,
+            abs(mmh3.hash(item)) % self.size,
+            abs(xxhash.xxh32(item).intdigest()) % self.size,
+        ]
 
     def close(self):
         """Explicitly close the memory-mapped file."""