Merge pull request #736 from iLikeToCode/improve_ci

Improve CI

Author: digininja

.github/workflows/codeql-analysis.yml

@@ -7,10 +7,12 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [master]
+    branches: 
+      - master
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [master]
+    branches: 
+      - master
   schedule:
     - cron: '0 15 * * 3'
 
@@ -24,26 +26,23 @@ jobs:
       matrix:
         # Override automatic language detection by changing the below list
         # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
-        language: ['javascript', 'python']
+        language:
+          - 'javascript'
+          - 'python'
         # Learn more...
         # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v6.0.2
       with:
         # We must fetch at least the immediate parents so that if this is
         # a pull request then we can checkout the head.
         fetch-depth: 2
 
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v4.33.0
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +53,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v4.33.0
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +67,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v4.33.0

.github/workflows/docker-image.yml

@@ -3,13 +3,14 @@ name: Docker Image CI
 on:
   workflow_dispatch:
   push:
-    branches: [master]
+    branches:
+      - master
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6.0.2
 
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3

.github/workflows/pytest.yml

@@ -2,9 +2,11 @@ name: CI
 
 on:
   push:
-    branches: [master]
+    branches:
+      - master
   pull_request:
-    branches: [master]
+    branches:
+      - master
   schedule:
     # Every Sunday at 0AM UTC
     - cron: "0 0 * * 0"
@@ -14,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v6.0.2
         with:
           fetch-depth: 0
 

.github/workflows/shiftleft-analysis.yml

@@ -6,37 +6,30 @@ name: SL Scan
 # This section configures the trigger for the workflow. Feel free to customize depending on your convention
 on:
   push:
-    branches: [master]
+    branches:
+      - master
   pull_request:
-    branches: [master]
+    branches:
+      - master
 
 jobs:
   Scan-Build:
-    # Scan runs on ubuntu, mac and windows
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v6.0.2
       with:
         fetch-depth: 0
-    # Instructions
-    # 1. Setup JDK, Node.js, Python etc depending on your project type
-    # 2. Compile or build the project before invoking scan
-    #    Example: mvn compile, or npm install or pip install goes here
-    # 3. Invoke Scan with the github token. Leave the workspace empty to use relative url
 
     - name: Perform Scan
-      uses: ShiftLeftSecurity/scan-action@master
+      uses: ShiftLeftSecurity/scan-action@v1.3.0
       env:
         WORKSPACE: ""
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         SCAN_AUTO_BUILD: true
       with:
         output: reports
-        # Scan auto-detects the languages in your project. To override uncomment the below variable and set the type
-        # type: credscan,java
-        # type: python
 
     - name: Upload report
-      uses: github/codeql-action/upload-sarif@v2
+      uses: github/codeql-action/upload-sarif@v4.33.0
       with:
         sarif_file: reports

.github/workflows/vulnerable.yml

@@ -22,7 +22,7 @@ jobs:
     name: Get Some Secrets
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6.0.2
         with:
           fetch-depth: 0
 

tests/test_url.py

@@ -27,17 +27,18 @@ def check_once(url):
     try:
         headers = {
             'User-Agent':
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit'
+            '/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
         }
-        response = requests.get(url, headers=headers)
+        response = requests.get(url, headers=headers, timeout=5)
     except requests.exceptions.ConnectionError:
         return False, -1
     return response.ok, response.status_code
 
 
 def check(url):
     # We try for 5 times, with 3 seconds interval.
-    try_count = 5
+    try_count = 1
     try_interval = 3
     for i in range(try_count):
         ok, status_code = check_once(url)
@@ -51,37 +52,70 @@ def test_url():
     # Need to rewrite this so it generates a single, unique list of URLs,
     # removes any which are to be ignored, and then checks them. Would be
     # much cleaner.
-    
+
     ignore_urls = [
-        "https://wpscan.com/", # Cloudflare doesn't like GitHub checking it
-        "http://www.w3.org/TR/html4/loose.dtd", # Don't need to check the DTD
-        "https://www.vmware.com/", # Throwing a 403 for some reason, but can't see it going anywhere
-        "https://twitter.com/digininja", # Twitter doesn't like GitHub checking it
-        "https://www.cgisecurity.com/xss-faq.html", # Throwing a 403 for some reason, but can't see it going anywhere
-        "https://www.cgisecurity.com/csrf-faq.html", # Throwing a 403 for some reason, but can't see it going anywhere
+        "https://wpscan.com/",
+        # Cloudflare doesn't like GitHub checking it
+
+        "http://www.w3.org/TR/html4/loose.dtd",
+        # Don't need to check the DTD
+
+        # "https://twitter.com/digininja",
+        # Twitter doesn't like GitHub checking it
+
+        "https://www.cgisecurity.com/xss-faq.html",
+        # Timeout
+
+        "https://www.cgisecurity.com/csrf-faq.html"
+        # Timeout
     ]
+
+    expected_codes = {
+        "https://www.vmware.com/": 403,
+        "https://www.virtualbox.org/": 402,
+
+        "https://github.com/digininja/DVWA/blob/master/README.md"
+        "#vendor-files": 429,
+
+        "https://github.com/digininja/DVWA/blob/master/README.md"
+        "#apache-modules": 429,
+
+        "https://hacks.mozilla.org/2020/08/"
+        "changes-to-samesite-cookie-behavior/": 403,
+
+        "https://blog.mozilla.org/security/2014/10/04/"
+        "csp-for-the-web-we-have/": 403,
+
+        "https://medium.com/@masjadaan/"
+        "oracle-padding-attack-a61369993c86": 403,
+
+        "https://www.golinuxcloud.com/brute-force-attack-web-forms": 403,
+    }
+
     all_urls = []
     broken_urls = []
     for php_file in get_php_files():
         for url in get_urls(php_file):
             all_urls.append(url)
-    
+
     # This removes any duplicates
     dedup_urls = list(dict.fromkeys(all_urls))
 
     for url in dedup_urls:
-        if not url in ignore_urls:
+        if url not in ignore_urls:
             # print("checking %s" % url)
             ok, status_code = check(url)
-            if not ok:
-                # The php_file variable is now broken as it was set in a previous loop
+            if not ok and status_code != expected_codes.get(url):
+                # The php_file variable is now broken
+                # as it was set in a previous loop
                 # and doesn't come across into this one.
 
-                #print("failed to access %s from file %s with code %d" % (url, php_file, status_code))
+                # print("failed to access %s from file %s with code
+                # %d" % (url, php_file, status_code))
                 # broken_urls.append((php_file, url, status_code))
                 broken_urls.append((url, status_code))
 
-    #for php_file, url, status_code in broken_urls:
+    # for php_file, url, status_code in broken_urls:
     #    print("%s\t%s\t%d" % (php_file, url, status_code))
 
     for url, status_code in broken_urls: