Add Docker workflow & small features to run.py (#14)

* misc(run.py): [dev] Suricata rules management, save digger mode used,
fix stop command.

* refactor(webapp): [dev] Small changes to webapp backend code structure.

* fix(webapp): [dev] Small fixed in code generation for raw flow data.

* chore: [dev] Remove tracking of suricata.rules and small edit to run.py

* fix(rules): Update CCIT.rules

* Improve run.py features (#13)

* feat(run.py): [dev] Preserve CTF config between docker downs

* feat(run.py): [dev] Can remove database volume.

* feat(run.py): [dev] Check if volume exists before trying to remove it.

* refactor(run.py): [dev] Create function for prompts that require y/n ans

---------

Co-authored-by: oscar <oscar@debian.oscar>

* Add worflow to build and publish Docker images

---------

Co-authored-by: oscar <oscar@debian.oscar>

Author: oscar0urselli

.github/workflows/docker-publish.yml

@@ -0,0 +1,105 @@
+name: Docker Publish
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'suricata/Dockerfile'
+      - 'webapp/Dockerfile'
+      - 'grafana/Dockerfile'
+      - 'pcap_broker/Dockerfile'
+      - 'suricata/**'
+      - 'webapp/**'
+      - 'grafana/**'
+      - 'pcap_broker/**'
+      - '.github/workflows/docker-publish.yml'
+  release:
+    types:
+      - published
+  workflow_dispatch:
+
+env:
+  REGISTRY: docker.io
+  IMAGE_NAME: ${{ github.repository_owner }}/digger
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - dockerfile: ./suricata/Dockerfile
+            image_name: digger-suricata
+            context: .
+          - dockerfile: ./webapp/Dockerfile
+            image_name: digger-webapp
+            context: ./webapp
+          - dockerfile: ./grafana/Dockerfile
+            image_name: digger-grafana
+            context: ./grafana
+          - dockerfile: ./pcap_broker/Dockerfile
+            image_name: digger-pcap-broker
+            context: ./pcap_broker
+
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image_name }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,prefix={{branch}}-
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          push: ${{ secrets.DOCKER_USERNAME != '' && secrets.DOCKER_PASSWORD != '' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image_name }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-${{ matrix.image_name }}:buildcache,mode=max
+
+  summary:
+    runs-on: ubuntu-latest
+    needs: build-and-push
+    if: always()
+    steps:
+      - name: Create job summary
+        run: |
+          echo "## Docker Images Published" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "The following images have been built and pushed:" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-digger-suricata\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-digger-webapp\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-digger-grafana\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-digger-pcap-broker\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Tags applied:**" >> $GITHUB_STEP_SUMMARY
+          echo "- Branch name (for push events)" >> $GITHUB_STEP_SUMMARY
+          echo "- Semantic version (for releases)" >> $GITHUB_STEP_SUMMARY
+          echo "- Commit SHA" >> $GITHUB_STEP_SUMMARY
+          echo "- \`latest\` (for main branch)" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -1,10 +1,12 @@
 # Copyright (C) 2024  ANSSI
 # SPDX-License-Identifier: CC0-1.0
+
 input_pcaps/*
+!input_pcaps/.gitkeep
 
-# Configurazione
-.env
-services_config.json
+# Configuration
+config/*
+!config/.gitkeep
 
 # Python
 **/__pycache__

.github/workflows/docker.yml config/.gitkeep.github/workflows/docker.yml renamed to config/.gitkeep

@@ -28,6 +28,7 @@ services:
     volumes:
       - "./input_pcaps:/input_pcaps:ro" # remove to prevent users from downloading pcaps
       - "./suricata/output:/suricata/output:ro"
+      - "./config:/config:rw"
     ports:
       - 0.0.0.0:8000:8000
     depends_on:

docker-compose-a.yml

@@ -15,6 +15,7 @@ services:
       - "./input_pcaps:/input_pcaps:rw"
       - "./suricata/rules:/suricata/rules:ro"
       - "./suricata/output:/suricata/output:rw"
+      - "./config:/config:rw"
 
     # Mode B: capture interface (fast, requires root on vulnbox and in Docker)
     # Drastically reduces ingest delay, but requires to setup traffic mirroring

docker-compose-b.yml

@@ -15,6 +15,7 @@ services:
       - "./input_pcaps:/input_pcaps:ro"
       - "./suricata/rules:/suricata/rules:ro"
       - "./suricata/output:/suricata/output:rw"
+      - "./config:/config:rw"
 
     # Mode C: PCAP-over-IP (fast, requires root on vulnbox)
     # Connects to a PCAP-over-IP server, such as pcap-broker to read PCAP data.