trying to consolidate docker build workflows

Author: dlebauer

.github/workflows/ci.yml

@@ -1,64 +1,64 @@
-name: CI
+name: Docker Stack SIPNET
 
 on:
-  push:
-    branches:
-      - main
-      - develop
-    tags:
-      - "v*.*.*"
   pull_request:
-  merge_group:
-  issue_comment:
-    types:
-      - created
   workflow_dispatch:
-
-env:
-  R_LIBS_USER: /usr/local/lib/R/site-library
-  LC_ALL: en_US.UTF-8
-  NCPUS: 2
-  PGHOST: postgres
-  CI: true
+  schedule:
+    - cron: '30 4 * * 4'
 
 jobs:
-  test:
-    if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build')
-    strategy:
-      fail-fast: false
-      matrix:
-        R:
-          - "4.4"
-          - "4.2"
-    uses: ./.github/workflows/test.yml
-    with:
-      R-version: ${{ matrix.R }}
-    secrets: inherit
+  sipnet:
+    name: sipnet
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: false
+          swap-storage: true
 
-  check:
-    if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build')
-    strategy:
-      fail-fast: false
-      matrix:
-        package: [check_base, check_modules, check_models]
-        R:
-          - "4.4"
-          - "4.2"
-    uses: ./.github/workflows/check.yml
-    with:
-      R-version: ${{ matrix.R }}
-      make-grouping: ${{ matrix.package }}
-    secrets: inherit
+      - name: Checkout source code
+        uses: actions/checkout@v4
 
-  sipnet:
-    if: github.event_name != 'issue_comment' || startsWith(github.event.comment.body, '/build')
-    strategy:
-      fail-fast: false
-      matrix:
-        R:
-          - "4.4"
-          - "4.2"
-    uses: ./.github/workflows/sipnet.yml
-    with:
-      R-version: ${{ matrix.R }}
-    secrets: inherit
+      - name: Set up Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install docker-compose
+
+      - name: Prepare override
+        run: |
+          cp docker-compose.dev.yml docker-compose.override.yml
+          sed -i'' '/R_library/d' docker-compose.override.yml
+
+      - name: Initialize data volumes
+        run: |
+          cp docker/env.example .env
+          echo "COMPOSE_PROJECT_NAME=pecan" >> .env
+          echo "PECAN_VERSION=develop" >> .env
+          echo "UID=$(id -u)" >> .env
+          echo "GID=$(id -g)" >> .env
+          docker compose up --wait -d postgres
+          docker run --rm --network pecan_pecan pecan/db:ci
+          docker run --rm --network pecan_pecan --volume pecan_pecan:/data --env FQDN=docker pecan/data:develop
+          docker run --rm --network pecan_pecan --volume pecan_pecan:/data pecan/data:develop chown -R "$(id -u):$(id -g)" /data
+          docker image rm pecan/db:ci pecan/data:develop || true
+
+      - name: Start required services
+        run: |
+          docker compose up --wait -d sipnet executor monitor rabbitmq
+
+      - name: List running containers
+        run: |
+          sleep 50
+          docker ps
+
+      - name: Run SIPNET workflow
+        run: |
+          docker compose exec --workdir /pecan/tests executor R CMD ../web/workflow.R --settings docker-ghaction.sipnet.xml

.github/workflows/docker-build-image.yml

@@ -1,153 +1,64 @@
-name: build-image
+name: Docker Stack SIPNET
 
 on:
-  workflow_call:
-    inputs:
-      image-name:
-        required: true
-        type: string
-      build-context:
-        required: true
-        type: string
-      dockerfile:
-        required: true
-        type: string
-      r-version:
-        required: true
-        type: string
-      parent-image:
-        required: false
-        default: ''
-        type: string
-      model-version:
-        required: false
-        default: ''
-        type: string
-      platforms:
-        required: false
-        default: "linux/amd64"
-        type: string
-    secrets:
-      DOCKERHUB_USERNAME:
-        description: 'DockerHub username used to push images'
-        required: false
-      DOCKERHUB_PASSWORD:
-        description: 'DockerHub password used to push images'
-        required: false
-
-env:
-  DEFAULT_R_VERSION: "4.4"
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '30 4 * * 4'
 
 jobs:
-  build:
+  sipnet:
+    name: sipnet
     runs-on: ubuntu-latest
-    permissions:
-      packages: write
-
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          docker-images: false
+          swap-storage: true
 
-      - name: lowercase image name
-        id: name
-        run: |
-          echo "image_name=$(echo ${{ inputs.image-name }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
-          echo "repository=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
+      - name: Checkout source code
+        uses: actions/checkout@v4
 
-      - name: set PARENT_IMAGE only if specified
-        id: parent
-        shell: bash
+      - name: Set up Docker Compose
         run: |
-          echo "PARENT_IMAGE_IF_SET=$(
-            [[ -n '${{ inputs.parent-image }}' ]] &&
-            echo "PARENT_IMAGE=ghcr.io/${{ steps.name.outputs.repository }}/"'${{ inputs.parent-image }}'
-            )" >> $GITHUB_OUTPUT
+          sudo apt-get update
+          sudo apt-get -y install docker-compose
 
-      - name: set MODEL_VERSION only if specified
-        id: modelver
-        shell: bash
+      - name: Prepare override
         run: |
-          echo "MODEL_VERSION_IF_SET=$(
-            [[ -n '${{ inputs.model-version }}' ]] &&
-            echo 'MODEL_VERSION=${{ inputs.model-version }}'
-            )" >> $GITHUB_OUTPUT
-
-      - uses: actions/checkout@v4
-
-      # create metadata for image
-      - name: Docker meta
-        env:
-          check_var: ${{ secrets.DOCKERHUB_USERNAME }}
-          is_default_R: ${{ inputs.r-version == env.DEFAULT_R_VERSION }}
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            name=ghcr.io/${{ steps.name.outputs.repository }}/${{ steps.name.outputs.image_name }}
-            name=pecan/${{ steps.name.outputs.image_name }},enable=${{ env.check_var != null }}
-          # generate Docker tags based on the following events/attributes
-          tags: |
-            type=schedule
-            type=ref,event=branch,enable=${{ env.is_default_R }}
-            type=ref,event=branch,suffix=-R${{ inputs.r-version }}
-            type=ref,event=pr
-            type=semver,pattern={{version}},enable=${{ env.is_default_R }}
-            type=semver,pattern={{major}}.{{minor}},enable=${{ env.is_default_R }}
-            type=semver,pattern={{major}},enable=${{ env.is_default_R }}
-            type=semver,pattern={{version}},suffix=-R${{ inputs.r-version }}
-
-      # setup docker build
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+          cp docker-compose.dev.yml docker-compose.override.yml
+          sed -i'' '/R_library/d' docker-compose.override.yml
 
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Inspect Builder
+      - name: Initialize data volumes
         run: |
-          echo "Name:      ${{ steps.buildx.outputs.name }}"
-          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
-          echo "Status:    ${{ steps.buildx.outputs.status }}"
-          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
-          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+          cp docker/env.example .env
+          echo "COMPOSE_PROJECT_NAME=pecan" >> .env
+          echo "PECAN_VERSION=develop" >> .env
+          echo "UID=$(id -u)" >> .env
+          echo "GID=$(id -g)" >> .env
+          docker compose up --wait -d postgres
+          docker run --rm --network pecan_pecan pecan/db:ci
+          docker run --rm --network pecan_pecan --volume pecan_pecan:/data --env FQDN=docker pecan/data:develop
+          docker run --rm --network pecan_pecan --volume pecan_pecan:/data pecan/data:develop chown -R "$(id -u):$(id -g)" /data
+          docker image rm pecan/db:ci pecan/data:develop || true
 
-      # login to registries
-      - name: Login to DockerHub
-        env: 
-          check_var: ${{ secrets.DOCKERHUB_USERNAME }}
-        if: env.check_var != null
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: Start required services
+        run: |
+          docker compose up --wait -d sipnet executor monitor rabbitmq
 
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: List running containers
+        run: |
+          sleep 50
+          docker ps
 
-      # build the docker images
-      - name: Build and push ${{ steps.name.outputs.image_name }}
-        uses: docker/build-push-action@v6
-        with:
-          context: ${{ inputs.build-context }}
-          file: ${{ inputs.dockerfile }}
-          push: true
-          platforms: ${{ inputs.platforms }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            VERSION=${{ steps.meta.outputs.version }}
-            IMAGE_VERSION=${{ steps.meta.outputs.version }}
-            PECAN_VERSION=${{ steps.meta.outputs.version }}
-            R_VERSION=${{ inputs.r-version }}
-            ${{ steps.parent.outputs.PARENT_IMAGE_IF_SET }}
-            ${{ steps.modelver.outputs.MODEL_VERSION_IF_SET }}
-            GITHUB_PAT=${{ secrets.GITHUB_TOKEN }}
-            PECAN_GIT_BRANCH=${{ github.head_ref || github.ref_name }}
-            PECAN_GIT_CHECKSUM=${{ github.sha }}
-            PECAN_GIT_DATE=${{ github.event.repository.updated_at }}
+      - name: Run SIPNET workflow
+        run: |
+          docker compose exec --workdir /pecan/tests executor R CMD ../web/workflow.R --settings docker-ghaction.sipnet.xml