Skip to content
d2e Docker Build

d2e Docker Build #13227

Sign in to view logs

d2e Docker Build

d2e Docker Build #13227

Workflow file for this run

.github/workflows/docker-build-push.yaml at c065556
name: d2e Docker Build
permissions:
contents: read
packages: write
on:
pull_request:
types: [opened, ready_for_review, reopened, synchronize]
merge_group:
push:
branches:
- develop
- "release/*"
workflow_dispatch:
inputs:
release_tag:
description: Enter version for release
required: true
type: string
artifacttype:
type: choice
required: true
default: "OSS-develop"
options:
- "OSS-develop"
- "OSS-release"
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}-${{ github.event_name }}
cancel-in-progress: true
env:
REG_URL: ghcr.io/ohdsi
DOCKER_BUILD_ARG1: GITHUB_PAT=${{ secrets.GITHUB_TOKEN }}
PREPULL_FLOW_IMAGES: "" # legacy flow images retired: flows run on the pixi process worker
jobs:
setup:
runs-on: ubuntu-latest
if: github.event_name != 'pull_request' || !github.event.pull_request.draft
outputs:
random_runner: ${{ steps.shuffle.outputs.random_runner }}
run_tag: ${{ steps.tag.outputs.run_tag }}
canonical_tag: ${{ steps.tag.outputs.canonical_tag }}
is_develop_push: ${{ steps.tag.outputs.is_develop_push }}
is_release_push: ${{ steps.tag.outputs.is_release_push }}
is_publish_push: ${{ steps.tag.outputs.is_publish_push }}
plugins_tag: ${{ steps.tag.outputs.plugins_tag }}
is_pr_or_mg: ${{ steps.tag.outputs.is_pr_or_mg }}
changes_src: ${{ steps.file_changes.outputs.src }}
changes_rkernel: ${{ steps.file_changes.outputs.rkernel }}
changes_plugins: ${{ steps.file_changes.outputs.plugins }}
changes_http_tests: ${{ steps.file_changes.outputs.http_tests }}
changes_ui: ${{ steps.file_changes.outputs.ui }}
changes_flow_package_json: ${{ steps.file_changes.outputs.flow_package_json }}
changes_e2e: ${{ steps.file_changes.outputs.e2e }}
changes_regression: ${{ steps.file_changes.outputs.regression }}
steps:
- id: shuffle
run: echo "random_runner=$(shuf -e ubuntu-22.04 ubuntu-24.04 | head -n 1)" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@v4
with:
submodules: recursive
- id: tag
run: |
SHORT_SHA=$(git rev-parse --short HEAD)
IS_PR_OR_MG=false
IS_DEVELOP_PUSH=false
IS_RELEASE_PUSH=false
CANONICAL_TAG=""
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
RUN_TAG="pr-${{ github.event.pull_request.number }}-${SHORT_SHA}"
IS_PR_OR_MG=true
elif [[ "${{ github.event_name }}" == "merge_group" ]]; then
RUN_TAG="mg-${SHORT_SHA}"
IS_PR_OR_MG=true
elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref_name }}" == "develop" ]]; then
RUN_TAG="develop-${SHORT_SHA}"
IS_DEVELOP_PUSH=true
CANONICAL_TAG="develop"
elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref_name }}" == release/* ]]; then
REL_SLUG="${GITHUB_REF_NAME//\//-}"
RUN_TAG="${REL_SLUG}-${SHORT_SHA}"
IS_RELEASE_PUSH=true
CANONICAL_TAG="${REL_SLUG}"
PLUGINS_TAG=""
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
RUN_TAG="${{ github.event.inputs.release_tag }}-beta"
IS_RELEASE_PUSH=true
CANONICAL_TAG="${{ github.event.inputs.release_tag }}"
PLUGINS_TAG="${{ github.event.inputs.release_tag }}"
else
RUN_TAG="dispatch-${SHORT_SHA}"
PLUGINS_TAG=""
fi
IS_PUBLISH_PUSH=false
if [[ "$IS_DEVELOP_PUSH" == "true" || "$IS_RELEASE_PUSH" == "true" ]]; then
IS_PUBLISH_PUSH=true
fi
{
echo "run_tag=${RUN_TAG}"
echo "canonical_tag=${CANONICAL_TAG}"
echo "plugins_tag=${PLUGINS_TAG}"
echo "is_pr_or_mg=${IS_PR_OR_MG}"
echo "is_develop_push=${IS_DEVELOP_PUSH}"
echo "is_release_push=${IS_RELEASE_PUSH}"
echo "is_publish_push=${IS_PUBLISH_PUSH}"
} >> "$GITHUB_OUTPUT"
{
echo "RUN_TAG=${RUN_TAG}"
echo "CANONICAL_TAG=${CANONICAL_TAG}"
} >> "$GITHUB_STEP_SUMMARY"
- uses: dorny/paths-filter@v3
id: file_changes
with:
initial-fetch-depth: 1
filters: |
src:
- "plugins/flows/**"
- "services/**"
- "package.json"
- "docker-compose.yml"
- ".github/workflows/docker-build-push.yaml"
rkernel:
- "services/enterprise-gateway/CustomKernelManager/**"
- "services/enterprise-gateway/kernels/**"
- "services/enterprise-gateway/r-strategus-lib/**"
- "plugins/flows/hades/renv.lock"
- "plugins/flows/_shared_flow_utils/postgresql-42.3.1.jar"
- ".github/workflows/docker-build-push.yaml"
plugins:
- "plugins/flows/**"
- "package.json"
- "docker-compose.yml"
- "plugins/ui/**"
- "plugins/atlas/**"
- "plugins/functions/**"
- "plugins/fhir_functions/**"
- "plugins/sibyl_functions/**"
- "services/trex/ext/**"
- ".github/workflows/docker-build-push.yaml"
http_tests:
- "plugins/functions/_shared/alp-libs/nodejs/**"
- "plugins/functions/analytics-svc/**"
- "plugins/functions/bookmark-svc/**"
- "plugins/functions/cdw-svc/**"
- "plugins/functions/mri-pa-config/**"
- "plugins/functions/mri-pg-config/**"
- "plugins/functions/query-gen-svc/**"
- "services/mri-db/**"
- "services/alp-logto/**"
- "services/alp-pg-management/**"
- "services/trex/**"
- "package.json"
- "docker-compose.yml"
- "docker-compose-local.yml"
- "scripts/**"
- "tests/backend_integration_tests/**"
- ".github/workflows/docker-build-push.yaml"
ui:
- "plugins/ui/**"
- ".github/workflows/docker-build-push.yaml"
flow_package_json:
- "plugins/flows/base/package.json"
- ".github/workflows/docker-build-push.yaml"
e2e:
- "tests/e2e/**"
- "docker-compose-local.yml"
- "services/trex/**"
- ".github/workflows/docker-build-push.yaml"
regression:
- "tests/regression/**"
- "docker-compose-local.yml"
- ".github/workflows/docker-build-push.yaml"
build_images:
needs: [setup]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_src == 'true' || needs.setup.outputs.changes_rkernel == 'true' || needs.setup.outputs.changes_plugins == 'true' || needs.setup.outputs.changes_e2e == 'true'))
runs-on: ${{ needs.setup.outputs.random_runner }}
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
# branch-scoped registry-cache suffix (tag-safe)
CACHE_SLUG: ${{ github.head_ref && format('pr-{0}', github.event.pull_request.number) || 'develop' }}
strategy:
fail-fast: false
matrix:
include:
- AZ_REG_REPOSITORY: d2e-pg-mgmt-init
DOCKER_BUILD_FILE_PATH: ./services/alp-pg-management/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: ./services/alp-pg-management
- AZ_REG_REPOSITORY: d2e-logto
DOCKER_BUILD_FILE_PATH: ./services/alp-logto/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: ./services/alp-logto
- AZ_REG_REPOSITORY: d2e-logto-post-init
DOCKER_BUILD_FILE_PATH: ./services/alp-logto/post-init/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: ./services/alp-logto/post-init
- AZ_REG_REPOSITORY: d2e-enterprise-gateway
DOCKER_BUILD_FILE_PATH: ./services/enterprise-gateway/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: ./services/enterprise-gateway
- AZ_REG_REPOSITORY: d2e-r-ohdsi-kernel
DOCKER_BUILD_FILE_PATH: ./services/enterprise-gateway/kernels/R_ohdsi_docker/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: .
build_disable: ${{ (needs.setup.outputs.changes_rkernel != 'true') && 'true' || 'false' }}
- AZ_REG_REPOSITORY: d2e-dataflow-gen-worker
DOCKER_BUILD_FILE_PATH: ./services/alp-dataflow-gen-worker/Dockerfile
# repo-root context: the image bakes plugins/flows (pixi envs + R)
DOCKER_BUILD_CONTEXT_PATH: .
# Registry build cache (mode=max exports every stage, including the
# expensive env stages): unchanged groups become blob downloads
# instead of renv compiles. PRs write to a per-branch cache tag and
# fall back to reading develop's; develop pushes write the shared
# tag. The hades stage reads GITHUB_PAT from a BuildKit secret so
# the rotating token cannot poison its cache key.
cache_enabled: "true"
needs_github_pat_secret: "true"
- AZ_REG_REPOSITORY: d2e-supabase-storage
DOCKER_BUILD_FILE_PATH: ./services/supabase-storage/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: ./services/supabase-storage
- AZ_REG_REPOSITORY: d2e-materialize-cohorts
DOCKER_BUILD_FILE_PATH: ./services/materialize-cohorts/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: .
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/setup-buildx-action@v3
id: builder
- name: Build and Push
if: ${{ matrix.build_disable != 'true' || github.event_name == 'workflow_dispatch' || needs.setup.outputs.is_develop_push == 'true' || contains(github.ref_name, 'release/') || github.event_name == 'merge_group' }}
uses: docker/build-push-action@v6
with:
builder: ${{ steps.builder.outputs.name }}
context: ${{ matrix.DOCKER_BUILD_CONTEXT_PATH }}
file: ${{ matrix.DOCKER_BUILD_FILE_PATH }}
platforms: linux/amd64
push: true
pull: ${{ matrix.pull_base == 'true' || false }}
tags: ${{ env.REG_URL }}/${{ matrix.AZ_REG_REPOSITORY }}:${{ env.RUN_TAG }}
provenance: false
sbom: false
cache-from: |
${{ matrix.cache_enabled == 'true' && format('type=registry,ref={0}/{1}:buildcache-{2}', env.REG_URL, matrix.AZ_REG_REPOSITORY, env.CACHE_SLUG) || '' }}
${{ matrix.cache_enabled == 'true' && format('type=registry,ref={0}/{1}:buildcache', env.REG_URL, matrix.AZ_REG_REPOSITORY) || '' }}
cache-to: ${{ matrix.cache_enabled == 'true' && format('type=registry,ref={0}/{1}:buildcache{2},mode=max', env.REG_URL, matrix.AZ_REG_REPOSITORY, needs.setup.outputs.is_publish_push == 'true' && '' || format('-{0}', env.CACHE_SLUG)) || '' }}
secrets: ${{ matrix.needs_github_pat_secret == 'true' && format('github_pat={0}', secrets.GITHUB_TOKEN) || '' }}
build-args: |
GIT_COMMIT_ARG=${{ github.sha }}
BUILD_TYPE=OSS
${{ env.DOCKER_BUILD_ARG1 }}
- name: Prune system
if: always()
run: docker system prune -af
# Lockfile discipline for pixi-managed flow plugin environments: manifests
# must not change without their committed pixi.lock, and locks must stay
# binary-only (see _pixi-lock-check.yml).
pixi_lock_check:
needs: [setup]
if: |
github.event_name == 'merge_group' ||
needs.setup.outputs.is_publish_push == 'true' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_plugins == 'true')
uses: ./.github/workflows/_pixi-lock-check.yml
build_plugins:
needs: [setup]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_plugins == 'true' || needs.setup.outputs.changes_e2e == 'true'))
strategy:
fail-fast: false
matrix:
include:
- PKGPATH: ./plugins/flows/base/
- PKGPATH: ./plugins/flows/i2b2/
- PKGPATH: ./plugins/flows/search_embedding/
- PKGPATH: ./plugins/flows/data_management/
- PKGPATH: ./plugins/flows/hades/
- PKGPATH: ./plugins/flows/loyalty_score/
- PKGPATH: ./plugins/flows/data_transformation/
- PKGPATH: ./plugins/ui
NPM: bun
NODE_VERSION: "20.x"
- PKGPATH: ./plugins/atlas
NODE_VERSION: "20.x"
ATLAS: true
- PKGPATH: ./plugins/functions/
DESTPATH: /usr/src/data/plugins/@data2evidence/d2e-functions
RUNNER: ubuntu-24.04
- PKGPATH: ./plugins/fhir_functions/
DESTPATH: /usr/src/data/plugins/@data2evidence/fhir
RUNNER: ubuntu-24.04
- PKGPATH: ./plugins/sibyl_functions/
DESTPATH: /usr/src/data/plugins/@data2evidence/sibyl
RUNNER: ubuntu-24.04
runs-on: ${{ matrix.RUNNER || needs.setup.outputs.random_runner }}
env:
NPM_ARTIFACT_TYPE: ${{ github.event.inputs.artifacttype || 'OSS-develop'}}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
with:
submodules: recursive
fetch-depth: 0
- name: Use Node.js
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.NODE_VERSION || '18.x' }}
registry-url: "https://npm.pkg.github.com"
scope: "@data2evidence"
- name: Setup bun
if: ${{ matrix.NPM == 'bun' }}
uses: oven-sh/setup-bun@v2
with:
bun-version: 1.2.23
- name: Setup NPM
if: ${{ matrix.NPM && matrix.NPM != 'bun' }}
run: npm install -g ${{ matrix.NPM }}
- name: Setup Java
if: ${{ matrix.DESTPATH }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
- name: Install dependencies
if: ${{ matrix.DESTPATH }}
run: |
npm install -g deno@2.5.6
node ./install-deno-deps.js ${{ matrix.PKGPATH }}
- name: Update version
run: |
cd ${{ matrix.PKGPATH }}
if [[ $GITHUB_EVENT_NAME == 'workflow_dispatch' ]]; then
RELEASE_VERSION=${{ needs.setup.outputs.plugins_tag }}
jq --arg v $RELEASE_VERSION '.version=$v' package.json > tmppkg; mv tmppkg package.json
else
jq --arg v "-$(date +%s)-$GITHUB_SHA" '.version+=$v' package.json > tmppkg; mv tmppkg package.json
fi
- name: Build
if: ${{ !matrix.ATLAS }}
run: |
cd ${{ matrix.PKGPATH }}
${{ matrix.NPM == 'bun' && 'bun install' ||'npm install --ignore-scripts' }}
env:
CI: ${{ matrix.NPM != 'bun' }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Atlas3 sub-plugins (network/notebook-plugin/strategus/studies) live in the
# trex-notebook submodule at plugins/atlas/trex-notebook and are unpublished;
# build them from source so plugins/atlas's file: deps resolve during its
# install/postinstall. This is a second checkout of trex-notebook, pinned to the
# restructured monorepo; plugins/ui/libs/react-notebook stays on the older
# single-package commit that the portal's webr-notebook builds against.
# results-viewer is NOT built here: it is consumed prebuilt from GitHub
# Packages (@ohdsi/results-viewer) with its WebR/shinylive runtime included,
# so no R toolchain is needed on this runner.
- name: Build Atlas sub-plugins (trex-notebook submodule)
if: ${{ matrix.ATLAS }}
env:
CI: "true"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
SUB=plugins/atlas/trex-notebook/plugins
# notebook-plugin bundles the notebook app's source, which imports webr.
( cd "$SUB/notebook" && npm install )
# Prefer build:pkg (targets dist/); the plain build targets the sibyl dev host.
for p in network notebook-plugin strategus studies; do
( cd "$SUB/$p" && npm install \
&& npm run "$(node -e "process.stdout.write(require('./package.json').scripts['build:pkg'] ? 'build:pkg' : 'build')")" )
done
# Atlas plugin: postinstall bakes Atlas3 + the Pythia UI into resources/;
# build the portal entry. Packs only resources/ (see package.json "files").
- name: Build Atlas plugin
if: ${{ matrix.ATLAS }}
working-directory: ${{ matrix.PKGPATH }}
env:
CI: "true"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
npm install
npm run build:portal
- name: Patch Package
if: ${{ matrix.NPM }}
working-directory: ${{ matrix.PKGPATH }}
run: jq '.private=false' package.json > tmppkg; mv tmppkg package.json
- name: DEST
working-directory: ${{ matrix.PKGPATH }}
if: ${{ matrix.DESTPATH }}
run: |
cp package.json package.org.json
sudo mkdir -p ${{ matrix.DESTPATH }}
sudo chown runner:docker ${{ matrix.DESTPATH }}
cp -a . ${{ matrix.DESTPATH }}
cd ${{ matrix.DESTPATH }}
export DENO_DIR="$HOME/.cache/deno"
node ${{ github.workspace }}/install-deno-deps.js .
npm run build
- name: Use Node.js - OSS Develop
uses: actions/setup-node@v4
if: env.NPM_ARTIFACT_TYPE == 'OSS-develop'
with:
node-version: ${{ matrix.NODE_VERSION || '18.x' }}
registry-url: "https://pkgs.dev.azure.com/data2evidence/d2e/_packaging/d2e/npm/registry/"
scope: "@data2evidence"
- name: Use Node.js - OSS Release
uses: actions/setup-node@v4
if: env.NPM_ARTIFACT_TYPE == 'OSS-release'
with:
node-version: ${{ matrix.NODE_VERSION || '18.x' }}
registry-url: "https://pkgs.dev.azure.com/data2evidence/d2e/_packaging/stable/npm/registry/"
scope: "@data2evidence"
- name: Publish or Pack
env:
CI: ${{ matrix.NPM != 'bun' }}
SHOULD_PUBLISH: ${{ needs.setup.outputs.is_publish_push == 'true' || github.event_name == 'workflow_dispatch' }}
NODE_AUTH_TOKEN: ${{ secrets.NPM_PROJECT_TOKEN }}
run: |
cd ${{ matrix.DESTPATH || matrix.PKGPATH }}
${{ matrix.NPM == 'bun' && 'bun pm pack' || format('{0} pack', matrix.NPM || 'npm') }}
FAILED=0
MAX_SIZE_MIB=500
MAX_SIZE=$((MAX_SIZE_MIB * 1024 * 1024))
shopt -s nullglob
TGZ_FILES=(*.tgz)
if [ ${#TGZ_FILES[@]} -eq 0 ]; then
echo "ERROR: no .tgz files found after pack"
exit 1
fi
for PACK_FILE in "${TGZ_FILES[@]}"; do
SIZE=$(wc -c < "$PACK_FILE")
SIZE_MIB=$(awk "BEGIN {printf \"%.2f\", $SIZE / 1048576}")
if [ "$SIZE" -gt "$MAX_SIZE" ]; then
echo "FAIL $PACK_FILE — ${SIZE_MIB}MiB (exceeds ${MAX_SIZE_MIB}MiB Azure limit)"
echo "::error::$PACK_FILE is ${SIZE_MIB}MiB, exceeds ${MAX_SIZE_MIB}MiB Azure limit"
FAILED=1
else
echo "OK $PACK_FILE — ${SIZE_MIB}MiB"
fi
done
[ $FAILED -eq 0 ] || exit $FAILED
if [[ $SHOULD_PUBLISH == true ]]; then
for PACK_FILE in "${TGZ_FILES[@]}"; do
npm publish "$PACK_FILE"
done
fi
- name: Slugify PKGPATH for artifact name
id: slug
run: |
slug=$(echo "${{ matrix.PKGPATH }}" | sed 's|^\./||; s|/$||; s|/|-|g')
echo "slug=$slug" >> "$GITHUB_OUTPUT"
- name: Upload plugin tarball artifact
uses: actions/upload-artifact@v4
with:
name: plugin-tgz-${{ steps.slug.outputs.slug }}
path: ${{ matrix.DESTPATH || matrix.PKGPATH }}/*.tgz
if-no-files-found: error
retention-days: 7
- name: Upload UI resources
if: matrix.PKGPATH == './plugins/ui'
uses: actions/upload-artifact@v4
with:
name: ui-resources
path: ./plugins/ui/resources
if-no-files-found: error
retention-days: 7
- name: Upload UI package tarball
if: matrix.PKGPATH == './plugins/ui'
uses: actions/upload-artifact@v4
with:
name: ui-tgz
path: ./plugins/ui/*.tgz
if-no-files-found: error
retention-days: 7
build_trex_image:
needs: [setup, build_plugins]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_src == 'true' || needs.setup.outputs.changes_plugins == 'true' || needs.setup.outputs.changes_e2e == 'true'))
runs-on: ${{ needs.setup.outputs.random_runner }}
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download plugin artifacts
uses: actions/download-artifact@v4
with:
pattern: plugin-tgz-*
path: ./services/trex/plugin-artifacts
merge-multiple: true
- name: Inspect bundled tarballs
run: |
ls -lh ./services/trex/plugin-artifacts/
echo "---"
for f in ./services/trex/plugin-artifacts/*.tgz; do
echo "$(basename "$f"): $(tar -xzOf "$f" package/package.json | jq -r '.name + " " + .version')"
done
- uses: docker/setup-buildx-action@v3
id: builder
- name: Build and Push trex image
uses: docker/build-push-action@v6
with:
builder: ${{ steps.builder.outputs.name }}
context: ./services/trex
file: ./services/trex/Dockerfile.v2
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ env.REG_URL }}/d2e-trex:${{ env.RUN_TAG }}
provenance: false
sbom: false
build-args: |
GIT_COMMIT_ARG=${{ github.sha }}
BUILD_TYPE=OSS
PLUGINS_FEED=${{ needs.setup.outputs.is_release_push == 'true' && 'stable' || 'd2e' }}
PLUGINS_FROM_REGISTRY=@ohdsi/pythia-agent@0.1.0-develop.14.gf8bbfc7
secrets: |
"PLUGINS_REGISTRY=${{ secrets.PLUGINS_REGISTRY }}"
"github_token=${{ secrets.GITHUB_TOKEN }}"
- name: Prune system
if: always()
run: docker system prune -af
build_e2e_image:
needs: [setup]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_e2e == 'true' || needs.setup.outputs.changes_plugins == 'true'))
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
steps:
- uses: actions/checkout@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/setup-buildx-action@v3
id: builder
- name: Build and Push e2e image
uses: docker/build-push-action@v6
with:
builder: ${{ steps.builder.outputs.name }}
context: ./tests/e2e
file: ./tests/e2e/Dockerfile
platforms: linux/amd64
push: true
tags: ${{ env.REG_URL }}/d2e-e2e:${{ env.RUN_TAG }}
provenance: false
sbom: false
helm_lint:
needs: [setup]
runs-on: ubuntu-latest
env:
CHART_DIR: charts/d2e-services
CI_VALUES: charts/d2e-services/ci/test-values.yaml
HELM_VERSION: v3.16.2
steps:
- uses: actions/checkout@v4
- uses: azure/setup-helm@v4
with:
version: ${{ env.HELM_VERSION }}
- name: Helm lint
run: helm lint "$CHART_DIR" -f "$CI_VALUES"
- name: Render templates
run: |
mkdir -p /tmp/rendered
helm template d2e "$CHART_DIR" \
-f "$CI_VALUES" \
--namespace d2e \
> /tmp/rendered/all.yaml
- name: Install kubeconform
run: |
curl -sSL -o /tmp/kubeconform.tgz \
https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz
tar -xzf /tmp/kubeconform.tgz -C /tmp
sudo mv /tmp/kubeconform /usr/local/bin/
- name: Validate rendered manifests
run: |
kubeconform -summary -strict \
-skip CustomResourceDefinition \
-ignore-missing-schemas \
/tmp/rendered/all.yaml
test_helm_kind:
# TEMPORARILY DISABLED: the Helm chart does not yet support the in-engine
# (trexsql-base) trex image; running it on helm needs the trex/WebAPI runtime
# env + DB-bootstrap gating wired into the chart. Re-enabled in a follow-up PR.
# When skipped, test_helm_kind_gate skips and docker-success treats it as OK.
if: false
needs: [setup, build_images, build_trex_image]
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
packages: read
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
PLUGINS_TAG: ${{ needs.setup.outputs.plugins_tag }}
CHART_DIR: charts/d2e-services
CI_VALUES: charts/d2e-services/ci/test-values.yaml
HELM_VERSION: v3.16.2
RELEASE: d2e
NAMESPACE: d2e
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Recreate /opt/hostedtoolcache
# free-disk-space deletes /opt/hostedtoolcache; helm/kind-action needs it.
run: sudo mkdir -p /opt/hostedtoolcache
- uses: actions/checkout@v4
- uses: helm/kind-action@v1
with:
cluster_name: helm-test
- uses: azure/setup-helm@v4
with:
version: ${{ env.HELM_VERSION }}
- name: Show cluster info
run: |
kubectl version
kubectl get nodes -o wide
- name: Create ghcr image-pull secret
run: |
kubectl create namespace "$NAMESPACE"
kubectl create secret docker-registry ghcr-pull \
--docker-server=ghcr.io \
--docker-username="${{ github.actor }}" \
--docker-password="${{ secrets.GITHUB_TOKEN }}" \
-n "$NAMESPACE"
kubectl patch serviceaccount default -n "$NAMESPACE" \
-p '{"imagePullSecrets":[{"name":"ghcr-pull"}]}'
- name: Deploy Postgres for CI
run: |
kubectl apply -n "$NAMESPACE" -f "$CHART_DIR/ci/postgres.yaml"
kubectl rollout status deployment/postgres -n "$NAMESPACE" --timeout=3m
- name: Generate throwaway TLS material
run: |
set -euo pipefail
D=/tmp/ci-tls
mkdir -p "$D"
openssl req -x509 -newkey rsa:2048 -nodes \
-keyout "$D/ca.key" -out "$D/ca.crt" \
-days 30 -subj "/CN=d2e-ci-ca" 2>/dev/null
openssl req -newkey rsa:2048 -nodes \
-keyout "$D/srv.key" -out "$D/srv.csr" \
-subj "/CN=d2e.cluster.local" 2>/dev/null
openssl x509 -req -in "$D/srv.csr" \
-CA "$D/ca.crt" -CAkey "$D/ca.key" -CAcreateserial \
-out "$D/srv.crt" -days 30 2>/dev/null
- name: Helm install (using just-built images)
run: |
D=/tmp/ci-tls
helm install "$RELEASE" "$CHART_DIR" \
-f "$CI_VALUES" \
--set "global.image.tag=${RUN_TAG}" \
--set-file "d2e-core.secrets.ROOT__PKI__CA_CRT=$D/ca.crt" \
--set-file "d2e-core.secrets.ROOT__PKI__CA_KEY=$D/ca.key" \
--set-file "d2e-core.secrets.SERVER__CRT=$D/srv.crt" \
--set-file "d2e-core.secrets.SERVER__KEY=$D/srv.key" \
--set-file "global.secrets.TLS__INTERNAL__CA_CRT=$D/ca.crt" \
--set-file "global.secrets.TLS__INTERNAL__CRT=$D/srv.crt" \
--set-file "global.secrets.TLS__INTERNAL__KEY=$D/srv.key" \
--namespace "$NAMESPACE" \
--wait --timeout 20m
- name: Wait for pods Ready
run: |
kubectl wait \
--for=condition=Ready pod \
--all \
-n "$NAMESPACE" \
--timeout=5m
- name: Show final state
if: always()
run: |
echo "=== pods ==="
kubectl get pods -n "$NAMESPACE" -o wide || true
echo "=== services ==="
kubectl get svc -n "$NAMESPACE" || true
- name: Failure diagnostics
if: failure()
run: |
echo "=== events ==="
kubectl get events -A --sort-by=.lastTimestamp | tail -100 || true
for pod in $(kubectl get pods -n "$NAMESPACE" -o name); do
echo "------------------------------"
echo "DESCRIBE $pod"
echo "------------------------------"
kubectl describe -n "$NAMESPACE" "$pod" || true
echo "------------------------------"
echo "LOGS $pod"
echo "------------------------------"
kubectl logs -n "$NAMESPACE" "$pod" --all-containers --tail=200 || true
done
- name: Mark helm kind attempt success
if: success()
run: |
mkdir -p ./_attempt
touch ./_attempt/success
- name: Upload helm kind attempt outcome
if: always()
uses: actions/upload-artifact@v4
with:
name: helm-kind-attempt
path: ./_attempt/
overwrite: true
- name: Cleanup
if: always()
run: helm uninstall "$RELEASE" -n "$NAMESPACE" || true
test_demosetup_dev:
needs: [setup, build_images, build_trex_image, build_plugins, build_e2e_image]
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
attempt: [1, 2]
env:
VERSION: develop
DOCKER_TAG_NAME: ${{ needs.setup.outputs.run_tag }}
PLUGINS_IMAGE_TAG: ${{ needs.setup.outputs.run_tag }}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Increase swapfile to 15G
run: |
sudo swapoff -a
sudo fallocate -l 15G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
sudo swapon --show
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: npm install
if: success() || failure()
run: npm install
- name: Install Atlas plugin resources
if: success() || failure()
working-directory: ./plugins/atlas
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: npm install
- name: Patch seed config for e2e tests
run: node ./scripts/patch-seed-for-e2e.js
- name: Download UI package tarball
uses: actions/download-artifact@v4
with:
name: ui-tgz
path: ./plugins/ui
- name: Extract UI package
run: |
TGZ=$(ls ./plugins/ui/data2evidence-d2e-ui-*.tgz | head -1)
rm -rf ./plugins/ui-built
mkdir -p ./plugins/ui-built
tar -xzf "$TGZ" -C ./plugins/ui-built --strip-components=1
- name: Initialise d2e
if: success() || failure()
id: init
run: |
init_choice=y ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop init
- name: Pre-pull flow images
run: |
for img in $PREPULL_FLOW_IMAGES; do
docker pull "${REG_URL}/d2e/${img}:${DOCKER_TAG_NAME}"
done
- name: Pull docker compose images
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop pull
docker images --format "table {{.ID}}\t{{.Repository}}\t{{.Tag}}"
- name: Start services
if: success() || failure()
uses: nick-fields/retry@v3
id: start
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 2
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -c ./docker-compose-ci.yml start
DC_EXIT_CODE=$?
DOCKER_STATUS_FILE=../docker-status-${GITHUB_RUN_ID:-ghRunId}-$(date '+%Y%m%dT%H%M%S').txt | tee -a $GITHUB_ENV
docker ps --format {{.Names}},{{.Status}} 2> /dev/null | sort | tee $DOCKER_STATUS_FILE | tee -a $GITHUB_STEP_SUMMARY
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Log status
if: success() || failure()
run: |
echo DC_EXIT_CODE=$DC_EXIT_CODE | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_EXIT_CODE=${{ steps.start.outputs.exit_code }} | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_OUTCOME=${{ steps.start.outcome }} | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_TOTAL_ATTEMPTS=${{ steps.start.outputs.total_attempts }} | tee -a $GITHUB_STEP_SUMMARY
DC_FAIL_COUNT=$(grep -icE "starting|unhealthy" $DOCKER_STATUS_FILE|| true)
echo DC_FAIL_COUNT=$DC_FAIL_COUNT | tee -a $GITHUB_STEP_SUMMARY
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} logs
- name: Wait for d2e-trex healthy
if: success() || failure()
run: |
status=missing
for i in $(seq 1 60); do
status=$(docker inspect -f '{{.State.Health.Status}}' d2e-trex 2>/dev/null || echo "missing")
echo "[$i] d2e-trex health: $status"
[ "$status" = "healthy" ] && break
sleep 5
done
docker ps --format '{{.Names}},{{.Status}}' | sort
if [ "$status" != "healthy" ]; then
echo "::error::d2e-trex did not become healthy (last status: $status)"
exit 1
fi
# demodb (postgres:16-alpine, docker-compose.yml) is behind the demodb profile and
# is otherwise pulled from Docker Hub lazily inside Setupdemo, where a transient
# registry timeout aborts the whole demo setup. Pre-pull it with retries so the
# image is cached before Setupdemo runs.
- name: Pre-pull demo db image
if: success() || failure()
uses: nick-fields/retry@v3
with:
timeout_seconds: 600
retry_wait_seconds: 30
max_attempts: 5
retry_on: any
command: docker pull postgres:16-alpine
- name: Setupdemo
if: success() || failure()
uses: nick-fields/retry@v3
id: setupdemo
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 3
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} setupdemo
DC_EXIT_CODE=$?
DOCKER_STATUS_FILE=../docker-status-${GITHUB_RUN_ID:-ghRunId}-$(date '+%Y%m%dT%H%M%S').txt | tee -a $GITHUB_ENV
docker ps --format {{.Names}},{{.Status}} 2> /dev/null | sort | tee $DOCKER_STATUS_FILE | tee -a $GITHUB_STEP_SUMMARY
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} logs
- name: Checkflow
if: success() || failure()
uses: nick-fields/retry@v3
id: checkflow
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 1
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} checkflow
DC_EXIT_CODE=$?
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Restart d2e services
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -c ./docker-compose-ci.yml stop
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -c ./docker-compose-ci.yml start
- name: Wait for webapi-init to complete
run: |
echo "Waiting for webapi-init to complete (ensures logto admin user is created)..."
for i in $(seq 1 60); do
status=$(docker ps -a --filter "name=webapi-init" --format '{{.Status}}')
if echo "$status" | grep -q "(healthy)"; then
echo "webapi-init completed successfully"
break
fi
if echo "$status" | grep -qE "Exited \([1-9]"; then
echo "webapi-init failed!"
docker logs d2e-webapi-init || true
exit 1
fi
echo "Waiting for webapi-init... ($i/60)"
sleep 5
done
- name: Pull e2e Docker image
run: |
docker pull ${REG_URL}/d2e-e2e:${DOCKER_TAG_NAME}
docker tag ${REG_URL}/d2e-e2e:${DOCKER_TAG_NAME} d2e-e2e
- name: Verify app is reachable
run: |
curl -k -s -o /dev/null -w "%{http_code}" https://localhost:443/d2e/portal || echo "App not reachable"
curl -k -I https://localhost:443/d2e/portal || true
- name: Disable IPv6 to prevent ERR_NETWORK_CHANGED
run: sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
- name: e2e tests
run: |
cd ./tests/e2e
docker run --rm \
--network=host \
--ipc=host \
-v "$(pwd)/test-results:/work/test-results" \
-v "$(pwd)/ctrf:/work/ctrf" \
-e D2E_BASE_URL=https://localhost:443 \
-e CI=true \
-e GITHUB_ACTIONS=true \
d2e-e2e npm test
- name: Mark e2e attempt success
if: success()
run: |
mkdir -p ./_attempt
touch ./_attempt/success
- name: Upload e2e attempt outcome
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-attempt-${{ matrix.attempt }}
path: ./_attempt/
overwrite: true
- name: Upload Error Context
if: always()
uses: actions/upload-artifact@v4
with:
name: error-context-attempt-${{ matrix.attempt }}
path: tests/e2e/test-results/**/*
overwrite: true
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} logs
- name: Publish Test Report
if: always()
uses: ctrf-io/github-test-reporter@v1
with:
report-path: "./tests/e2e/ctrf/*.json"
file-report: true
- name: Stop d2e services
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} stop
- name: Prune system
if: always()
run: docker system prune -af
test_demosetup_hana:
needs: [setup, build_images, build_trex_image, build_plugins]
runs-on: ${{ needs.setup.outputs.random_runner }}
strategy:
fail-fast: false
matrix:
attempt: [1, 2]
env:
VERSION: develop
DOCKER_TAG_NAME: ${{ needs.setup.outputs.run_tag }}
PLUGINS_IMAGE_TAG: ${{ needs.setup.outputs.run_tag }}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: npm install
if: success() || failure()
run: npm install
- name: Install Atlas plugin resources
if: success() || failure()
working-directory: ./plugins/atlas
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: npm install
- name: Initialise d2e
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop -n .env init
# HANA data-model flows need the sqlalchemy-hana dialect. #2824 moved its
# install to the INSTALL_SQLALCHEMY workpool command (blank by default), so
# set that in this test's .env — the dev/DuckDB path doesn't need it.
echo "INSTALL_SQLALCHEMY=bash -c 'uv pip install sqlalchemy-hana==2.2.0 && prefect flow-run execute'" >> .env
env:
init_choice: y
- name: Initialise HANA
if: success() || failure()
run: |
timeout 5m bash -c "ACCEPT_SAP_LICENSE=true DOTENV_FILE=.env ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop -n .env --hana inithana" || echo "HANA initialization timed out or failed, continuing..."
- name: Pre-pull flow images
run: |
for img in $PREPULL_FLOW_IMAGES; do
docker pull "${REG_URL}/d2e/${img}:${DOCKER_TAG_NAME}"
done
- name: Pull docker compose images
if: success() || failure()
run: |
DOTENV_FILE=.env ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop --hana -d ./plugins/functions pull
docker images --format "table {{.ID}}\t{{.Repository}}\t{{.Tag}}"
- name: Start services
if: success() || failure()
uses: nick-fields/retry@v3
id: start
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 2
retry_on: any
command: |
DOTENV_FILE=.env ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions --hana start
DC_EXIT_CODE=$?
DOCKER_STATUS_FILE=../docker-status-${GITHUB_RUN_ID:-ghRunId}-$(date '+%Y%m%dT%H%M%S').txt
echo "DOCKER_STATUS_FILE=$DOCKER_STATUS_FILE" >> $GITHUB_ENV
docker ps --format {{.Names}},{{.Status}} 2> /dev/null | sort | tee $DOCKER_STATUS_FILE | tee -a $GITHUB_STEP_SUMMARY
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Log status
if: success() || failure()
run: |
echo DC_EXIT_CODE=$DC_EXIT_CODE | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_EXIT_CODE=${{ steps.start.outputs.exit_code }} | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_OUTCOME=${{ steps.start.outcome }} | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_TOTAL_ATTEMPTS=${{ steps.start.outputs.total_attempts }} | tee -a $GITHUB_STEP_SUMMARY
DC_FAIL_COUNT=$(grep -icE "starting|unhealthy" $DOCKER_STATUS_FILE || true)
echo DC_FAIL_COUNT=$DC_FAIL_COUNT | tee -a $GITHUB_STEP_SUMMARY
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions logs
- name: Setupdemohana
if: success() || failure()
uses: nick-fields/retry@v3
id: setupdemo
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 1
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions setupdemohana
DC_EXIT_CODE=$?
DOCKER_STATUS_FILE=../docker-status-${GITHUB_RUN_ID:-ghRunId}-$(date '+%Y%m%dT%H%M%S').txt | tee -a $GITHUB_ENV
docker ps --format {{.Names}},{{.Status}} 2> /dev/null | sort | tee $DOCKER_STATUS_FILE | tee -a $GITHUB_STEP_SUMMARY
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions logs
- name: Checkflow
if: success() || failure()
uses: nick-fields/retry@v3
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 1
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions checkflow
DC_EXIT_CODE=$?
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Mark demo hana attempt success
if: success()
run: |
mkdir -p ./_attempt
touch ./_attempt/success
- name: Upload demo hana attempt outcome
if: always()
uses: actions/upload-artifact@v4
with:
name: demo-hana-attempt-${{ matrix.attempt }}
path: ./_attempt/
overwrite: true
- name: Stop d2e services
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions stop
- name: Prune system
if: always()
run: docker system prune -af
test_http_duckdb:
needs: [setup, build_images, build_trex_image, build_plugins]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_http_tests == 'true')
uses: ./.github/workflows/_test-http-duckdb.yml
with:
attempt: "1"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
changes_ui: ${{ needs.setup.outputs.changes_ui }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_http_duckdb_retry:
needs: [setup, build_images, build_trex_image, build_plugins, test_http_duckdb]
if: |
always() && needs.test_http_duckdb.result == 'failure' && (
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_http_tests == 'true')
)
uses: ./.github/workflows/_test-http-duckdb.yml
with:
attempt: "2"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
changes_ui: ${{ needs.setup.outputs.changes_ui }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_http_hana:
needs: [setup, build_images, build_trex_image, build_plugins]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_http_tests == 'true')
uses: ./.github/workflows/_test-http-hana.yml
with:
attempt: "1"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_http_hana_retry:
needs: [setup, build_images, build_trex_image, build_plugins, test_http_hana]
if: |
always() && needs.test_http_hana.result == 'failure' && (
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_http_tests == 'true')
)
uses: ./.github/workflows/_test-http-hana.yml
with:
attempt: "2"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_regression:
needs: [setup, build_images, build_trex_image, build_plugins]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_regression == 'true' || needs.setup.outputs.changes_http_tests == 'true'))
uses: ./.github/workflows/_test-regression.yml
with:
attempt: "1"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_regression_gate:
needs: [test_regression]
if: always() && needs.test_regression.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: regression-attempt-*
path: ./_attempts
- name: Pass if regression attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "Regression performance test attempt succeeded."
exit 0
fi
echo "Regression performance test attempt failed."
exit 1
test_demosetup_dev_gate:
needs: [test_demosetup_dev]
if: always() && needs.test_demosetup_dev.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: e2e-attempt-*
path: ./_attempts
- name: Pass if any attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "At least one e2e attempt succeeded."
exit 0
fi
echo "All e2e attempts failed."
exit 1
test_demosetup_hana_gate:
needs: [test_demosetup_hana]
if: always() && needs.test_demosetup_hana.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: demo-hana-attempt-*
path: ./_attempts
- name: Pass if any attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "At least one demo hana attempt succeeded."
exit 0
fi
echo "All demo hana attempts failed."
exit 1
test_http_duckdb_gate:
needs: [test_http_duckdb, test_http_duckdb_retry]
if: always() && needs.test_http_duckdb.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: http-duckdb-attempt-*
path: ./_attempts
- name: Pass if any attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "At least one HTTP duckdb attempt succeeded."
exit 0
fi
echo "All HTTP duckdb attempts failed."
exit 1
test_http_hana_gate:
needs: [test_http_hana, test_http_hana_retry]
if: always() && needs.test_http_hana.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: http-hana-attempt-*
path: ./_attempts
- name: Pass if any attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "At least one HTTP hana attempt succeeded."
exit 0
fi
echo "All HTTP hana attempts failed."
exit 1
test_helm_kind_gate:
needs: [test_helm_kind]
if: always() && needs.test_helm_kind.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: helm-kind-attempt
path: ./_attempts
- name: Pass if helm kind attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "Helm kind attempt succeeded."
exit 0
fi
echo "Helm kind attempt failed."
exit 1
promote:
needs: [setup, build_images, build_trex_image]
if: |
always() &&
needs.build_images.result == 'success' &&
needs.build_trex_image.result == 'success' &&
needs.setup.outputs.is_publish_push == 'true'
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
CANONICAL_TAG: ${{ needs.setup.outputs.canonical_tag }}
strategy:
fail-fast: false
matrix:
image:
- d2e-pg-mgmt-init
- d2e-logto
- d2e-logto-post-init
- d2e-enterprise-gateway
- d2e-r-ohdsi-kernel
- d2e-trex
- d2e-dataflow-gen-worker
- d2e-supabase-storage
- d2e-materialize-cohorts
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Re-tag run image as canonical
run: |
SRC=${REG_URL}/${{ matrix.image }}:${RUN_TAG}
DST=${REG_URL}/${{ matrix.image }}:${CANONICAL_TAG}
if [[ -z "$CANONICAL_TAG" ]]; then
echo "CANONICAL_TAG empty; nothing to promote."
exit 0
fi
if docker manifest inspect "$SRC" >/dev/null 2>&1; then
docker buildx imagetools create -t "$DST" "$SRC"
else
echo "Source manifest $SRC not present (likely build_disable=true); skipping promotion."
fi
# Multi-arch-aware cleanup: the per-arch child manifests of a manifest
# list are stored as untagged package versions in GHCR, so a blind
# delete-only-untagged sweep (actions/delete-package-versions) removes
# the children of freshly pushed multi-arch tags and leaves the tag
# unpullable ("manifest unknown"). This action resolves manifest-list
# references and only deletes truly orphaned versions.
- name: Delete orphaned untagged versions
uses: dataaxiom/ghcr-cleanup-action@v1
continue-on-error: true
with:
package: ${{ matrix.image }}
delete-untagged: true
validate: true
docker-success:
needs: [build_images, build_trex_image, build_plugins, build_e2e_image, helm_lint, test_helm_kind_gate, test_demosetup_dev_gate, test_demosetup_hana_gate, test_http_duckdb_gate, test_http_hana_gate, test_regression_gate]
if: always()
runs-on: ubuntu-latest
steps:
- name: Check upstream job results
env:
R_BUILD_IMAGES: ${{ needs.build_images.result }}
R_BUILD_TREX: ${{ needs.build_trex_image.result }}
R_BUILD_PLUGINS: ${{ needs.build_plugins.result }}
R_BUILD_E2E: ${{ needs.build_e2e_image.result }}
R_HELM_LINT: ${{ needs.helm_lint.result }}
R_HELM_KIND: ${{ needs.test_helm_kind_gate.result }}
R_DEMO_DEV: ${{ needs.test_demosetup_dev_gate.result }}
R_DEMO_HANA: ${{ needs.test_demosetup_hana_gate.result }}
R_HTTP_DUCKDB: ${{ needs.test_http_duckdb_gate.result }}
R_HTTP_HANA: ${{ needs.test_http_hana_gate.result }}
R_REGRESSION: ${{ needs.test_regression_gate.result }}
run: |
fail=0
for r in "$R_BUILD_IMAGES" "$R_BUILD_TREX" "$R_BUILD_PLUGINS" "$R_BUILD_E2E" "$R_HELM_LINT" "$R_HELM_KIND" "$R_DEMO_DEV" "$R_DEMO_HANA" "$R_HTTP_DUCKDB" "$R_HTTP_HANA" "$R_REGRESSION"; do
case "$r" in
success|skipped) ;;
*) fail=1 ;;
esac
done
exit $fail