.ci.yml

image: $CI_REGISTRY_IMAGE:rdfm-ci-base-trixie

.common_only: &common_only
  only:
    - main
    - merge_requests

variables:
  SSL_CERT_DIR: "certs"
  PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip
  GOROOT: ""
  SCALENODE_CPU: 2
  SCALENODE_RAM: 2048
  SCALENODE_DISK: 20
  SCALENODE_KERNEL: legacy

.build-base:
  variables:
    JWT_SECRET: testsecret123123123
  before_script:
    - python -V
    - (cd server && poetry build && poetry install)
    - (cd manager && poetry build && poetry install)
    - server/tests/certgen.sh certs IP.1:127.0.0.1 DEVICE no

.build:
  <<: *common_only
  extends: .build-base

.mypy-check-base:
  variables:
    TARGET: "" # fill this when extending
    ARGS:   "" # additional arguments for mypy, can be left empty
  stage: static-checks
  before_script:
    - (cd tools/rdfm-lint-scripts && pipx install .)
  script:
    - cd $TARGET && poetry install
    - rdfm-lint-mypy $ARGS
  allow_failure: true

stages:
  - docs-build
  - docs-deploy
  - build
  - static-checks
  - local-tests

python-pycodestyle-check:
  stage: static-checks
  before_script:
    - (cd tools/rdfm-lint-scripts && pipx install .)
  script:
    - (cd server && rdfm-lint-pycodestyle)
    - (cd manager && rdfm-lint-pycodestyle)
    - (cd common/communication && rdfm-lint-pycodestyle)
    - (cd tools/rdfm-plotter && rdfm-lint-pycodestyle --exclude log_pb2.py)

python-mypy-check-manager:
  extends: .mypy-check-base
  variables:
    TARGET: manager

python-mypy-check-server:
  extends: .mypy-check-base
  variables:
    TARGET: server
    ARGS: "--config-file ./mypy.ini"

python-mypy-check-common-communication:
  extends: .mypy-check-base
  variables:
    TARGET: common/communication

python-mypy-check-plotter:
  extends: .mypy-check-base
  variables:
    TARGET: tools/rdfm-plotter

test-alembic-migration-validity:
  extends: .build
  stage: local-tests
  <<: *common_only
  script:
    - (cd server && poetry run pytest tests/test-alembic-migrations.py -v)

test-package-api:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-packages.py

test-group-api:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-groups.py

test-server-update-policies:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-update-policies.py --sqlite
    - poetry run pytest tests/test-update-policies.py --postgres

test-server-update-resolver:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-update-resolver.py

test-server-update-api:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-update-checker.py --sqlite
    - poetry run pytest tests/test-update-checker.py --postgres

test-server-device-auth-api:
  extends: .build
  stage: local-tests
  variables:
    SCALENODE_RAM: 4096
  script:
    - cd server
    - poetry run pytest tests/test-server-auth.py --sqlite
    - poetry run pytest tests/test-server-auth.py --postgres

test-server-s3-storage:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-server-s3-storage.py

test-server-management-auth:
  extends: .build
  stage: local-tests
  variables:
    SCALENODE_RAM: 4096
  script:
    - cd server
    - poetry run pytest tests/test-server-management-auth.py --sqlite
    - poetry run pytest tests/test-server-management-auth.py --postgres

test-package-upload-auth:
  extends: .build
  stage: local-tests
  variables:
    SCALENODE_RAM: 4096
  script:
    - cd server
    - poetry run pytest tests/test-package-upload-auth.py --sqlite
    - poetry run pytest tests/test-package-upload-auth.py --postgres

test-server-route-correctness:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-server-route-correctness.py

test-server-websocket:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-server-ws.py --sqlite
    - poetry run pytest tests/test-server-ws.py --postgres

test-manager:
  extends: .build
  stage: local-tests
  variables:
    SCALENODE_RAM: 4096
  script:
    - cd server
    - poetry run pytest tests/test-manager.py --sqlite
    - poetry run pytest tests/test-manager.py --postgres

test-server-stress:
  extends: .build
  stage: local-tests
  variables:
    SCALENODE_CPU: 4
    SCALENODE_RAM: 16384
  script:
    - cd server
    - poetry run pytest tests/test-stress.py -vs
  artifacts:
    paths:
      - server/tests/logs/*
    when: always
  only:
    - schedules

test-server-logs:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-server-logs.py --sqlite
    - poetry run pytest tests/test-server-logs.py --postgres

test-server-file-download:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-file-download.py  --sqlite
    - poetry run pytest tests/test-file-download.py  --postgres

test-actions:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-actions.py  --sqlite
    - poetry run pytest tests/test-actions.py  --postgres

test-server-side-events:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-server-side-events.py  --sqlite
    - poetry run pytest tests/test-server-side-events.py  --postgres
  artifacts:
    paths:
      - server/tests/server_gunicorn.log
      - server/tests/server_redis.log
    when: always

test-server-pubsub-acl-handling:
  extends: .build
  stage: local-tests
  script:
    - cd server
    - poetry run pytest tests/test-pubsub-acl-handling.py

test-server-pubsub-acl-side-effects:
  extends: .build
  stage: local-tests
  <<: *common_only
  script:
    - cd server/
    - SIF_SECRET=$(apptainer inspect --environment deploy/pubsub-demo/tests/test_kafka.sif | rg '^\s+export RDFM_JWT_SECRET=([\w\W]+)$' -r '$1')
    - export JWT_SECRET=$SIF_SECRET
    - poetry run pytest -v tests/test-pubsub-acl-side-effects.py --sifpath $(pwd)/deploy/pubsub-demo/tests --sqlite
  artifacts:
    paths:
      - server/tests/server.log
    when: always

# ===== rdfm-client ===== #

build-rdfm-client:
  stage: build
  extends: .build
  script:
    - cd devices/linux-client
    - make

build-rdfm-logging-tool:
  stage: build
  script:
    - cd tools/loggers
    - make

test-rdfm-client:
  stage: local-tests
  <<: *common_only
  dependencies:
    - build-rdfm-artifact
  script:
    # Install rdfm-artifact, which is required for the tests
    - mv tools/rdfm-artifact/rdfm-artifact /usr/bin/
    - cd devices/linux-client
    - make generate
    - make test

test-rdfm-client-docker:
  stage: local-tests
  <<: *common_only
  dependencies:
    - build-rdfm-artifact
  script:
    # Install rdfm-artifact, which is required for the tests
    - mv tools/rdfm-artifact/rdfm-artifact /usr/bin/
    # The tests are running in a Docker container, install and set up
    # the Docker daemon
    - cd devices/linux-client
    - apt update
    - apt -qq -y --no-install-recommends install docker-cli docker.io crun fuse-overlayfs pigz ca-certificates
    - mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
    - pushd /sys/fs/cgroup
    - mkdir {cpu,cpuset,blkio,devices,memory,pids}
    - for name in cpu cpuset blkio devices memory pids; do mount -n -t cgroup -o $name cgroup $name; done
    - popd
    - ../../common/docker/start-docker.sh
    # HACK: For CI runs, we have to replace the -it flag in docker run
    #       as this is not a real terminal.
    # HACK: We also comment out git invocations, as only a subfolder is mounted
    #       in the container, which would cause that call to fail. This is fine,
    #       as the invocation is only required when container UID != source UID
    #       (i.e, not in CI runs, where we run everything as root).
    - sed -i "s/\-it//g" ./scripts/test-docker/test-artifact-info.sh
    - sed -i "s/git/#git/g" ./scripts/test-docker/data/check-artifact-info.sh
    - sed -i "s/\-it//g" ./scripts/test-docker/test-deltas.sh
    - sed -i "s/git/#git/g" ./scripts/test-docker/data/check-delta-patching.sh
    - sed -i "s/\-it//g" ./scripts/test-docker/test-delta-over-http.sh
    # Run rdfm-client Docker tests
    - ./scripts/test-docker/test-artifact-info.sh
    - cd scripts/test-docker/
    - ./make-delta-samples.sh
    - cd ../../
    - ./scripts/test-docker/test-deltas.sh
    - ./scripts/test-docker/test-delta-over-http.sh

# ===== rdfm-mcumgr-client ===== #

build-mcumgr-client:
  stage: build
  script:
    - make -C devices/mcumgr-client
  artifacts:
    expire_in: 1 day
    paths:
      - devices/mcumgr-client/rdfm-mcumgr-client

build-mcumgr-test-samples:
  variables:
    SCALENODE_CPU: 4
    SCALENODE_RAM: 8192
    ZEPHYR_WORKSPACE: $CI_PROJECT_DIR/zephyr
    ZEPHYR_BASE: $CI_PROJECT_DIR/zephyr/zephyr
    ZEPHYR_SDK_INSTALL_DIR: $CI_PROJECT_DIR/zephyr/sdk
  <<: *common_only
  stage: build
  before_script:
    - apt update
    - apt -qqy --no-install-recommends install cmake device-tree-compiler ninja-build
  script:
    - python3 -m venv venv
    - source venv/bin/activate
    - ./devices/mcumgr-client/tests/build/build_samples.sh
    - ls -la ./devices/mcumgr-client/tests/out/
  artifacts:
    expire_in: 1 day
    paths:
      - devices/mcumgr-client/tests/out

test-mcumgr-client:
  # NOTE: Prebuilt renode version used in this job won't resolve dependencies on debian:trixie.
  #       Bumping this should be done in a separate issue.
  image: debian:bookworm
  variables:
    SCALENODE_CPU: 4
    SCALENODE_RAM: 6144
    SCALENODE_DISK: 10
  <<: *common_only
  dependencies:
    - build-mcumgr-client
    - build-mcumgr-test-samples
    - build-rdfm-artifact
  stage: local-tests
  before_get_sources:
    - apt update && apt -qqy --no-install-recommends install build-essential ca-certificates curl git-lfs  iproute2 libpq-dev python3-venv python3-dev >/dev/null
  script:
    - set -eo pipefail
    # NOTE: Currently Renode is tracked using LFS since we're using a custom
    #       build with support for both STM32F764 and NRF52840 flash controllers.
    #       If they get added into main, this can be replaced with official release
    #       `wget --quiet https://builds.renode.io/renode_1.X.Y_amd64.deb`
    - git lfs pull && apt -qqy --no-install-recommends install ./devices/mcumgr-client/tests/prebuilt/renode*.deb >/dev/null
    - mv tools/rdfm-artifact/rdfm-artifact /usr/bin/
    - mv devices/mcumgr-client/rdfm-mcumgr-client /usr/bin/
    - python3 -m venv venv
    - source venv/bin/activate
    - pip3 install -e ./common/communication
    - pip3 install -e ./server
    - pip3 install -e ./manager
    - cd devices/mcumgr-client
    # Run the test
    - ./tests/end2end/test.sh

# ===== rdfm-artifact ===== #

build-rdfm-artifact:
  stage: build
  script:
    - cd tools/rdfm-artifact
    - make
  artifacts:
    paths:
      - tools/rdfm-artifact/rdfm-artifact

test-rdfm-artifact:
  <<: *common_only
  stage: local-tests
  script:
    - cd tools/rdfm-artifact
    - make test

# ===== rdfm-jwt-auth ===== #

build-rdfm-jwt-auth:
  stage: build
  script:
    - (cd server/deploy/pubsub-demo/rdfm-jwt-auth && mvn package)
    - cp server/deploy/pubsub-demo/rdfm-jwt-auth/target/rdfm-jwt-auth-*.jar server/deploy/pubsub-demo/tests
    - cd server/deploy/pubsub-demo/tests
    - mkdir tmp
    - APPTAINER_TMPDIR=$(pwd)/tmp apptainer -v build test_kafka.sif test_kafka.def
  artifacts:
    expire_in: 1 day
    paths:
      - server/deploy/pubsub-demo/rdfm-jwt-auth/target/rdfm-jwt-auth-*.jar
      - server/deploy/pubsub-demo/tests/test_kafka.sif

# ===== rdfm-plotter ===== #

test-rdfm-plotter:
  <<: *common_only
  stage: local-tests
  dependencies:
    - build-rdfm-jwt-auth
  script:
    - SIFPATH=$(pwd)/server/deploy/pubsub-demo/tests
    - cd tools/rdfm-plotter
    - poetry install
    - poetry run pytest tests/test-plotter.py -s -v --testpath $SIFPATH
    - poetry run pytest tests/test-rdfm-jwt-auth.py -s -v --testpath $SIFPATH

# ===== rdfm-frontend ===== #

build-rdfm-frontend:
  image: node:20
  stage: static-checks
  <<: *common_only
  script:
    - cd frontend
    - npm install
    - npm run format-check
    - npm run build
  artifacts:
    paths:
      - frontend/dist

# ===== documentation ===== #

docs-build:
  stage: docs-build
  <<: *common_only
  image: $CI_REGISTRY_IMAGE:rdfm-ci-documentation-trixie
  before_script:
    - cd documentation/
    - poetry install
  script:
    - cd documentation/
    - poetry run make html latexpdf
    - cp build/latex/*.pdf build/html/
    # Make sure the docs archive lands in the root of the repository
    - tar cf ../$CI_DOCS_ARCHIVE -C build/html/ .
  artifacts:
    paths:
      - documentation/build
      - $CI_DOCS_ARCHIVE

docs-deploy:
  variables:
    GIT_STRATEGY: none
  <<: *common_only
  dependencies:
    - docs-build
  stage: docs-deploy
  tags: ["docs"]
  script: echo 'Deploying docs'
  artifacts:
    paths:
      - $CI_DOCS_ARCHIVE