master-develop-test.yml

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Tests on production branches

on:
  push:
    branches:
      - "main"
      - "release/*"

jobs:
  unit_tests:

    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-latest ]
        python-version: [ '3.10' ]

    env:
      POSTGRES_DB: mlflow
      POSTGRES_USER: msdndfvk
      POSTGRES_PASSWORD: adfg3245
      AWS_ACCESS_KEY_ID: fdash
      AWS_SECRET_ACCESS_KEY: rstjDFH345!
      MLFLOW_S3_IGNORE_TLS: true
      MLFLOW_TRACKING_URI: http://0.0.0.0:85
      MLFLOW_S3_ENDPOINT_URL: http://0.0.0.0:8002


    steps:
    - uses: actions/checkout@v2
      with:  # required for GitPython checks https://github.com/actions/checkout/issues/124
        fetch-depth: 0

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}

    - name: create test .git in test project root
      working-directory: tests/data
      run: |
        git config --global init.defaultBranch main
        git config --global user.email "you@example.com"
        git config --global user.name "Your Name"
        git init
        git add .
        git commit -m "Initial commit"
        git init --bare origin_test
        git remote add origin_test origin_test
        git remote -v
        git push -u origin_test main

    - name: print dir
      run: |
        sudo apt-get install -y tree
        tree

    - name: Install python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install flake8 pytest pytest-cov torch .
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

    - name: start MLOps server
      run: |
        mkdir -p ~/MLOps_data/mlflow_minio
        mkdir -p ~/MLOps_data/mlflow_db
        docker compose -f mlflow_server/docker-compose.yml up -d --build

    - name: setup XNAT
      run: |
        git clone https://github.com/NrgXnat/xnat-docker-compose.git
        cp xnat-docker-compose/default.env xnat-docker-compose/.env
        docker compose -f xnat-docker-compose/docker-compose.yml up -d

    - name: Setup flake8 annotations
      uses: rbialon/flake8-annotations@v1

    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

    - name: Test with pytest
      run: |
         set -o pipefail
         pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=mlops tests/ | tee pytest-coverage.txt ; echo $?

    - name: Update coverage Badge
      if: github.ref == 'refs/heads/main'  # only on main
      uses: schneegans/dynamic-badges-action@v1.0.0
      with:
        auth: ${{ secrets.PYTEST_COVERAGE_COMMENT }}
        gistID: ba102d5f3e592fcd50451c2eff8a803d
        filename: mlops_pytest-coverage-comment.json
        label: Test coverage
        message: ${{ steps.coverageComment.outputs.coverage }}
        color: ${{ steps.coverageComment.outputs.color }}
        namedLogo: python

    - name: check docker services status
      if: always()
      run: |
        docker ps

    - name: Dump docker logs on failure
      if: failure()
      uses: jwalton/gh-docker-logs@v2

    - name: Stop containers
      if: always()
      run: |
        docker compose -f mlflow_server/docker-compose.yml down
        docker compose -f xnat-docker-compose/docker-compose.yml down