koheesio/.github/workflows/test.yml at main · Nike-Inc/koheesio · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
name: Test
run-name: ${{ 'Tests for PR ' }}${{ github.event.pull_request.number }}${{ ' by ' }}${{ github.event.pull_request.user.login }}


on:
  pull_request:
    branches:
    - main
    - release/*
  workflow_dispatch:
    inputs:
      logLevel:
        description: 'Log level'
        required: true
        default: 'warning'
        type: choice
        options:
        - info
        - warning
        - debug

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  STABLE_PYTHON_VERSION: '3.11'
  PYTHONUNBUFFERED: "1"
  FORCE_COLOR: "1"

jobs:
  check_changes:
    runs-on: ubuntu-latest
    outputs:
      python_changed: ${{ steps.check.outputs.python_changed }}
      toml_changed: ${{ steps.check.outputs.toml_changed }}
    steps:
    - name: Checkout code
      uses: actions/checkout@v6
      with:
        fetch-depth: 0
        ref: ${{ github.event.pull_request.head.sha }}
        repository: ${{ github.event.pull_request.head.repo.full_name }}

    - name: Check changes
      id: check
      run: |
          # Fetch the base branch
          git fetch origin ${{ github.event.pull_request.base.sha }}

          # Get the diff between PR base and head
          git_diff_output=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }})

          # Count the number of changes to Python and TOML files
          python_changed=$(echo "$git_diff_output" | grep '\.py$' | wc -l)
          toml_changed=$(echo "$git_diff_output" | grep '\.toml$' | wc -l)

          # Write the changes to the GITHUB_OUTPUT environment file
          echo "python_changed=$python_changed" >> $GITHUB_OUTPUT
          echo "toml_changed=$toml_changed" >> $GITHUB_OUTPUT

  tests:
    needs: check_changes
    if: needs.check_changes.outputs.python_changed > 0 || needs.check_changes.outputs.toml_changed > 0 || github.event_name == 'workflow_dispatch' || ${{ github.event.pull_request.head.repo.owner.login }} != ${{ github.event.pull_request.base.repo.owner.login }}
    name: Python ${{ matrix.python-version }} with PySpark ${{ matrix.pyspark-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
#        os: [ubuntu-latest, windows-latest, macos-latest]  # FIXME: Add Windows and macOS
        os: [ubuntu-latest]
        python-version: ['3.10', '3.11', '3.12']
        pyspark-version: ['33', '34', '35', '35r', '352', '352r']
        exclude:
          - python-version: '3.10'
            pyspark-version: '352'
          - python-version: '3.10'
            pyspark-version: '352r'
          - python-version: '3.11'
            pyspark-version: '33'
          - python-version: '3.11'
            pyspark-version: '34'
          - python-version: '3.11'
            pyspark-version: '352'
          - python-version: '3.11'
            pyspark-version: '352r'
          - python-version: '3.12'
            pyspark-version: '33'
          - python-version: '3.12'
            pyspark-version: '34'

    steps:
    - uses: actions/checkout@v6

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install Hatch (for local act)
      if: ${{ env.ACT }}
      run: pip install --break-system-packages --ignore-installed packaging hatch

    - name: Install Hatch (for GitHub Actions)
      if: ${{ !env.ACT }}
      uses: pypa/hatch@install

#    - name: Check Code Quality
#      run:  |-
#        hatch fmt --check --python=${{ matrix.python-version }}

    - name: Run tests
      run: hatch test --python=${{ matrix.python-version }} -i version=pyspark${{ matrix.pyspark-version }} --verbose

  # https://github.com/marketplace/actions/alls-green#why
  final_check: # This job does nothing and is only used for the branch protection
    if: always()

    needs:
    - check_changes
    - tests

    runs-on: ubuntu-latest

    steps:
    - name: Decide whether the needed jobs succeeded or failed
      uses: re-actors/alls-green@release/v1
      with:
        jobs: ${{ toJSON(needs) }}
        allowed-skips: tests