feat: Add Spectral Clustering with graph Laplacian eigenvectors #181
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ master, main ] | |
| pull_request: | |
| branches: [ master, main ] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ci-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| JAVA_VERSION: "17" | |
| SCALA_213: "2.13.14" | |
| SCALA_212: "2.12.18" | |
| SPARK_40: "4.0.1" | |
| SPARK_35: "3.5.1" | |
| SPARK_34: "3.4.3" | |
| SBT_OPTS: "-Dsbt.ci=true -Dsbt.log.noformat=true" | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| name: Lint & Style | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ env.JAVA_VERSION }} | |
| - uses: sbt/setup-sbt@v1 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.sbt | |
| ~/.cache/coursier | |
| key: ${{ runner.os }}-lint-${{ hashFiles('**/*.sbt', '**/project/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-lint- | |
| - name: Scalafmt & Scalastyle | |
| run: | | |
| sbt ++${{ env.SCALA_213 }} scalafmtCheckAll | |
| sbt ++${{ env.SCALA_213 }} scalastyle || true | |
| build: | |
| runs-on: ubuntu-latest | |
| name: Build & Package (2.13 + 2.12) | |
| needs: lint | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ env.JAVA_VERSION }} | |
| - uses: sbt/setup-sbt@v1 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.sbt | |
| ~/.cache/coursier | |
| key: ${{ runner.os }}-build-${{ hashFiles('**/*.sbt', '**/project/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-build- | |
| - name: Package (Scala 2.13, Spark 3.5) | |
| run: sbt ++${{ env.SCALA_213 }} -Dspark.version=${{ env.SPARK_35 }} clean package | |
| - name: Package (Scala 2.12, Spark 3.5) for PySpark | |
| run: sbt ++${{ env.SCALA_212 }} -Dspark.version=${{ env.SPARK_35 }} clean package | |
| - name: Upload JARs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: jars | |
| path: | | |
| target/scala-2.13/*.jar | |
| target/scala-2.12/*.jar | |
| if-no-files-found: error | |
| retention-days: 7 | |
| test-jvm: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # Legal Spark–Scala pairs only | |
| # Spark 4.0.x requires Scala 2.13 (2.12 support dropped) | |
| - { scala: "2.13.14", spark: "4.0.1" } | |
| - { scala: "2.13.14", spark: "3.5.1" } | |
| - { scala: "2.12.18", spark: "3.4.3" } | |
| - { scala: "2.12.18", spark: "3.5.1" } | |
| name: Test (Scala ${{ matrix.scala }}, Spark ${{ matrix.spark }}) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ env.JAVA_VERSION }} | |
| - uses: sbt/setup-sbt@v1 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.sbt | |
| ~/.cache/coursier | |
| key: ${{ runner.os }}-test-${{ matrix.scala }}-${{ matrix.spark }}-${{ hashFiles('**/*.sbt', '**/project/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-test-${{ matrix.scala }}-${{ matrix.spark }}- | |
| - name: Run tests | |
| run: sbt ++${{ matrix.scala }} -Dspark.version=${{ matrix.spark }} test | |
| examples: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| name: Examples (keep docs executable) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ env.JAVA_VERSION }} | |
| - uses: sbt/setup-sbt@v1 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.sbt | |
| ~/.cache/coursier | |
| key: ${{ runner.os }}-examples-${{ hashFiles('**/*.sbt', '**/project/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-examples- | |
| - name: Run documented examples (skip if none) | |
| run: | | |
| set -e | |
| if sbt ++${{ env.SCALA_213 }} -Dspark.version=${{ env.SPARK_35 }} "show Test / definedTests" | grep -qi Examples; then | |
| sbt ++${{ env.SCALA_213 }} -Dspark.version=${{ env.SPARK_35 }} "testOnly *Examples*" | |
| else | |
| echo "No *Examples* tests detected; skipping." | |
| fi | |
| test-python: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| name: PySpark Smoke (SE + non-SE) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: jars | |
| path: jars | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ env.JAVA_VERSION }} | |
| - name: Install PySpark and dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pyspark==${{ env.SPARK_35 }} numpy | |
| - name: Run smoke test with 2.12 JAR | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| run: | | |
| ls -la jars/ || true | |
| ls -la jars/scala-2.12/ || true | |
| JAR_212=$(find jars -name "*.jar" -type f -path "*/scala-2.12/*" | head -n1) | |
| if [ -z "$JAR_212" ]; then | |
| echo "ERROR: No Scala 2.12 JAR found in jars/" | |
| exit 1 | |
| fi | |
| echo "Using jar: $JAR_212" | |
| test -f python/smoke_test.py | |
| spark-submit --conf spark.ui.enabled=false --jars "$JAR_212" python/smoke_test.py | |
| coverage: | |
| runs-on: ubuntu-latest | |
| needs: test-jvm | |
| name: Coverage (scoverage) | |
| continue-on-error: true # Coverage is best-effort, don't block CI | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ env.JAVA_VERSION }} | |
| - uses: sbt/setup-sbt@v1 | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.ivy2/cache | |
| ~/.sbt | |
| ~/.cache/coursier | |
| key: ${{ runner.os }}-coverage-${{ hashFiles('**/*.sbt', '**/project/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-coverage- | |
| - name: Run scoverage | |
| run: | | |
| sbt ++${{ env.SCALA_213 }} -Dspark.version=${{ env.SPARK_35 }} coverage test coverageReport | |
| echo "perf_sanity_seconds=$(grep -hR --only-matching -E 'perf_sanity_seconds=[0-9.]+' target || true)" | |
| - name: Upload coverage artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: scoverage-report | |
| path: target/**/scoverage-report/* | |
| retention-days: 7 | |
| - name: Upload to Codecov | |
| uses: codecov/codecov-action@v3 | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| with: | |
| files: ./target/scala-2.13/scoverage-report/scoverage.xml | |
| fail_ci_if_error: false | |
| all-green: | |
| runs-on: ubuntu-latest | |
| needs: [ test-jvm, examples, test-python ] | |
| steps: | |
| - name: Gate | |
| run: | | |
| echo "test-jvm: ${{ needs.test-jvm.result }}" | |
| echo "examples: ${{ needs.examples.result }}" | |
| echo "test-python: ${{ needs.test-python.result }}" | |
| if [ "${{ needs.test-jvm.result }}" = "success" ] && | |
| [ "${{ needs.examples.result }}" = "success" ] && | |
| [ "${{ needs.test-python.result }}" = "success" ]; then | |
| echo "All validations passed ✅" | |
| else | |
| echo "Validations failed ❌" && exit 1 | |
| fi |