Do Not Merge: Integration Branch for GT4Py

.clang-format

@@ -1 +1 @@
-BasedOnStyle: Google 
+BasedOnStyle: Google

.coveragerc

@@ -19,11 +19,11 @@ exclude_lines =
     if False:
     if __name__ == .__main__.:
     pass
-    
+
 omit =
     # Omit files that cannot be tested
     dace/jupyter.py
-    
+
     # Omit deprecated files
     dace/frontend/tensorflow/__init__.py
     dace/frontend/tensorflow/tensorflow.py

.github/citation_format_authors.py

@@ -3,7 +3,7 @@
 Parse the authors file and print for CITATION.cff
 """
 with open("AUTHORS", "r") as f:
-  content = f.readlines()
+    content = f.readlines()
 
 for i, l in enumerate(content[4:]):
     if l == "\n":
@@ -18,5 +18,3 @@
     last_names = ' '.join(names[1:])
     text = f"- family-names: {last_names}\n  given-names: {first_name}"
     print(text)
-
-

.github/workflows/fpga-ci.yml

@@ -11,12 +11,16 @@ on:
 env:
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
 jobs:
   test-fpga:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-ci') }}
     runs-on: [self-hosted, linux, intel-fpga, xilinx-fpga]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Install dependencies

.github/workflows/general-ci.yml

@@ -8,21 +8,25 @@ on:
   merge_group:
     branches: [ main, ci-fix ]
 
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
 jobs:
   test:
     if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7,'3.12']
+        python-version: ['3.9','3.13']
         simplify: [0,1,autoopt]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -55,7 +59,7 @@ jobs:
         else
             export DACE_optimizer_automatic_simplification=${{ matrix.simplify }}
         fi
-        pytest -n auto --cov-report=xml --cov=dace --tb=short -m "not gpu and not verilator and not tensorflow and not mkl and not sve and not papi and not mlir and not lapack and not fpga and not mpi and not rtl_hardware and not scalapack and not datainstrument"
+        pytest -n auto --cov-report=xml --cov=dace --tb=short -m "not gpu and not verilator and not tensorflow and not mkl and not sve and not papi and not mlir and not lapack and not fpga and not mpi and not rtl_hardware and not scalapack and not datainstrument and not long"
         ./codecov
 
     - name: Test OpenBLAS LAPACK

.github/workflows/gpu-ci.yml

@@ -13,13 +13,16 @@ env:
   MKLROOT: /opt/intel/oneapi/mkl/latest/
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
 
 jobs:
   test-gpu:
     if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
     runs-on: [self-hosted, gpu]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Install dependencies

.github/workflows/hardware_test.yml

@@ -4,7 +4,7 @@ jobs:
   test-rtl:
     runs-on: [self-hosted, linux, xilinx-fpga]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Install dependencies
@@ -22,4 +22,4 @@ jobs:
         # Due to an internal bug in the Xilinx tools, where the current datetime is passed as an integer
         # and overflowed in the year 2022, run the RTL FPGA tests pretending like it's January 1st 2021.
         faketime -f "@2021-01-01 00:00:00" pytest -n auto --tb=short -m "rtl_hardware"
-        killall -9 xsim xsimk || true
+        killall -9 xsim xsimk || true

.github/workflows/heterogeneous-ci.yml

@@ -14,12 +14,16 @@ env:
   MKLROOT: /opt/intel/oneapi/mkl/latest/
   CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
 jobs:
   test-heterogeneous:
     if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
     runs-on: [self-hosted, linux]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Install dependencies
@@ -40,7 +44,7 @@ jobs:
       run: |
         papi_avail
 
-    - name: Run parallel pytest 
+    - name: Run parallel pytest
       run: |
         source ~/.venv/bin/activate # activate venv
         export DACE_cache=unique
@@ -55,7 +59,7 @@ jobs:
         source ~/.venv/bin/activate # activate venv
         ./tests/mpi_test.sh
 
-        
+
     - name: Test MPI with pytest
       run: |
         export NOSTATUSBAR=1
@@ -64,7 +68,7 @@ jobs:
         export DACE_cache=unique
         source ~/.venv/bin/activate # activate venv
         mpirun -n 2 coverage run --source=dace --parallel-mode -m pytest -x --with-mpi --tb=short -m "mpi"
-    
+
     - name: Test ScaLAPACK PBLAS with pytest
       run: |
         export NOSTATUSBAR=1

.github/workflows/linting.yml

@@ -0,0 +1,36 @@
+name: Code Quality
+
+on:
+  push:
+    branches: [ main, ci-fix ]
+  pull_request:
+    branches: [ main, ci-fix ]
+
+jobs:
+  linting:
+    if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
+    name: pre-commit
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check repository
+      uses: actions/checkout@v4
+
+    - name: Setup Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+        cache: 'pip'
+
+    - name: Install linting tools
+      run: pip install .[linting]
+
+    - name: Run linting tools
+      id: lint
+      continue-on-error: true
+      run: pre-commit run --all-files
+
+    - name: Show git diff
+      if: steps.lint.outcome == 'failure'
+      run: |
+        ./.github/workflows/scripts/show-git-diff.sh

.github/workflows/pyFV3-ci.yml

@@ -1,17 +1,26 @@
 name: NASA/NOAA pyFV3 repository build test
 
+# Temporarily disabled for main, and instead applied to a specific DaCe v1 maintenance branch (v1/maintenance). Once
+# the FV3 bridge has been adapted to DaCe v1, this will need to be reverted back to apply to main.
 on:
   push:
-    branches: [ main, ci-fix ]
+    #branches: [ main, ci-fix ]
+    branches: [ v1/maintenance, ci-fix ]
   pull_request:
-    branches: [ main, ci-fix ]
+    #branches: [ main, ci-fix ]
+    branches: [ v1/maintenance, ci-fix ]
   merge_group:
-    branches: [ main, ci-fix ]
+    #branches: [ main, ci-fix ]
+    branches: [ v1/maintenance, ci-fix ]
 
 defaults:
     run:
       shell: bash
 
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
 jobs:
     build_and_validate_pyFV3:
       if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
@@ -21,18 +30,18 @@ jobs:
               python-version: [3.11.7]
 
       steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
               repository: 'NOAA-GFDL/PyFV3'
               ref: 'ci/DaCe'
               submodules: 'recursive'
               path: 'pyFV3'
-      - uses: actions/checkout@v2
-        with: 
+      - uses: actions/checkout@v4
+        with:
             path: 'dace'
             submodules: 'recursive'
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
               python-version: ${{ matrix.python-version }}
       - name: Install library dependencies
@@ -41,9 +50,9 @@ jobs:
           sudo apt-get install -y libopenmpi-dev libboost-all-dev
           gcc --version
       # Because Github doesn't allow us to do a git checkout in code
-      # we use a trick to checkout DaCe first (not using the external submodule)    
+      # we use a trick to checkout DaCe first (not using the external submodule)
       # install the full suite via requirements_dev, then re-install the correct DaCe
-      - name: Install Python packages 
+      - name: Install Python packages
         run: |
           python -m pip install --upgrade pip wheel setuptools
           pip install -e ./pyFV3[develop]
@@ -53,11 +62,11 @@ jobs:
           cd pyFV3
           mkdir -p test_data
           cd test_data
-          wget https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.D_SW.tar.gz
+          wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.D_SW.tar.gz
           tar -xzvf 8.1.3_c12_6ranks_standard.D_SW.tar.gz
-          wget https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.RiemSolver3.tar.gz
+          wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.RiemSolver3.tar.gz
           tar -xzvf 8.1.3_c12_6ranks_standard.RiemSolver3.tar.gz
-          wget https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.Remapping.tar.gz
+          wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.Remapping.tar.gz
           tar -xzvf 8.1.3_c12_6ranks_standard.Remapping.tar.gz
           cd ../..
       # Clean up caches between run for stale un-expanded SDFG to trip the build system (NDSL side issue)

.github/workflows/release.sh

@@ -16,4 +16,3 @@ python -m build --sdist
 
 # Upload to PyPI
 twine upload dist/*
-

.github/workflows/scripts/show-git-diff.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Check for uncommitted changes in the working tree
+if [ -n "$(git status --porcelain)" ]; then
+    echo "Linting tools found the following changes are needed to comply"
+    echo "with our automatic styling."
+    echo ""
+    echo "Please run \"pre-commit run --all-files\" locally to fix these."
+    echo "See also https://github.com/spcl/dace/blob/main/CONTRIBUTING.md"
+    echo ""
+    echo "git status"
+    echo "----------"
+    git status
+    echo ""
+    echo "git diff"
+    echo "--------"
+    git --no-pager diff
+    echo ""
+
+    exit 1
+fi

.github/workflows/verilator_compatibility.yml

@@ -17,14 +17,14 @@ jobs:
     steps:
       - name: trigger reason
         run: echo "Trigger Reason:" ${{ github.event.inputs.reason }}
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: checkout submodules
         run: git submodule update --init --recursive
       - name: install apt packages
         run: sudo apt-get update && sudo apt-get -y install git make autoconf g++ flex bison libfl2 libfl-dev
       - name: compile verilator
         run: git clone https://github.com/verilator/verilator.git && cd verilator && git fetch origin && if [ ! "${{ matrix.verilator_version }}" == "master" ]; then  git checkout v${{ matrix.verilator_version }}; fi && autoconf && ./configure && make -j2 && sudo make install
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.8'
           architecture: 'x64'

.gitignore

@@ -141,6 +141,8 @@ src.VC.VC.opendb
 
 # DaCe
 .dacecache/
+# Ignore dacecache if added as a symlink
+.dacecache
 out.sdfg
 *.out
 results.log
@@ -190,3 +192,6 @@ _build/
 
 .*cache/
 /doc/source/config_schema.rst
+
+# Ignoring the test junk
+_all_tests/

.gitmodules

@@ -13,4 +13,4 @@
 	url = https://github.com/spcl/dace-webclient.git
 [submodule "dace/external/rtllib"]
 	path = dace/external/rtllib
-	url = https://github.com/carljohnsen/rtllib.git 
+	url = https://github.com/carljohnsen/rtllib.git

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+default_language_version:
+  python: python3
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+  - id: check-merge-conflict
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+- repo: https://github.com/google/yapf
+  rev: v0.43.0
+  hooks:
+    - id: yapf
+      name: yapf
+      language: python
+      entry: yapf
+      types: [python]

AUTHORS

@@ -1,4 +1,4 @@
-# This is the official list of DaCe authors (individuals or organizations) for 
+# This is the official list of DaCe authors (individuals or organizations) for
 # copyright purposes.
 
 ETH Zurich

CITATION.cff

@@ -1,6 +1,6 @@
 cff-version: 1.2.0
 title: "DaCe - Data Centric Parallel Programming"
-message: "Please cite as" 
+message: "Please cite as"
 # this can be parsed from the AUTHORS file using .github/citation_format_authors.py
 authors:
   - family-names: Ben-Nun
@@ -64,7 +64,7 @@ preferred-citation:
   year: "2019"
   type: conference-paper
   collection-title: "Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis"
-  conference: 
+  conference:
     name: "SC '19"
   authors:
     - family-names: Ben-Nun