Skip to content

Commit c98f085

Browse files
nllongnllong
authored
Fix modelica runner file ownership (#861)
1 parent cf93b18 commit c98f085

13 files changed

Lines changed: 523 additions & 106 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 131 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,13 @@ defaults:
1313

1414
jobs:
1515
test:
16+
name: Test (${{ matrix.os }}, Python ${{ matrix.python-version }})
1617
strategy:
1718
fail-fast: false
1819
matrix:
1920
os: [ubuntu-latest, windows-latest]
2021
python-version: ["3.10", "3.13"]
21-
test_env: [python, mypy]
2222
mbl_tag: [v12.1.0]
23-
exclude:
24-
# only test mypy on linux for all versions of python
25-
- os: windows-latest
26-
test_env: mypy
2723
runs-on: ${{ matrix.os }}
2824
permissions: write-all
2925
steps:
@@ -32,69 +28,165 @@ jobs:
3228
uses: actions/setup-python@v6
3329
with:
3430
python-version: ${{ matrix.python-version }}
31+
- name: Install Poetry
32+
uses: abatilo/actions-poetry@v4
33+
with:
34+
poetry-version: "latest"
35+
- name: Install dependencies with Poetry
36+
# poetry setuptools workaround sourced from https://github.com/python-poetry/poetry/issues/7611#issuecomment-1711443539
37+
run: |
38+
poetry --version
39+
poetry self add setuptools
40+
poetry install
41+
- name: Allow long filenames in Windows
42+
if: matrix.os == 'windows-latest'
43+
run: git config --system core.longpaths true
44+
- name: Install MBL
45+
uses: actions/checkout@v6
46+
with:
47+
repository: lbl-srg/modelica-buildings
48+
ref: ${{ matrix.mbl_tag }}
49+
path: modelica-buildings
50+
- name: Set MODELICAPATH
51+
run: echo "MODELICAPATH=${{ github.workspace }}/modelica-buildings" >> "$GITHUB_ENV"
52+
- name: Run pytest with coverage
53+
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
54+
run: make test-fast PYTEST_EXTRA_ARGS="-v --cov-report term-missing --cov ."
55+
- name: Run pytest
56+
if: matrix.os != 'ubuntu-latest' || matrix.python-version != '3.13'
57+
run: make test-fast PYTEST_EXTRA_ARGS=-v
58+
59+
resource-intensive-tests:
60+
name: Docker simulations and compilations (${{ matrix.os }})
61+
strategy:
62+
fail-fast: false
63+
matrix:
64+
os: [ubuntu-latest, windows-2025]
65+
mbl_tag: [v12.1.0]
66+
runs-on: ${{ matrix.os }}
67+
permissions: write-all
68+
steps:
69+
- uses: actions/checkout@v6
70+
- name: Set up Python 3.13
71+
if: runner.os == 'Linux'
72+
uses: actions/setup-python@v6
73+
with:
74+
python-version: "3.13"
3575
- name: Display system info
76+
if: runner.os == 'Linux'
3677
run: |
3778
python -c "import sys; print(sys.version)"
3879
docker --version
3980
docker compose version
4081
- name: Install Poetry
82+
if: runner.os == 'Linux'
4183
uses: abatilo/actions-poetry@v4
4284
with:
4385
poetry-version: "latest"
4486
- name: Install dependencies with Poetry
87+
if: runner.os == 'Linux'
4588
# poetry setuptools workaround sourced from https://github.com/python-poetry/poetry/issues/7611#issuecomment-1711443539
4689
run: |
4790
poetry --version
4891
poetry self add setuptools
4992
poetry install
50-
- name: Install modelicafmt
51-
run: |
52-
RUNNER_SYSTEM=$(python -c 'import platform; print(platform.system())')
53-
curl -SLO "https://github.com/urbanopt/modelica-fmt/releases/download/v0.2-pr.2/modelica-fmt_0.2-pr.2_${RUNNER_SYSTEM}_x86_64.tar.gz"
54-
tar --exclude='README.md' --exclude='CHANGELOG.md' -xzf modelica-fmt_0.2-pr.2_${RUNNER_SYSTEM}_x86_64.tar.gz
55-
chmod +x modelicafmt
56-
if [[ $RUNNER_SYSTEM == 'Linux' ]]; then
57-
sudo mv modelicafmt /usr/local/bin/
58-
else
59-
mv modelicafmt '/c/Program Files/'
60-
fi
93+
- name: Set up WSL2 for Linux containers
94+
if: runner.os == 'Windows'
95+
uses: Vampire/setup-wsl@v6
96+
with:
97+
distribution: Ubuntu-24.04
98+
additional-packages: docker.io make python3-pip python3-venv
6199
- name: Allow long filenames in Windows
62-
if: matrix.os == 'windows-latest'
63-
run: |
64-
git config --system core.longpaths true
100+
if: runner.os == 'Windows'
101+
run: git config --system core.longpaths true
65102
- name: Install MBL
66103
uses: actions/checkout@v6
67104
with:
68105
repository: lbl-srg/modelica-buildings
69106
ref: ${{ matrix.mbl_tag }}
70107
path: modelica-buildings
71108
- name: Set MODELICAPATH
109+
if: runner.os == 'Linux'
110+
run: echo "MODELICAPATH=${{ github.workspace }}/modelica-buildings" >> "$GITHUB_ENV"
111+
# Keep this as one non-xdist invocation so Docker workloads run in series.
112+
- name: Run resource-intensive tests in series on Linux
113+
if: runner.os == 'Linux'
114+
run: make test-resource-intensive PYTEST_EXTRA_ARGS=-v
115+
# The GMT runner image is Linux-only. On a Windows host, run it with the
116+
# Linux Docker daemon in WSL2, as Docker Desktop does for local users.
117+
- name: Run resource-intensive tests in series on Windows
118+
if: runner.os == 'Windows'
119+
shell: wsl-bash {0}
72120
run: |
73-
MODELICAPATH="${{ github.workspace }}/modelica-buildings"
74-
cd "${MODELICAPATH}"
75-
echo "Git branch is $(git branch)"
76-
# export MODELICAPATH for subsequent steps
77-
echo "MODELICAPATH=${MODELICAPATH}" >> $GITHUB_ENV
78-
- name: Run pytest (including simulations)
79-
env:
80-
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
81-
if: matrix.test_env == 'python' && matrix.os == 'ubuntu-latest'
82-
run: poetry run pytest tests --doctest-modules -v -m 'not dymola' --cov-report term-missing --cov .
83-
- name: Run Pytest (no compilation or simulation)
84-
if: matrix.test_env == 'python' && matrix.os == 'windows-latest'
85-
run: poetry run pytest tests --doctest-modules -v -m 'not simulation and not compilation and not dymola'
86-
- name: Run pre-commit
87-
run: poetry run pre-commit run --show-diff-on-failure --color=always --all-files
88-
- name: Run mypy
89-
if: matrix.test_env =='mypy'
90-
run: poetry run mypy --exclude='modelica-buildings/' --install-types --non-interactive --show-error-codes .
121+
sudo service docker start
122+
python3 -m venv /tmp/gmt-ci
123+
/tmp/gmt-ci/bin/pip install poetry
124+
export PATH="/tmp/gmt-ci/bin:$PATH"
125+
export MODELICAPATH="$PWD/modelica-buildings"
126+
poetry self add setuptools
127+
poetry install
128+
python --version
129+
docker --version
130+
make test-resource-intensive PYTEST_EXTRA_ARGS=-v
91131
- name: Job Failed
92132
if: ${{ failure() }}
93133
run: |
94134
echo "Maybe these logs will help?"
95-
ls -alt $GITHUB_WORKSPACE
96-
find $GITHUB_WORKSPACE -type f -name 'stdout.log' -print | while read filename; do
135+
ls -alt "$GITHUB_WORKSPACE"
136+
find "$GITHUB_WORKSPACE" -type f -name 'stdout.log' -print | while read filename; do
97137
echo "============================================ stdout.log ========================================="
98138
echo "$filename"
99139
cat "$filename"
100140
done
141+
142+
mypy:
143+
name: Mypy (Python ${{ matrix.python-version }})
144+
strategy:
145+
fail-fast: false
146+
matrix:
147+
python-version: ["3.10", "3.13"]
148+
runs-on: ubuntu-latest
149+
permissions: write-all
150+
steps:
151+
- uses: actions/checkout@v6
152+
- name: Set up Python ${{ matrix.python-version }}
153+
uses: actions/setup-python@v6
154+
with:
155+
python-version: ${{ matrix.python-version }}
156+
- name: Install Poetry
157+
uses: abatilo/actions-poetry@v4
158+
with:
159+
poetry-version: "latest"
160+
- name: Install dependencies with Poetry
161+
run: |
162+
poetry self add setuptools
163+
poetry install
164+
- name: Run mypy
165+
run: poetry run mypy --install-types --non-interactive --show-error-codes .
166+
167+
pre-commit:
168+
name: Pre-commit
169+
runs-on: ubuntu-latest
170+
permissions: write-all
171+
steps:
172+
- uses: actions/checkout@v6
173+
- name: Set up Python 3.13
174+
uses: actions/setup-python@v6
175+
with:
176+
python-version: "3.13"
177+
- name: Install Poetry
178+
uses: abatilo/actions-poetry@v4
179+
with:
180+
poetry-version: "latest"
181+
- name: Install dependencies with Poetry
182+
run: |
183+
poetry self add setuptools
184+
poetry install
185+
- name: Install modelicafmt
186+
run: |
187+
curl -SLO "https://github.com/urbanopt/modelica-fmt/releases/download/v0.2-pr.2/modelica-fmt_0.2-pr.2_Linux_x86_64.tar.gz"
188+
tar --exclude='README.md' --exclude='CHANGELOG.md' -xzf modelica-fmt_0.2-pr.2_Linux_x86_64.tar.gz
189+
chmod +x modelicafmt
190+
sudo mv modelicafmt /usr/local/bin/
191+
- name: Run pre-commit
192+
run: poetry run pre-commit run --show-diff-on-failure --color=always --all-files

‎Makefile‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,31 @@
1+
.PHONY: init test test-fast test-resource-intensive test-resource-intensive-dymola test-resources-intensive-dymola test-all
2+
3+
PYTEST := poetry run pytest
4+
PYTEST_ARGS := tests --doctest-modules
5+
PYTEST_EXTRA_ARGS ?=
6+
FAST_TEST_MARKERS := not simulation and not compilation and not dymola
7+
RESOURCE_TEST_MARKERS := (simulation or compilation) and not dymola
8+
DYMOLA_TEST_MARKERS := dymola
9+
110
init:
2-
pip install -r requirements.txt
11+
poetry install
12+
13+
# Default to the quick local feedback loop; this excludes Docker workloads.
14+
test: test-fast
15+
16+
test-fast:
17+
$(PYTEST) $(PYTEST_ARGS) -n auto --dist loadgroup -m '$(FAST_TEST_MARKERS)' $(PYTEST_EXTRA_ARGS)
18+
19+
# One pytest process intentionally keeps Docker simulations and compilations serial.
20+
test-resource-intensive:
21+
$(PYTEST) $(PYTEST_ARGS) -m '$(RESOURCE_TEST_MARKERS)' $(PYTEST_EXTRA_ARGS)
22+
23+
# Dymola requires a local installation and license, and must run in series.
24+
test-resources-intensive-dymola:
25+
$(PYTEST) $(PYTEST_ARGS) -m '$(DYMOLA_TEST_MARKERS)' $(PYTEST_EXTRA_ARGS)
26+
27+
# Provide the singular spelling alongside the requested command.
28+
test-resource-intensive-dymola: test-resources-intensive-dymola
329

4-
test:
5-
pytest
30+
# Run the fast suite first, then the resource-intensive suite in series.
31+
test-all: test-fast test-resource-intensive

‎docs/developer_resources.md‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,24 @@
22

33
## Tests
44

5-
Tests are run with pytest, e.g.,
5+
Run the unit and integration tests with the following command. We recommend using `make test` to run the tests that do not require simulation or compilation.
66

77
```bash
8-
poetry run pytest
8+
make test
9+
```
10+
11+
This runs tests that do not require simulation or compilation in parallel. The
12+
following additional targets run resource-intensive tests serially:
13+
14+
```bash
15+
# OpenModelica simulations and compilations in Docker
16+
make test-resource-intensive
17+
18+
# Fast tests followed by the Docker tests
19+
make test-all
20+
21+
# Dymola-only tests; requires a local Dymola installation and license
22+
make test-resources-intensive-dymola
923
```
1024

1125
## Snapshot Testing
@@ -52,12 +66,12 @@ Follow the instructions below in order to configure your local environment:
5266
- For developers, dependency management is through [Poetry](https://python-poetry.org/docs/). Poetry can be acquired by running `pip install poetry`.
5367
- If you haven't already installed a virtual environment, Poetry will automatically create a simplified environment for your project.
5468
- Move to the GMT root directory and run `poetry install` to install the dependencies.
55-
- Verify that everything is installed correctly by running `poetry run pytest -m 'not compilation and not simulation and not dymola'`. This will run all the unit and integration tests.
69+
- Verify that everything is installed correctly by running `make test`. This runs all unit and integration tests that do not require simulation or compilation.
5670
- Follow the instructions below to install pre-commit.
5771
- To confirm that models will build and simulate, you can run
5872

5973
```bash
60-
poetry run pytest -m 'not dymola' --cov-report term-missing --cov .
74+
make test-all
6175
```
6276

6377
The tests should all pass assuming the libraries, Docker, and all dependencies are installed correctly on your computer. Also, there will be a set

‎geojson_modelica_translator/model_connectors/load_connectors/_teaser_compat.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# :copyright (c) URBANopt, Alliance for Energy Innovation, LLC, and other contributors.
2+
# See also https://github.com/urbanopt/geojson-modelica-translator/blob/develop/LICENSE.md
3+
4+
"""Compatibility workarounds for importing TEASER."""
5+
6+
from pathlib import Path
7+
8+
# TEASER 1.3.1 creates this directory during import using a racy
9+
# exists-then-makedirs sequence. Ensure it exists before parallel processes
10+
# import TEASER simultaneously.
11+
Path.home().joinpath("TEASEROutput").mkdir(exist_ok=True)
12+
13+
from teaser.project import Project # noqa: E402
14+
15+
__all__ = ["Project"]

‎geojson_modelica_translator/model_connectors/load_connectors/teaser.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@
1111
import numpy as np
1212
from modelica_builder.model import Model
1313
from modelica_builder.package_parser import PackageParser
14-
from teaser.project import Project
1514

15+
from geojson_modelica_translator.model_connectors.load_connectors._teaser_compat import Project
1616
from geojson_modelica_translator.model_connectors.load_connectors.load_base import LoadBase
1717
from geojson_modelica_translator.utils import ModelicaPath, convert_c_to_k, copytree
1818

‎geojson_modelica_translator/modelica/lib/runner/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ docker build -t nrel/gmt-om-runner:<tag> .
2323

2424
The default tag will be `nrel/gmt-om-runner:4.0.0`, which is the default version used in the modelica_runner.py file.
2525

26+
If you run the image directly with a bind mount, note that the image runs as `root` so it can access the pre-installed Modelica libraries under `/root/.openmodelica`.
27+
28+
The GMT `ModelicaRunner` passes `HOST_UID`/`HOST_GID` into the container and `chown`s the mounted `/mnt/shared/<model_name>` directory at the end of the run so generated files are accessible on the host.
29+
2630
### Versioning
2731

2832
In GMT Runner Version 2.0.0 we detached the OM version from the GMT Runner version.

0 commit comments

Comments
 (0)