|
| 1 | +# SPDX-License-Identifier: MIT |
| 2 | + |
| 3 | +"""The projects that the integration tests build, and the cache of their archives. |
| 4 | +
|
| 5 | +``tests/test_integration.py`` imports this module, and CI runs it as a script to fill the cache once for the whole test |
| 6 | +matrix. It uses the standard library only, so CI can run it before an install. |
| 7 | +
|
| 8 | +""" |
| 9 | + |
| 10 | +from __future__ import annotations |
| 11 | + |
| 12 | +import http.client |
| 13 | +import os |
| 14 | +import shutil |
| 15 | +import time |
| 16 | +import urllib.request |
| 17 | + |
| 18 | +from pathlib import Path |
| 19 | + |
| 20 | + |
| 21 | +INTEGRATION_SOURCES: dict[str, tuple[str, str]] = { |
| 22 | + 'dateutil': ('dateutil/dateutil', '2.9.0'), |
| 23 | + 'pip': ('pypa/pip', '25.0.1'), |
| 24 | + 'Solaar': ('pwr-Solaar/Solaar', '1.1.14'), |
| 25 | + 'flit': ('pypa/flit', '3.12.0'), |
| 26 | +} |
| 27 | + |
| 28 | +STORE_DIR = Path(__file__).resolve().parents[1] / '.integration-sources' |
| 29 | +ATTEMPTS = 3 |
| 30 | +NETWORK_ERRORS = (OSError, http.client.HTTPException) |
| 31 | + |
| 32 | + |
| 33 | +def download_archive(name: str) -> tuple[Path, str]: |
| 34 | + """Put the archive in the cache and return its path and version. Hold a lock around this to make it parallel safe.""" |
| 35 | + github_org_repo, version = INTEGRATION_SOURCES[name] |
| 36 | + target = STORE_DIR / f'{name}-{version}.tar.gz' |
| 37 | + if target.exists(): |
| 38 | + return target, version |
| 39 | + |
| 40 | + url = f'https://github.com/{github_org_repo}/archive/{version}.tar.gz' |
| 41 | + STORE_DIR.mkdir(exist_ok=True) |
| 42 | + # write to a temporary name, because GitHub can drop a large archive request and leave a truncated file behind |
| 43 | + partial = target.with_suffix('.part') |
| 44 | + for attempt in range(1, ATTEMPTS + 1): |
| 45 | + try: |
| 46 | + with urllib.request.urlopen(url) as request, partial.open('wb') as file_handler: |
| 47 | + shutil.copyfileobj(request, file_handler) |
| 48 | + except NETWORK_ERRORS as exception: # noqa: PERF203 |
| 49 | + partial.unlink(missing_ok=True) |
| 50 | + if attempt == ATTEMPTS: |
| 51 | + raise |
| 52 | + print(f'attempt {attempt}/{ATTEMPTS} failed for {url}: {exception}') |
| 53 | + time.sleep(2**attempt) |
| 54 | + else: |
| 55 | + break |
| 56 | + os.replace(partial, target) |
| 57 | + return target, version |
| 58 | + |
| 59 | + |
| 60 | +def main() -> None: |
| 61 | + failed = [] |
| 62 | + for name in INTEGRATION_SOURCES: |
| 63 | + try: |
| 64 | + print(f'ready: {download_archive(name)[0].name}') |
| 65 | + except NETWORK_ERRORS as exception: # noqa: PERF203 |
| 66 | + print(f'failed: {name}: {exception}') |
| 67 | + failed.append(name) |
| 68 | + if failed: |
| 69 | + print(f'::warning::could not pre-download {", ".join(failed)}; the tests download these themselves') |
| 70 | + |
| 71 | + |
| 72 | +if __name__ == '__main__': |
| 73 | + main() |
0 commit comments