Merge pull request #3776 from camptocamp/revert-3767-renovate/master-own-packages

Revert "Update dependency c2casgiutils to v0.14.0 (master)"

Author: sbrunner

.github/workflows/main.yaml

@@ -28,13 +28,6 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: camptocamp/initialise-gopass-summon-action@17703c87720ba3766abc90da60b437783b530d93 # v2.0.1
-        with:
-          ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}}
-          github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}}
-          patterns: pypi docker
-        if: env.HAS_SECRETS == 'HAS_SECRETS'
-
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
       - run: python3 -m pip install --requirement=ci/requirements.txt
 

.github/workflows/rebuild.yaml

@@ -4,6 +4,9 @@ on:
   schedule:
     - cron: 30 2 * * *
 
+permissions:
+  contents: read
+
 jobs:
   rebuild:
     name: Rebuild

.pre-commit-config.yaml

@@ -104,7 +104,7 @@ repos:
     hooks:
       - id: shellcheck
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.37.4
+    rev: 0.38.0
     hooks:
       - id: check-github-actions
       - id: check-github-workflows
@@ -121,7 +121,7 @@ repos:
     hooks:
       - id: ripsecrets
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.16.1
+    rev: v0.16.2
     hooks:
       - id: ruff-format
   - repo: https://github.com/prospector-dev/prospector
@@ -140,8 +140,8 @@ repos:
         additional_dependencies:
           - prospector-profile-duplicated==1.12.0 # pypi
           - prospector-profile-utils==1.27.0 # pypi
-          - pylint[spelling]==4.0.6 # pypi
-          - ruff==0.16.1 # pypi
+          - pylint[spelling]==4.0.7 # pypi
+          - ruff==0.16.2 # pypi
       - id: prospector
         args:
           - --die-on-tool-error
@@ -150,8 +150,8 @@ repos:
           - --profile=utils:pre-commit
         additional_dependencies:
           - prospector-profile-utils==1.27.0 # pypi
-          - ruff==0.16.1 # pypi
-          - pylint[spelling]==4.0.6 # pypi
+          - ruff==0.16.2 # pypi
+          - pylint[spelling]==4.0.7 # pypi
   - repo: https://github.com/sbrunner/jsonschema-validator
     rev: 1.0.0
     hooks:
@@ -163,7 +163,7 @@ repos:
             |jsonschema\-gentypes\.yaml
           )$
   - repo: https://github.com/renovatebot/pre-commit-hooks
-    rev: 44.14.3
+    rev: 44.17.2
     hooks:
       - id: renovate-config-validator
   - repo: https://github.com/sbrunner/python-versions-hook

CHANGELOG.md

@@ -38,6 +38,7 @@
 - Fix PostgreSQL admin retry action so it requeues only errored meta tiles instead of restarting full queue seeding for the whole job.
 - Add Python 3.14 support and use the PyPI Mapnik bindings instead of the removed Ubuntu `python3-mapnik` package.
 - Mark PostgreSQL metatile queue entries as `error` (instead of deleting them) when a child tile fails during write/store, so generation failures are visible in job status.
+- Allow PostgreSQL queue workers to pick meta-tiles from any started job when the current job has only pending meta-tiles, so workers are not blocked waiting for other workers to finish.
 
 Environment variable migration (legacy -> new)
 

poetry.lock

@@ -23,7 +23,7 @@ version = "0.0.0"
 [tool.poetry.dependencies]
 # Minimal version should also be set in the jsonschema-gentypes.yaml file
 python = ">=3.12,<3.15"
-c2casgiutils = { version = "0.14.0", extras = ["all"] }
+c2casgiutils = { version = "0.13.4", extras = ["all"] }
 fastapi = {extras = ["standard"], version = "0.141.1"}
 python-dateutil = "2.9.0.post0"
 tilecloud = { version = "1.13.5", extras = ["azure", "aws", "redis", "wsgi"] }
@@ -43,7 +43,7 @@ sqlalchemy = { version = "2.0.51", extras = ["asyncio"] }
 anyio = "4.14.2"
 asyncpg = "0.31.0"
 html-sanitizer = "2.6.0"
-pydantic-settings = "2.14.2"
+pydantic-settings = "2.15.0"
 mapnik = "4.3.0"
 prometheus-fastapi-instrumentator = "8.1.0"
 
@@ -61,8 +61,8 @@ types-requests = "2.33.0.20260712"
 pytest-asyncio = "1.4.0"
 pytest-check = "2.9.1"
 types-pyyaml = "6.0.12.20260724"
-ruff = "0.16.1"
-pylint = "4.0.6"
+ruff = "0.16.2"
+pylint = "4.0.7"
 
 [tool.poetry-dynamic-versioning]
 enable = true

pyproject.toml

@@ -729,14 +729,7 @@ async def list(self) -> AsyncIterator[Tile]:
 
             if self.jobs:
                 config_filename = None
-                try:
-                    config_filename = next(iter(self.jobs))
-                except StopIteration:
-                    pass
-
-                if config_filename is None:
-                    continue
-                job_id = self.jobs.pop(config_filename)
+                job_id = None
                 try:
                     if settings.postgresql.objgraph_postgresql:
                         for generation in range(3):
@@ -753,13 +746,15 @@ async def list(self) -> AsyncIterator[Tile]:
                     async with self.SessionMaker() as session:
                         result = await session.execute(
                             select(Queue)
+                            .join(Job, Queue.job_id == Job.id)
                             .with_for_update(of=Queue, skip_locked=True)
-                            .order_by(Queue.id.asc())
-                            .where(and_(Queue.status == _STATUS_CREATED, Queue.job_id == job_id)),
+                            .order_by(Job.created_at.asc(), Queue.id.asc())
+                            .where(and_(Queue.status == _STATUS_CREATED, Job.status == _STATUS_STARTED)),
                         )
                         sqlalchemy_tile = result.scalar()
                         if sqlalchemy_tile is None:
                             continue
+                        job_id = sqlalchemy_tile.job_id
                         sqlalchemy_tile.status = _STATUS_PENDING
                         now = datetime.datetime.now(tz=datetime.UTC)
                         sqlalchemy_tile.started_at = now
@@ -779,10 +774,14 @@ async def list(self) -> AsyncIterator[Tile]:
                             postgresql_id=sqlalchemy_tile.id,
                         )
                         await session.commit()
+                    config_filename = next(
+                        (k for k, v in self.jobs.items() if v == job_id),
+                        None,
+                    )
                     yield meta_tile
                 except Exception:  # pylint: disable=broad-except
                     _LOGGER.exception("Error while reading from Postgres")
-                    _READ_ERROR_COUNTER.labels(job_id, config_filename).inc()
+                    _READ_ERROR_COUNTER.labels(job_id or -1, config_filename or "unknown").inc()
                     await asyncio.sleep(1)
 
     async def put_one(self, tile: Tile) -> Tile:

tilecloud_chain/store/postgresql.py

@@ -363,3 +363,52 @@ async def test_controller_status_postgresql(SessionMaker: sessionmaker) -> None:
                 session.query(Job).filter(Job.id == job_id).delete()
                 session.commit()
         await gene.close()
+
+
+@pytest.mark.asyncio
+async def test_list_picks_next_job_when_first_job_has_only_pending_metatiles(
+    SessionMaker: sessionmaker,
+    tilestore: PostgresqlTileStore,
+):
+    """When a job has only pending meta-tiles, list() should pick from the next job."""
+    with SessionMaker() as session:
+        for job in session.query(Job).filter(Job.name.in_(["test-job-1", "test-job-2"])).all():
+            session.delete(job)
+        session.commit()
+
+    await tilestore.create_job("test-job-1", "generate-tiles", Path("config.yaml"))
+    await tilestore.create_job("test-job-2", "generate-tiles", Path("config.yaml"))
+
+    with SessionMaker() as session:
+        job1 = session.query(Job).filter(Job.name == "test-job-1").one()
+        job2 = session.query(Job).filter(Job.name == "test-job-2").one()
+        job1.status = _STATUS_STARTED
+        job2.status = _STATUS_STARTED
+        job1_id = job1.id
+        job2_id = job2.id
+        session.commit()
+
+    await tilestore.put_one(
+        Tile(TileCoord(0, 0, 0), metadata={"job_id": job1_id}),
+    )
+    await tilestore.put_one(
+        Tile(TileCoord(1, 0, 0), metadata={"job_id": job2_id}),
+    )
+    await tilestore.close()
+
+    with SessionMaker() as session:
+        session.query(Queue).filter(Queue.job_id == job1_id).update(
+            {Queue.status: _STATUS_PENDING},
+        )
+        session.commit()
+
+    await tilestore._maintenance()
+
+    tile = await anext(tilestore.list())
+    assert tile.tilecoord.z == 1
+    assert tile.metadata["job_id"] == job2_id
+
+    with SessionMaker() as session:
+        session.query(Queue).filter(Queue.job_id.in_([job1_id, job2_id])).delete()
+        session.query(Job).filter(Job.id.in_([job1_id, job2_id])).delete()
+        session.commit()

tilecloud_chain/tests/test_postgresql.py

@@ -42,7 +42,6 @@
 import pyproj
 from c2casgiutils import auth
 from c2casgiutils import config as c2c_config
-from c2casgiutils.config import GitHubAccessType
 from fastapi import Depends, FastAPI, Form, HTTPException, Query, Request
 from fastapi.responses import HTMLResponse
 from fastapi.templating import Jinja2Templates
@@ -109,12 +108,11 @@ async def _get_access(
         return True
 
     auth_config = config.config.get("authentication", {})
-    github_access_type = auth_config.get("github_access_type", "pull")
     return await auth.check_access_config(
         auth_info,
         auth.AuthConfig(
             github_repository=auth_config.get("github_repository", ""),
-            github_access_type_admin=GitHubAccessType(github_access_type),
+            github_access_type=auth_config.get("github_access_type", ""),
         ),
     )