feat(accent): ship OpenJTalk as the sole accent backend (commercial)

.dockerignore

@@ -10,8 +10,18 @@ __pycache__/
 .venv/
 .env
 .env.*
-!.env.example
+tests/
 README.md
 Procfile
 render.yaml
 vercel.json
+
+# Local tooling, scratch, and editor state — never needed in the image
+.claude/
+.claude
+tmp/
+data/
+output/
+.idea/
+.vscode/
+.zed/

.github/workflows/cd.yml

@@ -8,16 +8,22 @@ on:  # yamllint disable-line rule:truthy
   workflow_dispatch:
 
 permissions:
+  actions: read
   contents: read
   packages: write
 
 concurrency:
-  group: cd
-  cancel-in-progress: true
+  group: cd-${{ github.ref }}
+  cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/v') }}
 
 jobs:
+  tests:
+    name: Validate source
+    uses: ./.github/workflows/tests.yml
+
   build-and-push:
     name: Build and push image to GHCR
+    needs: [tests]
     runs-on: ubuntu-latest
     env:
       REGISTRY: ghcr.io
@@ -26,6 +32,23 @@ jobs:
       # Must match deploy/compose.yml's api-tools image in jpcorrect-backend.
       IMAGE_NAME: sessatakuma/api-tools
     steps:
+      - name: Wait for earlier CD runs before publishing a release
+        if: startsWith(github.ref, 'refs/tags/v')
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          while true; do
+            blocking_run=$(gh api \
+              "repos/${GITHUB_REPOSITORY}/actions/workflows/cd.yml/runs?per_page=100" \
+              --jq ".workflow_runs[] | select(.run_number < ${GITHUB_RUN_NUMBER} and (.status == \"queued\" or .status == \"in_progress\")) | .id" \
+              | head -n 1)
+            if [[ -z "$blocking_run" ]]; then
+              break
+            fi
+            echo "Waiting for earlier CD run $blocking_run"
+            sleep 15
+          done
+
       - name: Checkout
         uses: actions/checkout@v6
 
@@ -54,16 +77,40 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
             type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
 
-      - name: Build and push
+      - name: Build image
         uses: docker/build-push-action@v7
         with:
           context: .
-          file: ./Dockerfile          # adjust if the api-tools Dockerfile path differs
+          file: ./Dockerfile
           platforms: linux/amd64
-          push: true
-          provenance: true
-          sbom: true
+          push: false
+          load: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+      - name: Smoke-test production lifecycle offline
+        run: |
+          image=$(printf '%s\n' '${{ steps.meta.outputs.tags }}' | head -n 1)
+          container=api-tools-release-smoke-${{ github.run_id }}
+          trap 'docker rm -f "$container" >/dev/null 2>&1 || true' EXIT
+          docker run -d --name "$container" --network none --read-only --tmpfs /tmp \
+            -e PYTHONDONTWRITEBYTECODE=1 \
+            "$image"
+          for attempt in $(seq 1 30); do
+            if docker exec "$container" python -c \
+              "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/openapi.json', timeout=2)"; then
+              break
+            fi
+            if [[ "$attempt" == 30 ]]; then
+              docker logs "$container"
+              exit 1
+            fi
+            sleep 1
+          done
+          docker exec "$container" python -c \
+            "import json, urllib.request; request=urllib.request.Request('http://127.0.0.1:8000/api/MarkAccent/', data=json.dumps({'text':'東京'}).encode(), headers={'Content-Type':'application/json'}); response=json.load(urllib.request.urlopen(request, timeout=30)); assert response['status'] == 200 and response['result']"
+
+      - name: Push smoke-tested image
+        run: docker push --all-tags "${REGISTRY}/${IMAGE_NAME}"

.github/workflows/docker-build.yml

@@ -0,0 +1,69 @@
+---
+name: Docker Build Check
+
+# Build the image on PRs (no push) so a broken Dockerfile — most importantly
+# the dict-bake layers that make the runtime self-contained/offline
+# (open_jtalk_dic via pyopenjtalk, and the UniDic dict — ~700 MB compressed
+# download, ~1.3 GB installed — from scripts/download_unidic.sh) — fails the
+# PR instead of the release build.
+# The smoke-test step then runs the built image with `--network none` and
+# exercises BOTH baked dictionaries: the OpenJTalk open_jtalk_dic (via
+# pyopenjtalk) AND the fugashi/UniDic dicdir (via the tokeniser). A broken
+# or missing download of either dict fails the gate offline.
+on:  # yamllint disable-line rule:truthy
+  workflow_call:
+  pull_request:
+    branches: ["main", "dev"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docker-build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build image (no push)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64
+          push: false
+          load: true
+          tags: api-tools:pr-${{ github.event.pull_request.number || github.run_id }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Smoke-test the accent pipeline in the built image
+        run: |
+          image=api-tools:pr-${{ github.event.pull_request.number || github.run_id }}
+          container=api-tools-smoke-${{ github.run_id }}
+          trap 'docker rm -f "$container" >/dev/null 2>&1 || true' EXIT
+          docker run -d --name "$container" --network none --read-only --tmpfs /tmp \
+            -e PYTHONDONTWRITEBYTECODE=1 \
+            "$image"
+          for attempt in $(seq 1 30); do
+            if docker exec "$container" python -c \
+              "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/openapi.json', timeout=2)"; then
+              break
+            fi
+            if [[ "$attempt" == 30 ]]; then
+              docker logs "$container"
+              exit 1
+            fi
+            sleep 1
+          done
+          docker exec "$container" python -c \
+            "import json, urllib.request; request=urllib.request.Request('http://127.0.0.1:8000/api/MarkAccent/', data=json.dumps({'text':'東京'}).encode(), headers={'Content-Type':'application/json'}); response=json.load(urllib.request.urlopen(request, timeout=30)); assert response['status'] == 200 and response['result']"

.github/workflows/tests.yml

@@ -0,0 +1,51 @@
+---
+name: Tests
+
+# Runs lint, type-check, and the unit test suite on every PR. Unit tests mock
+# the UniDic boundary, while the mandatory Docker build check exercises both
+# real dictionaries offline. This keeps the fast gate independent of the
+# ~1.3 GB UniDic download without losing integration coverage.
+on:  # yamllint disable-line rule:truthy
+  workflow_call:
+  pull_request:
+    branches: ["main", "dev"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Lint, type-check, pytest
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.9.0"
+
+      - name: Install build toolchain for pyopenjtalk
+        # pyopenjtalk ships source-only and builds its bundled OpenJTalk/
+        # HTS-engine C++ from sdist during `uv sync` (same as the Dockerfile).
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends build-essential cmake
+
+      - name: Install dependencies
+        run: uv sync --frozen
+
+      - name: Ruff
+        run: uv run ruff check .
+
+      - name: Mypy
+        run: uv run mypy api
+
+      - name: Pytest
+        run: uv run pytest -q

.gitignore

@@ -215,5 +215,14 @@ __marimo__/
 # Streamlit
 .streamlit/secrets.toml
 
-# Secrest files
-secret*
+# Secret files
+secret*
+
+# Local data / generated output
+data/
+output/
+
+# AI assistant local config (not shared)
+CLAUDE.md
+AGENTS.md
+.claude

Dockerfile

@@ -2,20 +2,51 @@ FROM python:3.11-slim AS builder
 
 WORKDIR /app
 
-# Install uv for fast dependency management (pinned for reproducible builds)
-RUN pip install --no-cache-dir "uv==0.9.0"
+# Install uv for fast dependency management (pinned for reproducible builds),
+# curl for downloading the NINJAL UniDic archive (scripts/download_unidic.sh
+# line ~74), and unzip for extracting it (Deflate64-compressed).
+#
+# build-essential (g++/make) and cmake are required to compile pyopenjtalk:
+# it ships source-only on PyPI (no wheels for any arch) and builds its bundled
+# OpenJTalk/HTS-engine C++ from sdist during `uv sync` below. Without them the
+# build fails at cmake configure ("no CXX compiler"). This is a builder-stage
+# cost only — the toolchain never lands in the slim runtime image.
+RUN pip install --no-cache-dir "uv==0.9.0" \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+       curl unzip build-essential cmake \
+    && rm -rf /var/lib/apt/lists/*
 
 # Copy dependency files first for better layer caching
 COPY pyproject.toml uv.lock ./
 
 # Install dependencies using uv
 RUN uv sync --frozen --no-dev --no-install-project
 
+# Download the OpenJTalk dictionary (open_jtalk_dic_utf_8 ~23 MB) into the
+# venv at build time. pyopenjtalk fetches it lazily on first frontend call;
+# baking it here means the runtime (unprivileged appuser, read-only app dir,
+# possibly offline) never has to download it. It lands inside the pyopenjtalk
+# package under /app/.venv, which is copied wholesale into the final image.
+# Own layer (gated by uv.lock) so app-code edits don't bust the download.
+RUN /app/.venv/bin/python -c "import pyopenjtalk; pyopenjtalk.extract_fullcontext('テスト')"
+
+# Download and install the UniDic dictionary into the venv so the image is
+# self-contained — the `unidic` pip package ships the loader but not the
+# dicdir, and fugashi.Tagger() fails at runtime without it. UniDic CWJ is
+# a ~700 MB compressed download that expands to ~1.3 GB installed.
+# Uses scripts/download_unidic.sh so local dev and Docker share the same
+# download logic.  Kept as its own layer (before COPY . .) so app-code
+# edits don't bust the download cache; only re-runs when the script or
+# uv.lock changes.
+COPY scripts/download_unidic.sh scripts/
+RUN VIRTUAL_ENV=/app/.venv bash scripts/download_unidic.sh
+
 COPY . .
 
 # Compile only our app code (skip .venv) and drop the .py sources
-RUN python -m compileall -b -q main.py api config \
-    && find main.py api config -name "*.py" -delete
+RUN python -m compileall -b -q main.py api \
+    && find main.py api -name "*.py" -delete
 
 FROM python:3.11-slim
 
@@ -30,11 +61,16 @@ WORKDIR /app
 COPY --from=builder /app/.venv /app/.venv
 COPY --from=builder /app/main.pyc /app/main.pyc
 COPY --from=builder /app/api /app/api
-COPY --from=builder /app/config /app/config
 
 # Set PATH to use venv binaries
 ENV PATH="/app/.venv/bin:$PATH"
 ENV PYTHONUNBUFFERED=1
+# Don't attempt to write .pyc caches at runtime — app code is already compiled
+# to .pyc in the builder, and this lets the container run with a read-only
+# rootfs (see compose.deploy.yml) without Python tripping on cache writes.
+ENV PYTHONDONTWRITEBYTECODE=1
+# Local timezone so log timestamps match the deployment (tzdata ships in slim).
+ENV TZ=Asia/Taipei
 
 EXPOSE 8000