Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,23 @@ jobs:
working-directory: ./ctex
run: |
set -x
l3build check -q "$L3BUILD_EXTRA_OPTIONS"
l3build check -c test/config-cmap -q "$L3BUILD_EXTRA_OPTIONS"
l3build check -c test/config-contrib -q "$L3BUILD_EXTRA_OPTIONS"
l3build check -c test/config-ctxdoc -q "$L3BUILD_EXTRA_OPTIONS" || {
# 并行 4 engine, 实测 wall-clock ~5-8min (串行旧法 ~13-17min).
# 进程间隔离方式: wrapper 给每个 engine 一份 tmp/parallel-check/<engine>/
# 独立子工作目录 (git ls-files | tar snapshot), 各自 build/check 相互
# 不争抢. CONFIGS 透过 env 喂给 Phase 2 串行跑.
CONFIGS="test/config-cmap test/config-contrib test/config-ctxdoc" \
../scripts/check-parallel.sh "$L3BUILD_EXTRA_OPTIONS" || {
echo "::group::patch-health raw log"
cat build/check-test/config-ctxdoc/patch-health.log 2>/dev/null || echo "raw log not found"
# patch-health.log 可能在两个位置:
# - build/check-test/config-ctxdoc/patch-health.log (config 串行跑, 默认 testdir 子目录)
# - tmp/parallel-check/<engine>/ctex/build/check/.../patch-health.log
# (主测在 wrapper 独立子工作目录跑)
# 递归找两类位置.
for f in build/check-test/config-ctxdoc/patch-health.log \
../tmp/parallel-check/*/ctex/build/check/patch-health.log \
build/check-*/patch-health.log; do
[ -f "$f" ] && { echo "=== $f ==="; cat "$f"; }
done
echo "::endgroup::"
exit 1
}
Expand Down
25 changes: 23 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
# doc —— l3build doc:生成 PDF 文档
# unpack —— l3build unpack:解包 .dtx 得到 .sty/.cls/.cfg 等运行时文件
# ctan —— l3build ctan:打 ctan 发布包(含 doc + tds.zip)
# check —— l3build check:跑回归测试(单包 20min+,本地慎用,默认 CI 跑)
# check —— l3build check:跑回归测试(ctex 经 check-ctex 4-engine 并行 ~8min,
# 串行旧法 ~20min;小包通常几十秒)
# clean —— l3build clean
#
# 另: hooks / check-pr-ci 是 git workflow 入口, build-gbk2uni 走子 Makefile.
Expand All @@ -19,7 +20,7 @@ L3BUILD_PKGS := xeCJK ctex CJKpunct xCJK2uni xpinyin zhlineskip \

VERBS := doc unpack ctan check clean

.PHONY: help hooks check-pr-ci \
.PHONY: help hooks check-pr-ci check-ctex-serial \
$(VERBS) \
$(foreach v,$(VERBS),$(v)-all) \
$(foreach v,$(VERBS),$(addprefix $(v)-,$(L3BUILD_PKGS))) \
Expand Down Expand Up @@ -68,6 +69,10 @@ clean-all: $(addprefix clean-,$(L3BUILD_PKGS)) clean-gbk2uni

# ── 单包 target: $(verb)-$(pkg) → cd $(pkg) && l3build $(verb) ─────────────
# 用 pattern rule 写法, 一条规则覆盖全部 verb × pkg 笛卡儿积.
#
# 注: check 的 ctex / xeCJK 等"大包"的 pattern 规则会被下方的 per-pkg
# override (check-ctex 走 scripts/check-parallel.sh 多 engine 并行).
# 其他小包仍走默认串行规则.
define L3BUILD_PKG_RULES
$(addprefix doc-, $(1)): doc-%: ; cd $$* && l3build doc
$(addprefix unpack-, $(1)): unpack-%: ; cd $$* && l3build unpack
Expand All @@ -77,6 +82,22 @@ $(addprefix clean-, $(1)): clean-%: ; cd $$* && l3build clean
endef
$(eval $(call L3BUILD_PKG_RULES,$(L3BUILD_PKGS)))

# ── check 大包并行加速 (override 上方 pattern rule) ────────────────────────
# ctex 默认跑 4 engine, 串行 ~20min wall-clock; 并行后 ~8min. 走
# scripts/check-parallel.sh, 给每个 engine 准备一份独立子工作目录
# (tmp/parallel-check/<engine>/, git ls-files + tar 快照), 各 engine 进程在
# 自己的目录下跑 l3build check, 互不争抢. 同步 ctex 主测 + 各 -c config 三个
# (cmap / contrib / ctxdoc) 都跑.
check-ctex: ## ctex 4-engine 并行 l3build check (~8min)
cd ctex && CONFIGS="test/config-cmap test/config-contrib test/config-ctxdoc" \
../scripts/check-parallel.sh

check-ctex-serial: ## ctex 串行 l3build check (~20min, 用于调试并行)
cd ctex && l3build check
cd ctex && l3build check -c test/config-cmap
cd ctex && l3build check -c test/config-contrib
cd ctex && l3build check -c test/config-ctxdoc

# ── gbk2uni 走子 Makefile ──────────────────────────────────────────────────
doc-gbk2uni unpack-gbk2uni:
$(MAKE) -C gbk2uni
Expand Down
10 changes: 3 additions & 7 deletions llmdoc/reference/build-and-test.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
- `make hooks`:一次性安装 git hooks(`git config core.hooksPath .githooks`)。
- `make check-pr-ci`:手动触发 PR CI watch + review 抓取(同 `pre-push` 调用的 `./.githooks/check-pr-ci.sh`)。

注意 `make check`全包回归)单包即需 20min+,本地慎用,默认仍由 CI 跑。hook 的详细说明见 `.githooks/README.md`。
注意 `make check`(全包回归)单包动辄 8min+(`make check-ctex` 经 4-engine 并行已从 ~20min 压到 ~8min),本地按需用。hook 的详细说明见 `.githooks/README.md`。

## `support/build-config.lua` 的角色

Expand Down Expand Up @@ -245,12 +245,8 @@ PR #799 暴露了一个稳定信号:`xeCJK/testfiles/listings-hash01.lvt` 新

CI 中当前执行的测试步骤是:

- `Test ctex`:在 `./ctex` 运行
- `l3build check -q`
- `l3build check -c test/config-cmap -q`
- `l3build check -c test/config-contrib -q`
- `l3build check -c test/config-ctxdoc -q`
- `Test xeCJK`:在 `./xeCJK` 运行 `l3build check -q`
- `Test ctex`:在 `./ctex` 运行 `../scripts/check-parallel.sh`,**4 engine 并行**,`CONFIGS` env 透传 `test/config-cmap test/config-contrib test/config-ctxdoc` 让 4 个 engine 各跑主测 + 3 个 config. 实测 wall-clock ~5–7min(串行 ~20min). 失败时自动 dump 各 engine 的 `patch-health.log`.
- `Test xeCJK`:在 `./xeCJK` 运行 `l3build check -q`(单 engine xetex, 不需并行)
- `Test zhnumber`:在 `./zhnumber` 运行 `l3build check -q`
- `Test CJKpunct`:在 `./CJKpunct` 运行 `l3build check -q`
- `Test zhlineskip`:在 `./zhlineskip` 运行 `l3build check -q`
Expand Down
172 changes: 172 additions & 0 deletions scripts/check-parallel.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
#!/usr/bin/env bash
# scripts/check-parallel.sh — 加速 l3build check, 多 engine 并行.
#
# 设计: l3build 内部用相对路径写 .tlg, 让 build/check 多一层 (如
# build/check-<engine>) 会让 .log 里的相对路径 "../foo" 而非 "foo", 与
# .tlg 不匹配, 整个 baseline 报 diff.
#
# 解决: 每个 engine 进程在**独立的子工作目录**里跑 (整个包 cp 一份),
# testdir 仍是 build/check (相对路径不变, .tlg 兼容).
#
# 用法 (在 ctex/ 等包目录内执行):
# ENGINES="pdftex xetex luatex uptex"
# CONFIGS="test/config-cmap test/config-contrib test/config-ctxdoc"
# ../scripts/check-parallel.sh [extra-l3build-args...]
#
# bash 兼容: 不依赖 bash 4+ 特性 (declare -A 等). macOS 自带 bash 3.2 可跑.
set -uo pipefail

ENGINES="${ENGINES:-pdftex xetex luatex uptex}"
CONFIGS="${CONFIGS:-}"
# 用数组保 EXTRA_ARGS, 避免单字符串模式下 word splitting 出错 (例如未来
# 有用户传 --first foo 这种带空格的参数). bash 3.2 安全: 不依赖 4+ 特性.
EXTRA_ARGS=("$@")

if [ ! -f build.lua ]; then
echo "ERROR: 必须在含 build.lua 的包目录里跑这个脚本" >&2
exit 1
fi

pkg_dir="$(pwd)"
pkg_name="$(basename "$pkg_dir")"
parent_dir="$(dirname "$pkg_dir")"

# 工作目录在仓库根的 tmp/parallel-check/<engine>/, 每个 engine 一份完整的
# 包目录 cp. 复用 git ls-files 保证只 cp 受版本控制的文件 (不带 build/).
work_root="${parent_dir}/tmp/parallel-check"
mkdir -p "$work_root"
# 失败时**保留** work_root, 让上层 (test.yml 的 || 块) 能 cat 各 engine
# 的 patch-health.log / *.log 做诊断. 成功时再删.
cleanup() { [ "$?" -eq 0 ] && rm -rf "$work_root"; }
trap cleanup EXIT

# 用 git archive | tar 高效生成快照: 比 cp -r 快 (无需扫 build/ 等大目录).
# 需要包目录在 git 控制下.
# 注: 不吞 stderr — snapshot 失败会让 l3build 在难以诊断的位置炸, 让错误
# 直接冒出来更省事.
snapshot_pkg() {
local dest="$1"
mkdir -p "$dest"
# 包根目录的所有 git 跟踪文件; 排除 build/ (l3build 工作区).
(cd "$pkg_dir" && git ls-files -z | tar --null -T- -cf -) \
| tar -xf - -C "$dest"
}

# ctex 的 checkdeps = {"../xeCJK", "../zhnumber"}. 包目录在 work_root/<engine>/<pkg>
# 跑时, ../<dep> 应当是 work_root/<engine>/<dep>. 所以 deps 也要 cp.
snapshot_dep() {
local dep="$1"
local dest_root="$2"
local dep_src="${parent_dir}/${dep}"
local dep_dest="${dest_root}/${dep}"
[ -d "$dep_src" ] || return 0
mkdir -p "$dep_dest"
(cd "$dep_src" && git ls-files -z | tar --null -T- -cf -) \
| tar -xf - -C "$dep_dest"
}

# 从 build.lua 抠出 checkdeps. ctex 是 {"../xeCJK", "../zhnumber"}.
CHECKDEPS=$(awk '/^checkdeps/,/}/ {
while (match($0, /"\.\.\/[^"]+"/)) {
s = substr($0, RSTART+4, RLENGTH-5)
print s
$0 = substr($0, RSTART+RLENGTH)
}
}' build.lua | sort -u)

# 始终需要的兄弟目录: support/ 含共享 build-config.lua, 被各包的 build.lua
# 末尾 dofile. 即使 checkdeps 没列也必须 cp.
SIBLING_ALWAYS="support"

# Phase 1: 主测多 engine 并行
echo "==================== Phase 1: 主测 (并行 engines: $ENGINES) ===================="

# 为每个 engine 准备一个独立子工作目录, 后台跑.
declare_pids() { :; } # no-op marker
pid_list="" # space-separated; 用字符串避免 declare -A
engine_for_pid="" # "pid:engine pid:engine ..." 格式
exit_for_engine="" # "engine:rc engine:rc ..."

for engine in $ENGINES; do
engine_workdir="${work_root}/${engine}/${pkg_name}"
snapshot_pkg "$engine_workdir"
for dep in $CHECKDEPS; do
snapshot_dep "$dep" "${work_root}/${engine}"
done
for sib in $SIBLING_ALWAYS; do
snapshot_dep "$sib" "${work_root}/${engine}"
done

(
cd "$engine_workdir"
# bash 3.2 安全: 空数组用 ${arr[@]+"${arr[@]}"} 形式避免 set -u 炸.
# 用 awk 而非 sed -u 加前缀: awk 默认按行刷, 跨平台 (sed -u 是 GNU 扩展,
# windows git bash 的 sed 不支持).
l3build check -e "${engine}" -q ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} 2>&1 \
| awk -v prefix="[${engine}] " '{ print prefix $0; fflush() }'
exit "${PIPESTATUS[0]}"
) &
pid=$!
pid_list="$pid_list $pid"
engine_for_pid="$engine_for_pid ${pid}:${engine}"
done

MAIN_FAIL=0
for pid in $pid_list; do
wait "$pid"
rc=$?
# 从 engine_for_pid 找回对应 engine
engine=$(echo "$engine_for_pid" | tr ' ' '\n' | grep "^${pid}:" | cut -d: -f2)
exit_for_engine="$exit_for_engine ${engine}:${rc}"
[ "$rc" -ne 0 ] && MAIN_FAIL=1
done

# Phase 2: -c configs 串行, 在原 pkg_dir 跑 (configs 不并行, 相对路径无忧).
CONFIG_FAIL=0
config_exits=""
if [ -n "$CONFIGS" ]; then
echo ""
echo "==================== Phase 2: configs (串行: $CONFIGS) ===================="
for c in $CONFIGS; do
echo "--- config: $c ---"
if l3build check -c "$c" -q ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}; then
config_exits="$config_exits ${c}:0"
else
rc=$?
config_exits="$config_exits ${c}:${rc}"
CONFIG_FAIL=1
fi
done
fi

# 汇总
echo ""
echo "==================== check-parallel summary ===================="
echo "Phase 1 (main test, parallel by engine):"
for entry in $exit_for_engine; do
engine="${entry%:*}"
rc="${entry#*:}"
if [ "$rc" -eq 0 ]; then
echo " ✓ ${engine}"
else
echo " ✗ ${engine} (rc=${rc})"
fi
done
if [ -n "$CONFIGS" ]; then
echo "Phase 2 (configs, serial):"
for entry in $config_exits; do
c="${entry%:*}"
rc="${entry#*:}"
if [ "$rc" = "0" ]; then
echo " ✓ ${c}"
else
echo " ✗ ${c} (rc=${rc})"
fi
done
fi
echo "================================================================"

if [ "$MAIN_FAIL" -ne 0 ] || [ "$CONFIG_FAIL" -ne 0 ]; then
exit 1
fi
exit 0
Loading