Skip to content

Commit fb43bb2

Browse files
committed
Cap pytest-xdist's auto worker count to the container CPU quota
A container's CPU limit is a CFS quota, which nproc and sched_getaffinity do not see, so --numprocesses=auto sizes itself from the whole machine's core count. On an 8 vCPU OSDC pod that meant 39+ workers, which exhausted the memory limit and got the container OOMKilled. pytest-parallelism.sh reads the quota from cgroup v2 or v1 and exports PYTEST_XDIST_AUTO_NUM_WORKERS, falling back to nproc where there is no quota, so it is a no-op off OSDC. Sourced from the two scripts here that run --numprocesses=auto; backends/arm/test/test_arm_backend.sh follows in the pull.yml change. Authored with Claude Code.
1 parent 878e10a commit fb43bb2

3 files changed

Lines changed: 43 additions & 0 deletions

File tree

.ci/scripts/pytest-parallelism.sh

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
# Pin pytest-xdist's `auto` worker count. A container's CPU limit is a CFS quota,
9+
# which nproc and sched_getaffinity do not see, so `auto` fans out to the whole
10+
# machine's core count and the workers exhaust the container's memory limit.
11+
12+
pytest_num_workers() {
13+
local quota period
14+
if [[ -r /sys/fs/cgroup/cpu.max ]]; then # cgroup v2
15+
read -r quota period < /sys/fs/cgroup/cpu.max
16+
if [[ "${quota}" != "max" ]] && (( period > 0 )); then
17+
echo $(( quota / period ))
18+
return
19+
fi
20+
fi
21+
if [[ -r /sys/fs/cgroup/cpu/cpu.cfs_quota_us ]]; then # cgroup v1
22+
quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us)
23+
period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us)
24+
if (( quota > 0 && period > 0 )); then
25+
echo $(( quota / period ))
26+
return
27+
fi
28+
fi
29+
nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1
30+
}
31+
32+
if [[ -z "${PYTEST_XDIST_AUTO_NUM_WORKERS:-}" ]]; then
33+
workers=$(pytest_num_workers)
34+
(( workers < 1 )) && workers=1
35+
export PYTEST_XDIST_AUTO_NUM_WORKERS="${workers}"
36+
echo "PYTEST_XDIST_AUTO_NUM_WORKERS=${PYTEST_XDIST_AUTO_NUM_WORKERS}"
37+
fi

.ci/scripts/test_backend.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
# LICENSE file in the root directory of this source tree.
88
set -eux
99

10+
# Cap pytest-xdist's `auto` workers to the container's CPU quota.
11+
source .ci/scripts/pytest-parallelism.sh
12+
1013
SUITE=$1
1114
FLOW=$2
1215
ARTIFACT_DIR=$3

.ci/scripts/unittest-linux-cmake.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
# LICENSE file in the root directory of this source tree.
88
set -eux
99

10+
# Cap pytest-xdist's `auto` workers to the container's CPU quota.
11+
source .ci/scripts/pytest-parallelism.sh
12+
1013
# Some ARM/TOSA-adjacent tests import modules that require tosa_serializer.
1114
# Install from a local tosa-tools checkout when available. If absent in this
1215
# checkout layout, clone the pinned upstream tag and install from there.

0 commit comments

Comments
 (0)