Skip to content

Commit 80826eb

Browse files
aliafzalaliafzal
committed
Add OSS CI to cross-compile and run the Cadence Xtensa backend
The upstream executor_runner cannot cross-compile to Xtensa because gflags pulls in mkdir(2), absent from Xtensa newlib. Add cadence_executor_runner, a gflags-free ExecuTorch runner for the Cadence Xtensa cores targeting the Instruction Set Simulator (xt-run): it uses plain argv parsing like the Arm and NXP backends, loads a .pte via xt-run semi-hosting, runs the first method with all-ones inputs, and prints outputs. EXECUTORCH_BUILD_CADENCE_RUNNER builds it, linking cadence_ops_lib transitively (no --whole-archive, which would double-run static kernel registration); -lidma is linked only for Vision/Fusion-G3 cores, whose ops reference iDMA and whose LSPs ship libidma, while HiFi4 does not. Also register op_quantized_depthwise_conv1d_{ncl,nlc}.cpp in the HiFi4 operators CMakeLists, which codegen references (omitting the sources broke the cross-compile link). Add an xtensa-build job to the Cadence Build & Test workflow (build-cadence-runner.yml), alongside the existing host cpu-build/cpu-test, to cross-compile the backend for the Xtensa cores. It is a build stage producing a runner artifact; the ISS test stage follows separately (cf. cpu-build -> cpu-test). The Xtensa toolchain and core configs are licensed and fetched at runtime from an auth-gated object store via a short-lived OIDC credential; the role, region, and store are supplied through CI variables and are not committed. setup-xtensa-tools.sh downloads and installs the toolchain/core for a backend, rewrites the vendor params to local paths, and exports the Xtensa env; build-cadence-xtensa.sh cross-compiles cadence_executor_runner. The job builds a [hifi4, vision] matrix and uploads the runner. fusion_g3 is omitted from the matrix until the upstream fusion_g3 <-> nnlib API skew is fixed (its runner does not link).
1 parent 5526971 commit 80826eb

6 files changed

Lines changed: 545 additions & 0 deletions

File tree

‎.ci/scripts/build-cadence-xtensa.sh‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
#
8+
# Cross-compile cadence_executor_runner for a Cadence Xtensa core and (by
9+
# default) smoke-test it on the Instruction Set Simulator with a trivial model.
10+
#
11+
# Requires the Xtensa toolchain env to already be set (run
12+
# .ci/scripts/setup-xtensa-tools.sh <backend> first): XTENSA_TOOLCHAIN,
13+
# TOOLCHAIN_VER, XTENSA_SYSTEM, XTENSA_CORE, XTENSAD_LICENSE_FILE,
14+
# CADENCE_OPT_FLAG, and xt-clang on PATH.
15+
#
16+
# Usage:
17+
# .ci/scripts/build-cadence-xtensa.sh [--no-run]
18+
# --no-run : compile only, skip the ISS smoke test
19+
20+
set -euo pipefail
21+
22+
RUN_SMOKE=1
23+
[[ "${1:-}" == "--no-run" ]] && RUN_SMOKE=0
24+
25+
: "${XTENSA_TOOLCHAIN:?run setup-xtensa-tools.sh first}"
26+
: "${TOOLCHAIN_VER:?run setup-xtensa-tools.sh first}"
27+
: "${XTENSA_CORE:?run setup-xtensa-tools.sh first}"
28+
: "${CADENCE_OPT_FLAG:?run setup-xtensa-tools.sh first}"
29+
30+
NPROC=$(nproc)
31+
echo "=== building cadence_executor_runner for ${XTENSA_CORE} (${CADENCE_OPT_FLAG}) ==="
32+
xt-clang --version | head -1
33+
34+
rm -rf cmake-out
35+
CXXFLAGS="-fno-exceptions -fno-rtti" cmake \
36+
-DCMAKE_TOOLCHAIN_FILE=./backends/cadence/cadence.cmake \
37+
-DCMAKE_INSTALL_PREFIX=cmake-out \
38+
-DCMAKE_BUILD_TYPE=Release \
39+
-DEXECUTORCH_BUILD_CADENCE=ON \
40+
"-D${CADENCE_OPT_FLAG}=ON" \
41+
-DEXECUTORCH_BUILD_PORTABLE_OPS=ON \
42+
-DEXECUTORCH_BUILD_CADENCE_RUNNER=ON \
43+
-DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \
44+
-DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
45+
-DEXECUTORCH_ENABLE_LOGGING=ON \
46+
-DEXECUTORCH_BUILD_PTHREADPOOL=OFF \
47+
-DEXECUTORCH_BUILD_CPUINFO=OFF \
48+
-DEXECUTORCH_USE_DL=OFF \
49+
-DEXECUTORCH_BUILD_KERNELS_LLM=OFF \
50+
-DEXECUTORCH_BUILD_DEVTOOLS=OFF \
51+
-DHAVE_FNMATCH_H=OFF \
52+
-DFLATCC_ALLOW_WERROR=OFF \
53+
-DPYTHON_EXECUTABLE="$(which python3)" \
54+
-Bcmake-out .
55+
56+
cmake --build cmake-out --target cadence_executor_runner -j"${NPROC}"
57+
58+
RUNNER="cmake-out/backends/cadence/cadence_executor_runner"
59+
if [[ ! -f "${RUNNER}" ]]; then
60+
echo "ERROR: ${RUNNER} was not produced" >&2
61+
exit 1
62+
fi
63+
file "${RUNNER}"
64+
echo "Build OK: ${RUNNER}"
65+
66+
if [[ "${RUN_SMOKE}" == "0" ]]; then
67+
echo "Skipping ISS smoke test (--no-run)."
68+
exit 0
69+
fi
70+
71+
echo "=== ISS smoke test: export add.pte and run on xt-run --turbo ==="
72+
python3 -m examples.portable.scripts.export --model_name=add >/dev/null
73+
LOG=$(mktemp)
74+
xt-run --turbo "${RUNNER}" --model_path=add.pte 2>&1 | tee "${LOG}"
75+
if ! grep -q "Model executed successfully" "${LOG}"; then
76+
echo "ERROR: ISS smoke test did not report success for ${XTENSA_CORE}" >&2
77+
exit 1
78+
fi
79+
echo "ISS smoke test passed for ${XTENSA_CORE}."

‎.ci/scripts/setup-xtensa-tools.sh‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
#
8+
# Download and install the licensed Cadence Xtensa toolchain + core config for
9+
# a given backend, then export the environment that
10+
# backends/cadence/cadence.cmake and xt-run need.
11+
#
12+
# The artifacts (host tools, the core tarball, and the bundled license) cannot
13+
# be hosted publicly, so they are fetched at runtime from an auth-gated object
14+
# store. The store location is provided by the caller via XTENSA_S3_BUCKET (set
15+
# from a CI variable); credentials are obtained out of band before this runs.
16+
#
17+
# Usage:
18+
# XTENSA_S3_BUCKET=<bucket> .ci/scripts/setup-xtensa-tools.sh <backend>
19+
# backend = hifi4 | vision | fusion_g3
20+
#
21+
# In GitHub Actions this appends the toolchain env to $GITHUB_ENV so later
22+
# steps inherit it. Run locally to populate a workspace for manual builds.
23+
#
24+
# Modeled on .ci/scripts/setup-arm-baremetal-tools.sh.
25+
26+
set -euo pipefail
27+
28+
BACKEND="${1:-}"
29+
if [[ -z "${BACKEND}" ]]; then
30+
echo "ERROR: usage: XTENSA_S3_BUCKET=<bucket> $0 <hifi4|vision|fusion_g3>" >&2
31+
exit 1
32+
fi
33+
34+
S3_BUCKET="${XTENSA_S3_BUCKET:-}"
35+
if [[ -z "${S3_BUCKET}" ]]; then
36+
echo "ERROR: XTENSA_S3_BUCKET is not set (provide it from a CI variable)." >&2
37+
exit 1
38+
fi
39+
S3_TOOLCHAIN_PREFIX="${XTENSA_S3_TOOLCHAIN_PREFIX:-toolchains}"
40+
S3_CORE_PREFIX="${XTENSA_S3_CORE_PREFIX:-cores}"
41+
42+
# Per-backend mapping: core tarball, toolchain tarball, core name, OPT flag.
43+
# The toolchain's clang major must match the core's codegen plugin:
44+
# hifi4 / fusion_g3 cores (RI-2022.10, clang 10) -> RI-2022.9 host tools
45+
# vision core (RJ-2025.5, clang 15) -> RJ-2025.5 host tools
46+
case "${BACKEND}" in
47+
hifi4)
48+
CORE_NAME="hifi4_ss_spfpu_7_et_ci2"
49+
CORE_TARBALL="hifi4_ss_spfpu_7_et_ci2_linux.tgz"
50+
TOOLCHAIN_TARBALL="XtensaTools_RI_2022_9_linux.tgz"
51+
TOOLCHAIN_VER="RI-2022.9-linux"
52+
OPT_FLAG="EXECUTORCH_NNLIB_OPT"
53+
;;
54+
fusion_g3)
55+
CORE_NAME="XRC_FuG3_TYP_SPVFPU_et_c2"
56+
CORE_TARBALL="XRC_FuG3_TYP_SPVFPU_et_c2_linux.tgz"
57+
TOOLCHAIN_TARBALL="XtensaTools_RI_2022_9_linux.tgz"
58+
TOOLCHAIN_VER="RI-2022.9-linux"
59+
OPT_FLAG="EXECUTORCH_FUSION_G3_OPT"
60+
;;
61+
vision)
62+
CORE_NAME="XRC_Vision_110_AO_et_ci2"
63+
CORE_TARBALL="XRC_Vision_110_AO_et_ci2_linux.tgz"
64+
TOOLCHAIN_TARBALL="XtensaTools_RJ_2025_5_linux.tgz"
65+
TOOLCHAIN_VER="RJ-2025.5-linux"
66+
OPT_FLAG="EXECUTORCH_VISION_OPT"
67+
;;
68+
*)
69+
echo "ERROR: unknown backend '${BACKEND}' (expected hifi4|vision|fusion_g3)" >&2
70+
exit 1
71+
;;
72+
esac
73+
74+
XTENSA_ROOT="${XTENSA_ROOT:-/tmp/xtensa}"
75+
TOOLS_ROOT="${XTENSA_ROOT}/tools" # contains <ver>-linux/XtensaTools
76+
CORES_ROOT="${XTENSA_ROOT}/cores" # contains <corever>-linux/<core>
77+
REGISTRY_ROOT="${XTENSA_ROOT}/registry/${CORE_NAME}"
78+
DL_DIR="${XTENSA_ROOT}/download"
79+
mkdir -p "${TOOLS_ROOT}" "${CORES_ROOT}" "${REGISTRY_ROOT}" "${DL_DIR}"
80+
81+
s3_get() {
82+
# $1 = s3 key, $2 = local dest
83+
local key="$1" dest="$2"
84+
echo "Downloading s3://${S3_BUCKET}/${key} ..."
85+
aws s3 cp "s3://${S3_BUCKET}/${key}" "${dest}" --only-show-errors
86+
}
87+
88+
# 1. Toolchain (host xt-clang/xt-run). Skip re-extract if already present.
89+
if [[ ! -d "${TOOLS_ROOT}/${TOOLCHAIN_VER}/XtensaTools" ]]; then
90+
s3_get "${S3_TOOLCHAIN_PREFIX}/${TOOLCHAIN_TARBALL}" "${DL_DIR}/${TOOLCHAIN_TARBALL}"
91+
tar xzf "${DL_DIR}/${TOOLCHAIN_TARBALL}" -C "${TOOLS_ROOT}"
92+
fi
93+
TOOLCHAIN_HOME="${TOOLS_ROOT}/${TOOLCHAIN_VER}/XtensaTools"
94+
if [[ ! -x "${TOOLCHAIN_HOME}/bin/xt-clang" ]]; then
95+
echo "ERROR: xt-clang not found at ${TOOLCHAIN_HOME}/bin after extract" >&2
96+
exit 1
97+
fi
98+
99+
# 2. Core config (ISA libs, params, examples, bundled magic-key license).
100+
s3_get "${S3_CORE_PREFIX}/${CORE_TARBALL}" "${DL_DIR}/${CORE_TARBALL}"
101+
tar xzf "${DL_DIR}/${CORE_TARBALL}" -C "${CORES_ROOT}"
102+
CORE_DIR=$(echo "${CORES_ROOT}"/*/"${CORE_NAME}")
103+
if [[ ! -d "${CORE_DIR}" ]]; then
104+
echo "ERROR: core dir for ${CORE_NAME} not found under ${CORES_ROOT}" >&2
105+
exit 1
106+
fi
107+
108+
# 3. Build a local Xtensa core registry with the XPG-internal build paths in
109+
# the params file rewritten to our extracted toolchain + core locations.
110+
# The vendor ships params referencing /././home/xpgcust/... build paths.
111+
PARAMS_SRC="${CORE_DIR}/config/${CORE_NAME}-params"
112+
TOOLS_PFX=$(sed -n 's/^install-prefix = //p' "${PARAMS_SRC}" | head -1)
113+
TOOLSUB_PFX=$(sed -n 's/^xtensa-tools = //p' "${PARAMS_SRC}" | head -1)
114+
CFG_PFX=$(sed -n 's/^config-prefix = //p' "${PARAMS_SRC}" | head -1)
115+
sed \
116+
-e "s|${TOOLS_PFX}|${TOOLCHAIN_HOME}|g" \
117+
-e "s|${TOOLSUB_PFX}|${TOOLCHAIN_HOME}/Tools|g" \
118+
-e "s|${CFG_PFX}|${CORE_DIR}|g" \
119+
"${PARAMS_SRC}" > "${REGISTRY_ROOT}/${CORE_NAME}-params"
120+
ln -sf "${CORE_NAME}-params" "${REGISTRY_ROOT}/default-params"
121+
122+
LICENSE_FILE="${CORE_DIR}/misc/license.dat"
123+
124+
# 4. Export environment. cadence.cmake reads XTENSA_TOOLCHAIN/TOOLCHAIN_VER;
125+
# xt-clang/xt-run read XTENSA_SYSTEM/XTENSA_CORE; xtensad reads
126+
# XTENSAD_LICENSE_FILE (the bundled uncounted magic key, no server needed).
127+
emit() {
128+
# Export into the current shell (so callers that `source` this script get the
129+
# vars) and append to $GITHUB_ENV (so later workflow steps inherit them too).
130+
echo "$1"
131+
export "${1?}"
132+
if [[ -n "${GITHUB_ENV:-}" ]]; then echo "$1" >> "${GITHUB_ENV}"; fi
133+
}
134+
echo "=== Xtensa env for backend '${BACKEND}' (core ${CORE_NAME}) ==="
135+
emit "XTENSA_TOOLCHAIN=${TOOLS_ROOT}"
136+
emit "TOOLCHAIN_VER=${TOOLCHAIN_VER}"
137+
emit "XTENSA_SYSTEM=${REGISTRY_ROOT}"
138+
emit "XTENSA_CORE=${CORE_NAME}"
139+
emit "XTENSAD_LICENSE_FILE=${LICENSE_FILE}"
140+
emit "CADENCE_OPT_FLAG=${OPT_FLAG}"
141+
if [[ -n "${GITHUB_PATH:-}" ]]; then
142+
echo "${TOOLCHAIN_HOME}/bin" >> "${GITHUB_PATH}"
143+
fi
144+
export PATH="${TOOLCHAIN_HOME}/bin:${PATH}"
145+
146+
echo "=== sanity ==="
147+
xt-clang --version 2>&1 | head -1
148+
xt-run --show-config=cores 2>&1 | sed -n '/available/,/registry/p' | head -6
149+
echo "Xtensa toolchain ready for ${BACKEND}."

‎.github/workflows/build-cadence-runner.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,85 @@ jobs:
5050
uses: ./.github/workflows/_test_cadence.yml
5151
with:
5252
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
53+
54+
# Cross-compile cadence_executor_runner for the Cadence Xtensa cores (build
55+
# stage; an ISS test stage follows later, cf. cpu-build -> cpu-test). The
56+
# licensed toolchain + core configs are fetched at runtime from an auth-gated
57+
# store. Role and store are environment secrets (masked in logs); region is a
58+
# variable. None are committed.
59+
#
60+
# This is a native job, not a linux_job_v2 call, because the GitHub OIDC token
61+
# must be minted on the runner host: the ACTIONS_ID_TOKEN_REQUEST_* vars do not
62+
# cross into linux_job_v2's docker exec. So the role is assumed on the host,
63+
# then the build runs inside the CI image via docker run with the creds passed
64+
# in. Binding the environment also gives the OIDC token the environment claim.
65+
xtensa-build:
66+
name: xtensa-build-${{ matrix.backend }}
67+
runs-on: linux.2xlarge
68+
environment: cadence
69+
permissions:
70+
id-token: write
71+
contents: read
72+
strategy:
73+
fail-fast: false
74+
matrix:
75+
# fusion_g3 is omitted until the upstream fusion_g3 operator <->
76+
# nnlib-FusionG3 API skew is resolved (its op wrappers call kernels
77+
# absent from the vendored nnlib, so the runner does not link).
78+
backend: [hifi4, vision]
79+
steps:
80+
- name: Checkout executorch
81+
uses: actions/checkout@v4
82+
with:
83+
submodules: recursive
84+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
85+
86+
- name: Calculate docker image
87+
id: calculate-docker-image
88+
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
89+
with:
90+
docker-image-name: ci-image:executorch-ubuntu-22.04-clang12
91+
92+
- name: Pull docker image
93+
run: docker pull "${{ steps.calculate-docker-image.outputs.docker-image }}"
94+
95+
- name: Assume Cadence artifacts role (host OIDC)
96+
uses: aws-actions/configure-aws-credentials@v4
97+
with:
98+
role-to-assume: ${{ secrets.CADENCE_CI_AWS_ROLE }}
99+
aws-region: ${{ vars.CADENCE_CI_AWS_REGION }}
100+
101+
- name: Cross-compile cadence_executor_runner
102+
env:
103+
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
104+
BACKEND: ${{ matrix.backend }}
105+
XTENSA_S3_BUCKET: ${{ secrets.CADENCE_CI_S3_BUCKET }}
106+
shell: bash
107+
run: |
108+
set -eux
109+
# OIDC/role assumption already happened on the host above; pass the
110+
# resulting AWS creds and the store/backend into the CI image, where
111+
# the toolchain download + cross-compile run.
112+
docker run --rm \
113+
-e BACKEND -e XTENSA_S3_BUCKET \
114+
-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN \
115+
-e AWS_DEFAULT_REGION -e AWS_REGION \
116+
-v "${GITHUB_WORKSPACE}:/work" -w /work \
117+
"${DOCKER_IMAGE}" \
118+
bash -c '
119+
set -exo pipefail
120+
eval "$(/opt/conda/bin/conda shell.bash hook)"
121+
conda activate "$(conda env list --json | jq -r ".envs | .[-1]")"
122+
./install_requirements.sh > /dev/null
123+
pip install --quiet awscli
124+
source .ci/scripts/setup-xtensa-tools.sh "${BACKEND}"
125+
.ci/scripts/build-cadence-xtensa.sh --no-run
126+
chmod -R a+rX cmake-out
127+
'
128+
129+
- name: Upload runner
130+
uses: actions/upload-artifact@v4
131+
with:
132+
name: cadence-xtensa-build-${{ matrix.backend }}
133+
path: cmake-out/backends/cadence/cadence_executor_runner
134+
if-no-files-found: error

‎backends/cadence/CMakeLists.txt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,3 +97,38 @@ else()
9797
endif()
9898

9999
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${TARGET_DIR}/operators)
100+
101+
# Cadence executor_runner: cross-compiled ExecuTorch runner for the Xtensa ISS
102+
# (xt-run / xt-run --turbo). Self-contained, gflags-free argv parser, reads .pte
103+
# via xt-run semi-hosting.
104+
#
105+
# Usage: cmake ... -DEXECUTORCH_BUILD_CADENCE_RUNNER=ON xt-run --turbo
106+
# cmake-out/backends/cadence/cadence_executor_runner \ --model_path=add.pte
107+
if(EXECUTORCH_BUILD_CADENCE_RUNNER)
108+
add_executable(cadence_executor_runner cadence_executor_runner.cpp)
109+
target_compile_definitions(
110+
cadence_executor_runner PRIVATE ET_ENABLE_ENUM_STRINGS=0
111+
)
112+
target_include_directories(
113+
cadence_executor_runner
114+
PRIVATE ${_common_include_directories} ${CMAKE_BINARY_DIR}
115+
${CMAKE_BINARY_DIR}/include
116+
)
117+
# Mirror the upstream executor_runner cadence link list (top-level
118+
# CMakeLists.txt: list(APPEND _executor_runner_libs cadence_ops_lib)). Do NOT
119+
# add --whole-archive: cadence_ops_lib is also pulled transitively, and
120+
# forcing a second copy double-runs its static kernel-registration
121+
# initializers and asserts at runtime.
122+
target_link_libraries(
123+
cadence_executor_runner PRIVATE executorch extension_evalue_util
124+
extension_runner_util cadence_ops_lib
125+
)
126+
# Vision and Fusion-G3 ops (e.g. op_softmax) reference iDMA scheduling symbols
127+
# and those cores ship libidma in their LSP. HiFi4 and generic cores do not
128+
# use iDMA and their LSPs may not provide libidma, so only link it for the
129+
# cores that need it.
130+
if(EXECUTORCH_VISION_OPT OR EXECUTORCH_FUSION_G3_OPT)
131+
target_link_options(cadence_executor_runner PRIVATE -lidma)
132+
endif()
133+
target_link_options(cadence_executor_runner PRIVATE -static -lm)
134+
endif()

0 commit comments

Comments
 (0)