Skip to content

Commit c7109be

Browse files
ahmadelyousseffacebook-github-bot
ahmadelyoussef
authored and
facebook-github-bot
committed
Implmenting CI workflow for tao_bench and Spark installation (#120)
Summary: Pull Request resolved: #120 Reviewed By: excelle08 Differential Revision: D74892657 fbshipit-source-id: 5b008013ce7565c38dabcce488a8affa91c266e1
1 parent 059a26f commit c7109be

8 files changed

Lines changed: 276 additions & 15 deletions

File tree

‎.github/scripts/dcperf_install_spark.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install Spark
15+
################################################################################
16+
17+
install_spark () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install Spark"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[INSTALL] Installing Spark ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py install spark_standalone_remote) || return 1
39+
}

‎.github/scripts/dcperf_install_tao_bench.bash‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# tao_bench Install Functions
15+
################################################################################
16+
17+
install_tao_bench () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install Tao_bench"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
34+
# shellcheck disable=SC2155
35+
local env_prefix=$(env_name_or_prefix "${env_name}")
36+
37+
echo "[INSTALL] Installing Tao_bench ..."
38+
(print_exec conda run --no-capture-output ${env_prefix} \
39+
python ./benchpress_cli.py install tao_bench_autoscale) || return 1
40+
}

‎.github/scripts/setup_env.bash‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,7 @@
1717
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_django.bash"
1818
# shellcheck disable=SC1091,SC2128
1919
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_feedsim.bash"
20+
# shellcheck disable=SC1091,SC2128
21+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_tao_bench.bash"
22+
# shellcheck disable=SC1091,SC2128
23+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_spark.bash"

‎.github/scripts/utils_build.bash‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ install_python_tools () {
9090
# shellcheck disable=SC2086
9191
(exec_with_retries 3 conda install ${env_prefix} -c conda-forge --override-channels -y \
9292
click \
93-
numpy \
93+
numpy=1.* \
9494
pandas \
9595
pyyaml \
9696
tabulate) || return 1

‎.github/workflows/dcperf_ci_spark.yml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
# This workflow is used for DCPerf CI.
7+
name: DCPerf Spark CI
8+
9+
on:
10+
# PR Trigger
11+
#
12+
pull_request:
13+
14+
# Push Trigger (enable to catch errors coming out of multiple merges)
15+
#
16+
push:
17+
branches:
18+
- main
19+
20+
# Manual Trigger
21+
#
22+
workflow_dispatch:
23+
24+
concurrency:
25+
# Cancel previous runs in the PR if a new commit is pushed
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
# Build on CPU hosts (generic GitHub runners)
31+
build_artifact:
32+
runs-on: ${{ matrix.host-machine.instance }}
33+
container:
34+
image: ${{ matrix.host-machine.container }}
35+
options: --user root --privileged --pid=host
36+
volumes:
37+
- /var/run/docker.sock:/var/run/docker.sock
38+
defaults:
39+
run:
40+
shell: bash
41+
env:
42+
PRELUDE: .github/scripts/setup_env.bash
43+
BUILD_ENV: build_env
44+
continue-on-error: false
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
host-machine: [
49+
{ arch: x86, instance: "ubuntu-latest", container: "ubuntu:22.04", type: "ubuntu" },
50+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "ubuntu:22.04", type: "ubuntu" },
51+
{ arch: x86, instance: "ubuntu-latest", container: "quay.io/centos/centos:stream9", type: "centos" },
52+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "quay.io/centos/centos:stream9", type: "centos" },
53+
]
54+
python-version: [ "3.12" ]
55+
steps:
56+
- name: Setup Build Container (ubuntu-based)
57+
if: ${{ matrix.host-machine.type == 'ubuntu' }}
58+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode
59+
60+
- name: Setup Build Container (centos-based)
61+
if: ${{ matrix.host-machine.type == 'centos' }}
62+
run: dnf update -y; dnf install -y git pciutils which dmidecode
63+
64+
- name: Checkout the Repository
65+
uses: actions/checkout@v4
66+
with:
67+
submodules: true
68+
69+
- name: Free Disk Space on Host
70+
run: . $PRELUDE; free_disk_space_on_host
71+
72+
- name: Display System Info
73+
run: . $PRELUDE; print_system_info
74+
75+
- name: Install Build Tools
76+
run: . $PRELUDE; install_build_tools
77+
78+
- name: Setup Miniconda
79+
run: . $PRELUDE; setup_miniconda $HOME/miniconda
80+
81+
- name: Create Conda Environment
82+
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
83+
84+
- name: Install Python Tools
85+
run: . $PRELUDE; install_python_tools $BUILD_ENV
86+
87+
- name: Install Spark
88+
run: . $PRELUDE; install_spark $BUILD_ENV
89+
continue-on-error: false

‎.github/workflows/dcperf_ci_tao_bench.yml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
# This workflow is used for DCPerf CI.
7+
name: DCPerf Tao_bench CI
8+
9+
on:
10+
# PR Trigger
11+
#
12+
pull_request:
13+
14+
# Push Trigger (enable to catch errors coming out of multiple merges)
15+
#
16+
push:
17+
branches:
18+
- main
19+
20+
# Manual Trigger
21+
#
22+
workflow_dispatch:
23+
24+
concurrency:
25+
# Cancel previous runs in the PR if a new commit is pushed
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
# Build on CPU hosts (generic GitHub runners)
31+
build_artifact:
32+
runs-on: ${{ matrix.host-machine.instance }}
33+
container:
34+
image: ${{ matrix.host-machine.container }}
35+
options: --user root --privileged --pid=host
36+
volumes:
37+
- /var/run/docker.sock:/var/run/docker.sock
38+
defaults:
39+
run:
40+
shell: bash
41+
env:
42+
PRELUDE: .github/scripts/setup_env.bash
43+
BUILD_ENV: build_env
44+
continue-on-error: false
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
host-machine: [
49+
{ arch: x86, instance: "ubuntu-latest", container: "ubuntu:22.04", type: "ubuntu" },
50+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "ubuntu:22.04", type: "ubuntu" },
51+
{ arch: x86, instance: "ubuntu-latest", container: "quay.io/centos/centos:stream9", type: "centos" },
52+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "quay.io/centos/centos:stream9", type: "centos" },
53+
]
54+
python-version: [ "3.12" ]
55+
steps:
56+
- name: Setup Build Container (ubuntu-based)
57+
if: ${{ matrix.host-machine.type == 'ubuntu' }}
58+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode curl
59+
60+
- name: Setup Build Container (centos-based)
61+
if: ${{ matrix.host-machine.type == 'centos' }}
62+
run: dnf update -y; dnf install -y git pciutils which dmidecode
63+
64+
- name: Checkout the Repository
65+
uses: actions/checkout@v4
66+
with:
67+
submodules: true
68+
69+
- name: Free Disk Space on Host
70+
run: . $PRELUDE; free_disk_space_on_host
71+
72+
- name: Display System Info
73+
run: . $PRELUDE; print_system_info
74+
75+
- name: Install Build Tools
76+
run: . $PRELUDE; install_build_tools
77+
78+
- name: Setup Miniconda
79+
run: . $PRELUDE; setup_miniconda $HOME/miniconda
80+
81+
- name: Create Conda Environment
82+
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
83+
84+
- name: Install Python Tools
85+
run: . $PRELUDE; install_python_tools $BUILD_ENV
86+
87+
- name: Install Tao_bench
88+
run: . $PRELUDE; install_tao_bench $BUILD_ENV
89+
continue-on-error: false

‎packages/tao_bench/install_tao_bench_aarch64.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,15 +32,15 @@ else
3232
fi
3333

3434
if distro_is_like centos; then
35-
sudo dnf install -y cmake iperf3 autoconf automake \
35+
dnf install -y cmake iperf3 autoconf automake \
3636
libevent-devel openssl openssl-devel \
3737
zlib-devel bzip2-devel xz-devel lz4-devel libzstd-devel \
3838
snappy-devel libaio-devel libunwind-devel patch \
3939
double-conversion-devel libsodium-devel \
4040
gflags-devel-2.2.2 fmt-devel perl libtool pcre-devel \
4141
git python3-devel ${GLOG_NAME}
4242
elif distro_is_like ubuntu; then
43-
sudo apt install -y cmake iperf3 autoconf automake flex bison \
43+
apt install -y cmake iperf3 autoconf automake flex bison \
4444
libevent-dev openssl libssl-dev \
4545
libzstd-dev lz4 liblz4-dev xzip libsnappy-dev zlib1g-dev bzip2 \
4646
libaio-dev libunwind-dev patch libghc-double-conversion-dev \
@@ -56,7 +56,7 @@ mkdir -p "${TAO_BENCH_DEPS}"
5656
pushd "${TAO_BENCH_ROOT}"
5757

5858
if ! [ -f "/usr/local/bin/cmake" ]; then
59-
sudo ln -s /usr/bin/cmake /usr/local/bin/cmake
59+
ln -s /usr/bin/cmake /usr/local/bin/cmake
6060
fi
6161

6262
# Install openssl
@@ -132,10 +132,10 @@ fi
132132

133133
# Build and install
134134
if ! [ -f "/usr/bin/aclocal-1.16" ]; then
135-
sudo ln -s /usr/bin/aclocal /usr/bin/aclocal-1.16
135+
ln -s /usr/bin/aclocal /usr/bin/aclocal-1.16
136136
fi
137137
if ! [ -f "/usr/bin/automake-1.16" ]; then
138-
sudo ln -s /usr/bin/automake /usr/bin/automake-1.16
138+
ln -s /usr/bin/automake /usr/bin/automake-1.16
139139
fi
140140

141141
./configure --with-folly="${FOLLY_INSTALLED_PATH}" --with-fmt="${FMT_INSTALLED_PATH}" \
@@ -144,10 +144,10 @@ fi
144144
make -j"$(nproc)"
145145

146146
if [ -L /usr/bin/aclocal-1.16 ]; then
147-
sudo rm -f /usr/bin/aclocal-1.16
147+
rm -f /usr/bin/aclocal-1.16
148148
fi
149149
if [ -L /usr/bin/automake-1.16 ]; then
150-
sudo rm -f /usr/bin/automake-1.16
150+
rm -f /usr/bin/automake-1.16
151151
fi
152152

153153
cp memcached "${TAO_BENCH_ROOT}/tao_bench_server"

‎packages/tao_bench/install_tao_bench_x86_64.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,15 @@ else
3030
fi
3131

3232
if [ "$LINUX_DIST_ID" = "ubuntu" ]; then
33-
sudo apt install -y cmake iperf3 autoconf automake flex bison \
33+
apt install -y cmake iperf3 autoconf automake flex bison \
3434
libevent-dev openssl libssl-dev \
3535
libzstd-dev lz4 liblz4-dev xzip libsnappy-dev zlib1g-dev bzip2 \
3636
libaio-dev libunwind-dev patch libghc-double-conversion-dev \
3737
libsodium-dev libfmt-dev libtool perl git \
3838
libgoogle-glog-dev python3-dev pkg-config \
3939
git libpcre3 libpcre3-dev libgflags2.2 libgflags-dev
4040
elif [ "$LINUX_DIST_ID" = "centos" ]; then
41-
sudo dnf install -y cmake iperf3 autoconf automake \
41+
dnf install -y cmake iperf3 autoconf automake \
4242
libevent-devel openssl openssl-devel \
4343
zlib-devel bzip2-devel xz-devel lz4-devel libzstd-devel \
4444
snappy-devel libaio-devel libunwind-devel patch \
@@ -52,7 +52,7 @@ mkdir -p "${TAO_BENCH_DEPS}"
5252
pushd "${TAO_BENCH_ROOT}"
5353

5454
if ! [ -f "/usr/local/bin/cmake" ]; then
55-
sudo ln -s /usr/bin/cmake /usr/local/bin/cmake
55+
ln -s /usr/bin/cmake /usr/local/bin/cmake
5656
fi
5757

5858
# Install openssl
@@ -127,10 +127,10 @@ fi
127127

128128
# Build and install
129129
if ! [ -f "/usr/bin/aclocal-1.16" ]; then
130-
sudo ln -s /usr/bin/aclocal /usr/bin/aclocal-1.16
130+
ln -s /usr/bin/aclocal /usr/bin/aclocal-1.16
131131
fi
132132
if ! [ -f "/usr/bin/automake-1.16" ]; then
133-
sudo ln -s /usr/bin/automake /usr/bin/automake-1.16
133+
ln -s /usr/bin/automake /usr/bin/automake-1.16
134134
fi
135135

136136
./configure --with-folly="${FOLLY_INSTALLED_PATH}" --with-fmt="${FMT_INSTALLED_PATH}" \
@@ -139,10 +139,10 @@ fi
139139
make -j"$(nproc)"
140140

141141
if [ -L /usr/bin/aclocal-1.16 ]; then
142-
sudo rm -f /usr/bin/aclocal-1.16
142+
rm -f /usr/bin/aclocal-1.16
143143
fi
144144
if [ -L /usr/bin/automake-1.16 ]; then
145-
sudo rm -f /usr/bin/automake-1.16
145+
rm -f /usr/bin/automake-1.16
146146
fi
147147

148148
cp memcached "${TAO_BENCH_ROOT}/tao_bench_server"

0 commit comments

Comments
 (0)