Skip to content

Commit aea401a

Browse files
ahmadelyousseffacebook-github-bot
ahmadelyoussef
authored and
facebook-github-bot
committed
Implmenting CI workflow for video_transcode_bench and mediawiki (#121)
Summary: Pull Request resolved: #121 Differential Revision: D75008473
1 parent 0e3f19a commit aea401a

10 files changed

Lines changed: 282 additions & 6 deletions

‎.github/scripts/dcperf_install_mediawiki.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install Mediawiki
15+
################################################################################
16+
17+
install_mediawiki () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install Mediawiki"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[INSTALL] Installing Mediawiki ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py install oss_performance_mediawiki_mlp) || return 1
39+
}

‎.github/scripts/dcperf_install_video_transcode_bench.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install FeedSim
15+
################################################################################
16+
17+
install_video_transcode_bench () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install VideoTranscodeBench"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[INSTALL] Installing VideoTranscodeBench ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py install video_transcode_bench_svt) || return 1
39+
}

‎.github/scripts/setup_env.bash‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,7 @@
2121
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_tao_bench.bash"
2222
# shellcheck disable=SC1091,SC2128
2323
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_spark.bash"
24+
# shellcheck disable=SC1091,SC2128
25+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_video_transcode_bench.bash"
26+
# shellcheck disable=SC1091,SC2128
27+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_mediawiki.bash"

‎.github/scripts/utils_build.bash‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,3 +103,21 @@ install_python_tools () {
103103

104104
echo "[INSTALL] Successfully installed all the build tools"
105105
}
106+
107+
################################################################################
108+
# HHVM Setup
109+
################################################################################
110+
111+
install_hhvm (){
112+
echo "[INSTALL] Installing hhvm ..."
113+
if which dnf; then
114+
wget https://github.com/facebookresearch/DCPerf/releases/download/hhvm/hhvm-3.30-multplatform-binary.tar.xz
115+
tar -Jxf hhvm-3.30-multplatform-binary.tar.xz
116+
else
117+
wget https://github.com/facebookresearch/DCPerf/releases/download/hhvm/hhvm-3.30-multplatform-binary-ubuntu.tar.xz
118+
tar -Jxf hhvm-3.30-multplatform-binary-ubuntu.tar.xz
119+
fi
120+
cd hhvm || return 1
121+
./pour-hhvm.sh
122+
cd .. || return 1
123+
}

‎.github/workflows/dcperf_ci_feedsim.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,3 @@ jobs:
8686

8787
- name: Install FeedSim
8888
run: . $PRELUDE; install_feedsim $BUILD_ENV
89-
continue-on-error: false

‎.github/workflows/dcperf_ci_mediawiki.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
# This workflow is used for DCPerf CI.
7+
name: DCPerf Mediawiki CI
8+
9+
on:
10+
# PR Trigger
11+
#
12+
pull_request:
13+
14+
# Push Trigger (enable to catch errors coming out of multiple merges)
15+
#
16+
push:
17+
branches:
18+
- main
19+
20+
# Manual Trigger
21+
#
22+
workflow_dispatch:
23+
24+
concurrency:
25+
# Cancel previous runs in the PR if a new commit is pushed
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
# Build on CPU hosts (generic GitHub runners)
31+
build_artifact:
32+
runs-on: ${{ matrix.host-machine.instance }}
33+
container:
34+
image: ${{ matrix.host-machine.container }}
35+
options: --user root --privileged --pid=host
36+
volumes:
37+
- /var/run/docker.sock:/var/run/docker.sock
38+
defaults:
39+
run:
40+
shell: bash
41+
env:
42+
PRELUDE: .github/scripts/setup_env.bash
43+
BUILD_ENV: build_env
44+
continue-on-error: false
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
host-machine: [
49+
{ arch: x86, instance: "ubuntu-latest", container: "ubuntu:22.04", type: "ubuntu" },
50+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "ubuntu:22.04", type: "ubuntu" },
51+
{ arch: x86, instance: "ubuntu-latest", container: "quay.io/centos/centos:stream9", type: "centos" },
52+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "quay.io/centos/centos:stream9", type: "centos" },
53+
]
54+
python-version: [ "3.12" ]
55+
steps:
56+
- name: Setup Build Container (ubuntu-based)
57+
if: ${{ matrix.host-machine.type == 'ubuntu' }}
58+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode
59+
60+
- name: Setup Build Container (centos-based)
61+
if: ${{ matrix.host-machine.type == 'centos' }}
62+
run: dnf update -y; dnf install -y git pciutils which dmidecode
63+
64+
- name: Checkout the Repository
65+
uses: actions/checkout@v4
66+
with:
67+
submodules: true
68+
69+
- name: Free Disk Space on Host
70+
run: . $PRELUDE; free_disk_space_on_host
71+
72+
- name: Display System Info
73+
run: . $PRELUDE; print_system_info
74+
75+
- name: Install Build Tools
76+
run: . $PRELUDE; install_build_tools
77+
78+
- name: Setup Miniconda
79+
run: . $PRELUDE; setup_miniconda $HOME/miniconda
80+
81+
- name: Create Conda Environment
82+
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
83+
84+
- name: Install Python Tools
85+
run: . $PRELUDE; install_python_tools $BUILD_ENV
86+
87+
- name: Install HHVM
88+
run: . $PRELUDE; install_hhvm $BUILD_ENV
89+
90+
- name: Install FeedSim
91+
run: . $PRELUDE; install_mediawiki $BUILD_ENV

‎.github/workflows/dcperf_ci_spark.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,3 @@ jobs:
8686

8787
- name: Install Spark
8888
run: . $PRELUDE; install_spark $BUILD_ENV
89-
continue-on-error: false

‎.github/workflows/dcperf_ci_tao_bench.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,3 @@ jobs:
8686

8787
- name: Install Tao_bench
8888
run: . $PRELUDE; install_tao_bench $BUILD_ENV
89-
continue-on-error: false

‎.github/workflows/dcperf_ci_video_transcode_bench.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
# This workflow is used for DCPerf CI.
7+
name: DCPerf VideoTranscodeBench CI
8+
9+
on:
10+
# PR Trigger
11+
#
12+
pull_request:
13+
14+
# Push Trigger (enable to catch errors coming out of multiple merges)
15+
#
16+
push:
17+
branches:
18+
- main
19+
20+
# Manual Trigger
21+
#
22+
workflow_dispatch:
23+
24+
concurrency:
25+
# Cancel previous runs in the PR if a new commit is pushed
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
# Build on CPU hosts (generic GitHub runners)
31+
build_artifact:
32+
runs-on: ${{ matrix.host-machine.instance }}
33+
container:
34+
image: ${{ matrix.host-machine.container }}
35+
options: --user root --privileged --pid=host
36+
volumes:
37+
- /var/run/docker.sock:/var/run/docker.sock
38+
defaults:
39+
run:
40+
shell: bash
41+
env:
42+
PRELUDE: .github/scripts/setup_env.bash
43+
BUILD_ENV: build_env
44+
continue-on-error: false
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
host-machine: [
49+
{ arch: x86, instance: "ubuntu-latest", container: "ubuntu:22.04", type: "ubuntu" },
50+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "ubuntu:22.04", type: "ubuntu" },
51+
{ arch: x86, instance: "ubuntu-latest", container: "quay.io/centos/centos:stream9", type: "centos" },
52+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "quay.io/centos/centos:stream9", type: "centos" },
53+
]
54+
python-version: [ "3.12" ]
55+
steps:
56+
- name: Setup Build Container (ubuntu-based)
57+
if: ${{ matrix.host-machine.type == 'ubuntu' }}
58+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode gcc clang
59+
60+
- name: Setup Build Container (centos-based)
61+
if: ${{ matrix.host-machine.type == 'centos' }}
62+
run: dnf update -y; dnf install -y git pciutils which dmidecode
63+
64+
- name: Checkout the Repository
65+
uses: actions/checkout@v4
66+
with:
67+
submodules: true
68+
69+
- name: Free Disk Space on Host
70+
run: . $PRELUDE; free_disk_space_on_host
71+
72+
- name: Display System Info
73+
run: . $PRELUDE; print_system_info
74+
75+
- name: Install Build Tools
76+
run: . $PRELUDE; install_build_tools
77+
78+
- name: Setup Miniconda
79+
run: . $PRELUDE; setup_miniconda $HOME/miniconda
80+
81+
- name: Create Conda Environment
82+
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
83+
84+
- name: Install Python Tools
85+
run: . $PRELUDE; install_python_tools $BUILD_ENV
86+
87+
- name: Install VideoTranscodeBench
88+
run: . $PRELUDE; install_video_transcode_bench $BUILD_ENV

‎packages/video_transcode_bench/install_video_transcode_bench.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,11 @@ FFMPEG_DATASETS="${FFMPEG_ROOT}/datasets/cuts"
4040
LINUX_DIST_ID="$(awk -F "=" '/^ID=/ {print $2}' /etc/os-release | tr -d '"')"
4141

4242
if [ "$LINUX_DIST_ID" = "ubuntu" ]; then
43-
sudo apt install -y cmake autoconf automake flex bison \
43+
apt install -y cmake autoconf automake flex bison \
4444
meson nasm clang patch git \
4545
python3-dev pkg-config time parallel p7zip
4646
elif [ "$LINUX_DIST_ID" = "centos" ]; then
47-
sudo dnf install -y cmake autoconf automake flex bison \
47+
dnf install -y cmake autoconf automake flex bison \
4848
meson nasm clang patch \
4949
git python3-devel time parallel p7zip
5050
fi
@@ -59,7 +59,7 @@ mkdir -p "${FFMPEG_BUILD}"
5959
mkdir -p "${FFMPEG_DATASETS}"
6060

6161
if ! [ -f "/usr/local/bin/cmake" ]; then
62-
sudo ln -s /usr/bin/cmake /usr/local/bin/cmake
62+
ln -s /usr/bin/cmake /usr/local/bin/cmake
6363
fi
6464

6565
##################### BUILD AND INSTALL FUNCTIONS #########################

0 commit comments

Comments
 (0)