Skip to content

Commit ad24d09

Browse files
ahmadelyousseffacebook-github-bot
ahmadelyoussef
authored and
facebook-github-bot
committed
Implmenting CI workflow for video_transcode_bench and mediawiki (#121)
Summary: Pull Request resolved: #121 Differential Revision: D75008473
1 parent 0f8e205 commit ad24d09

13 files changed

Lines changed: 294 additions & 20 deletions

‎.github/scripts/dcperf_install_mediawiki.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash -x
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install Mediawiki
15+
################################################################################
16+
17+
install_mediawiki () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install Mediawiki"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
export DEBIAN_FRONTEND=noninteractive
36+
echo "[INSTALL] Installing Mediawiki ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py install oss_performance_mediawiki_mlp) || return 1
39+
}

‎.github/scripts/dcperf_install_video_transcode_bench.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash -x
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install FeedSim
15+
################################################################################
16+
17+
install_video_transcode_bench () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install VideoTranscodeBench"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
clang --version
36+
echo "[INSTALL] Installing VideoTranscodeBench ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py install video_transcode_bench_svt) || return 1
39+
}

‎.github/scripts/setup_env.bash‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,7 @@
2121
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_tao_bench.bash"
2222
# shellcheck disable=SC1091,SC2128
2323
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_spark.bash"
24+
# shellcheck disable=SC1091,SC2128
25+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_video_transcode_bench.bash"
26+
# shellcheck disable=SC1091,SC2128
27+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_mediawiki.bash"

‎.github/scripts/utils_build.bash‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/bin/bash -x
22
# Copyright (c) Meta Platforms, Inc. and affiliates.
33
# All rights reserved.
44
#
@@ -103,3 +103,21 @@ install_python_tools () {
103103

104104
echo "[INSTALL] Successfully installed all the build tools"
105105
}
106+
107+
################################################################################
108+
# HHVM Setup
109+
################################################################################
110+
111+
install_hhvm (){
112+
echo "[INSTALL] Installing hhvm ..."
113+
if which dnf; then
114+
wget https://github.com/facebookresearch/DCPerf/releases/download/hhvm/hhvm-3.30-multplatform-binary.tar.xz
115+
tar -Jxf hhvm-3.30-multplatform-binary.tar.xz
116+
else
117+
wget https://github.com/facebookresearch/DCPerf/releases/download/hhvm/hhvm-3.30-multplatform-binary-ubuntu.tar.xz
118+
tar -Jxf hhvm-3.30-multplatform-binary-ubuntu.tar.xz
119+
fi
120+
cd hhvm || return 1
121+
bash -x ./pour-hhvm.sh
122+
cd .. || return 1
123+
}

‎.github/workflows/dcperf_ci_feedsim.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,3 @@ jobs:
8686

8787
- name: Install FeedSim
8888
run: . $PRELUDE; install_feedsim $BUILD_ENV
89-
continue-on-error: false

‎.github/workflows/dcperf_ci_mediawiki.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
name: DCPerf Mediawiki CI
7+
on:
8+
push:
9+
branches: [main]
10+
pull_request:
11+
workflow_dispatch:
12+
concurrency:
13+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
14+
cancel-in-progress: true
15+
jobs:
16+
build:
17+
strategy:
18+
fail-fast: false
19+
matrix:
20+
include:
21+
- name: Ubuntu x86
22+
runs-on: ubuntu-latest
23+
container: ubuntu:22.04
24+
type: ubuntu
25+
arch: x86
26+
- name: Ubuntu ARM
27+
runs-on: ubuntu-24.04-arm
28+
container: ubuntu:22.04
29+
type: ubuntu
30+
arch: aarch64
31+
- name: CentOS x86
32+
runs-on: ubuntu-latest
33+
container: quay.io/centos/centos:stream9
34+
type: centos
35+
arch: x86
36+
- name: CentOS ARM (self-hosted)
37+
runs-on: [self-hosted, centos9-arm]
38+
container: ""
39+
type: centos
40+
arch: aarch64
41+
runs-on: ${{ matrix.runs-on }}
42+
container: ${{ matrix.container != '' && matrix.container || null }}
43+
env:
44+
PRELUDE: .github/scripts/setup_env.bash
45+
BUILD_ENV: build_env
46+
steps:
47+
- name: Checkout the Repository
48+
uses: actions/checkout@v4
49+
with:
50+
submodules: true
51+
- name: Print System Info
52+
run: uname -a
53+
- name: Print Matrix Info
54+
run: |
55+
echo "Type: ${{ matrix.type }}"
56+
echo "Arch: ${{ matrix.arch }}"
57+
echo "Container: ${{ matrix.container }}"
58+
59+
60+
- name: Free Disk Space on Host
61+
run: . $PRELUDE; free_disk_space_on_host
62+
63+
- name: Display System Info
64+
run: . $PRELUDE; print_system_info
65+
66+
- name: Install Build Tools
67+
run: . $PRELUDE; install_build_tools
68+
69+
- name: Setup Miniconda
70+
run: . $PRELUDE; setup_miniconda $HOME/miniconda
71+
72+
- name: Create Conda Environment
73+
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
74+
75+
- name: Install Python Tools
76+
run: . $PRELUDE; install_python_tools $BUILD_ENV
77+
78+
- name: Install HHVM
79+
run: . $PRELUDE; install_hhvm $BUILD_ENV
80+
81+
- name: Install Mediawiki
82+
run: . $PRELUDE; install_mediawiki $BUILD_ENV

‎.github/workflows/dcperf_ci_spark.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,3 @@ jobs:
8686

8787
- name: Install Spark
8888
run: . $PRELUDE; install_spark $BUILD_ENV
89-
continue-on-error: false

‎.github/workflows/dcperf_ci_tao_bench.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,3 @@ jobs:
8686

8787
- name: Install Tao_bench
8888
run: . $PRELUDE; install_tao_bench $BUILD_ENV
89-
continue-on-error: false

‎.github/workflows/dcperf_ci_video_transcode_bench.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
# This workflow is used for DCPerf CI.
7+
name: DCPerf VideoTranscodeBench CI
8+
9+
on:
10+
# PR Trigger
11+
#
12+
pull_request:
13+
14+
# Push Trigger (enable to catch errors coming out of multiple merges)
15+
#
16+
push:
17+
branches:
18+
- main
19+
20+
# Manual Trigger
21+
#
22+
workflow_dispatch:
23+
24+
concurrency:
25+
# Cancel previous runs in the PR if a new commit is pushed
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
# Build on CPU hosts (generic GitHub runners)
31+
build_artifact:
32+
runs-on: ${{ matrix.host-machine.instance }}
33+
container:
34+
image: ${{ matrix.host-machine.container }}
35+
options: --user root --privileged --pid=host
36+
volumes:
37+
- /var/run/docker.sock:/var/run/docker.sock
38+
defaults:
39+
run:
40+
shell: bash
41+
env:
42+
PRELUDE: .github/scripts/setup_env.bash
43+
BUILD_ENV: build_env
44+
continue-on-error: false
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
host-machine: [
49+
{ arch: x86, instance: "ubuntu-latest", container: "ubuntu:22.04", type: "ubuntu" },
50+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "ubuntu:22.04", type: "ubuntu" },
51+
{ arch: x86, instance: "ubuntu-latest", container: "quay.io/centos/centos:stream9", type: "centos" },
52+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "quay.io/centos/centos:stream9", type: "centos" },
53+
]
54+
python-version: [ "3.12" ]
55+
steps:
56+
- name: Setup Build Container (ubuntu-based)
57+
if: ${{ matrix.host-machine.type == 'ubuntu' }}
58+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode gcc clang; clang --version; gcc --version
59+
60+
- name: Setup Build Container (centos-based)
61+
if: ${{ matrix.host-machine.type == 'centos' }}
62+
run: dnf update -y; dnf install -y git pciutils which dmidecode gcc clang
63+
64+
- name: Checkout the Repository
65+
uses: actions/checkout@v4
66+
with:
67+
submodules: true
68+
69+
- name: Free Disk Space on Host
70+
run: . $PRELUDE; free_disk_space_on_host
71+
72+
- name: Display System Info
73+
run: . $PRELUDE; print_system_info
74+
75+
- name: Install Build Tools
76+
run: . $PRELUDE; install_build_tools
77+
78+
- name: Setup Miniconda
79+
run: . $PRELUDE; setup_miniconda $HOME/miniconda
80+
81+
- name: Create Conda Environment
82+
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
83+
84+
- name: Install Python Tools
85+
run: . $PRELUDE; install_python_tools $BUILD_ENV
86+
87+
- name: Install VideoTranscodeBench
88+
run: . $PRELUDE; install_video_transcode_bench $BUILD_ENV

‎packages/mediawiki/cleanup_oss_performance_mediawiki.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@
77
MW_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P)
88
BENCHPRESS_ROOT="$(readlink -f "$MW_ROOT/../..")"
99

10-
sudo systemctl stop mariadb
10+
pkill mariadb
1111
rm -rf "${BENCHPRESS_ROOT}/oss-performance"
1212
rm -rf "${BENCHPRESS_ROOT}/benchmarks/oss_performance_mediawiki"

0 commit comments

Comments
 (0)