Skip to content

Commit 4d527e4

Browse files
excelle08meta-codesync[bot]
excelle08
authored and
meta-codesync[bot]
committed
Move DjangoBench source code to packages/django_workload/srcs, deprecate the patching approach (#244)
Summary: Pull Request resolved: #244 This diff deprecates the git clone and patch approach when installing DjangoBench; instead we copy the full DjangoBench source code to DCPerf's packages/django_workload/srcs folder. This will ensure easier code maintenance and development. Differential Revision: D83792235
1 parent 773be77 commit 4d527e4

67 files changed

Lines changed: 2597 additions & 1270 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎packages/django_workload/alternative_install_django_workload_aws.sh‎

Lines changed: 0 additions & 159 deletions
This file was deleted.

‎packages/django_workload/install_django_workload.sh‎

Lines changed: 11 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,10 @@ cd "$BP_TMP" || exit 1
4343
dnf install -y git memcached libmemcached-devel zlib-devel screen python36 \
4444
python36-devel python3-numpy
4545

46-
# Clone django-workload git repository
47-
git clone https://github.com/facebookarchive/django-workload
48-
# $OUT is set by benchpress and equal to "/path/to/benchpress/benchmarks"
49-
mv django-workload "${OUT}/"
46+
# Copy django-workload from srcs directory instead of cloning from GitHub
47+
mkdir -p "${OUT}/django-workload"
48+
cp -r "${DJANGO_PKG_ROOT}/srcs/django-workload/"* "${OUT}/django-workload/"
49+
cp -r "${DJANGO_PKG_ROOT}/srcs/bin" "${OUT}/"
5050

5151
# Download pip third-party dependencies for django-workload
5252
mkdir -p "${OUT}/django-workload/django-workload/third_party"
@@ -77,8 +77,10 @@ wget "https://files.pythonhosted.org/packages/47/33/c824f799128dfcfce2142f18d9bc
7777
wget "https://files.pythonhosted.org/packages/c7/75/45234f7b441c59b1eefd31ba3d1041a7e3c89602af24488e2a22e11e7259/uWSGI-2.0.19.1.tar.gz"
7878
popd
7979

80-
# Copy run script w/ execute permissions
81-
install -m755 -D "${TEMPLATES_DIR}/run.sh" "${OUT}/bin/run.sh"
80+
# Copy bin directory from srcs
81+
mkdir -p "${OUT}/bin"
82+
cp -r "${DJANGO_PKG_ROOT}/srcs/bin/"* "${OUT}/bin/"
83+
chmod +x "${OUT}/bin/"*.sh
8284

8385
# 2. Install JDK
8486
#JDK_NAME=fb-jdk_8u60-64
@@ -125,7 +127,7 @@ if ! [ -d "cinder" ]; then
125127
git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git
126128
pushd cinder
127129
mkdir -p cinder-build
128-
./configure --prefix="$(pwd)/cinder-build" --enable-optimizations
130+
./configure --prefix="$(pwd)/cinder-build" --enable-optimizations LN="ln -s"
129131
make -j
130132
make install
131133
popd
@@ -152,23 +154,8 @@ pip install "django-statsd-mozilla" --no-index --find-links file://"$OUT/django-
152154
pip install numpy --no-index --find-links file://"$OUT/django-workload/django-workload/third_party"
153155
pip install -e . --no-index --find-links file://"$OUT/django-workload/django-workload/third_party"
154156

155-
# Configure Django and uWSGI
156-
cp "${TEMPLATES_DIR}/cluster_settings.py" "${OUT}/django-workload/django-workload/cluster_settings.py.template" || exit 1
157-
cp "${TEMPLATES_DIR}/uwsgi.ini" "${OUT}/django-workload/django-workload/uwsgi.ini" || exit 1
158-
cp "${TEMPLATES_DIR}/urls_template.txt" "${OUT}/django-workload/client/urls_template.txt" || exit 1
159-
160-
# Install the modified run-siege script
161-
cp "${TEMPLATES_DIR}/run-siege" "${DJANGO_REPO_ROOT}/client/run-siege" || exit 1
162-
163-
# Patch for MLP and icache buster
164-
# cltorres: Disable MLP patch. MLP implemented in Python does not work as intented due to bytecode abstraction
165-
# git apply --check "${TEMPLATES_DIR}/django_mlp.patch" && git apply "${TEMPLATES_DIR}/django_mlp.patch"
166-
pushd "${DJANGO_REPO_ROOT}"
167-
git apply --check "${TEMPLATES_DIR}/django_genurl.patch" && git apply "${TEMPLATES_DIR}/django_genurl.patch"
168-
git apply --check "${TEMPLATES_DIR}/django_libib.patch" && git apply "${TEMPLATES_DIR}/django_libib.patch"
169-
popd # ${DJANGO_REPO_ROOT}
170-
171157
# Build oldisim icache buster library
158+
# Will only build once, the outcome library can be used in both CPython and Cinder
172159
set +u
173160
if [ ! -f "${OUT}/django-workload/django-workload/libicachebuster.so" ]; then
174161
if [ -z "${IBCC}" ]; then
@@ -187,11 +174,7 @@ if [ ! -f "${OUT}/django-workload/django-workload/libicachebuster.so" ]; then
187174
rm -rfv build/
188175
fi
189176

190-
# Apply Memcache tuning
191-
cd "${OUT}/django-workload/django-workload/django_workload" || exit 1
192-
git apply --check "${TEMPLATES_DIR}/0002-Memcache-Tuning.patch" && git apply "${TEMPLATES_DIR}/0002-Memcache-Tuning.patch"
193-
# Apply db caching
194-
git apply --check "${TEMPLATES_DIR}/0003-bundle_tray_caching.patch" && git apply "${TEMPLATES_DIR}/0003-bundle_tray_caching.patch"
177+
# No need to copy template files as they are already in the srcs directory
195178

196179
deactivate
197180

@@ -206,7 +189,6 @@ pip install "django-statsd-mozilla" --no-index --find-links file://"$OUT/django-
206189
pip install numpy --no-index --find-links file://"$OUT/django-workload/django-workload/third_party"
207190
pip install -e . --no-index --find-links file://"$OUT/django-workload/django-workload/third_party"
208191

209-
deactivate
210192
popd # ${OUT}/django-workload/django-workload
211193
pip install -e . --no-index --find-links file://"$OUT/django-workload/django-workload/third_party"
212194

‎packages/django_workload/install_django_workload_aarch64.sh‎

Lines changed: 26 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,14 @@ dnf groupinstall "Development Tools" -y --exclude="texlive*"
1919
dnf install -y memcached libmemcached-awesome-devel zlib-devel screen \
2020
openssl-devel bzip2-devel libffi-devel wget make xz-devel
2121
#s
22-
# Clone django-workload git repository
22+
# Copy django-workload from srcs directory instead of cloning from GitHub
2323
mkdir -p "${DJANGO_WORKLOAD_ROOT}"
2424
pushd "${DJANGO_WORKLOAD_ROOT}"
2525
if ! [ -d "django-workload" ]; then
26-
git clone https://github.com/facebookarchive/django-workload
26+
mkdir -p "django-workload"
27+
cp -r "${DJANGO_PKG_ROOT}/srcs/django-workload/"* "django-workload/"
2728
else
28-
echo "[SKIPPED] cloning django-workload"
29+
echo "[SKIPPED] copying django-workload"
2930
fi
3031

3132
# Download pip third-party dependencies for django-workload
@@ -130,8 +131,10 @@ wget "https://files.pythonhosted.org/packages/11/35/575091de594677e40440a24be319
130131
popd
131132
unalias wget 2>/dev/null || echo "[Finished] downloading dependencies"
132133

133-
# Copy run script w/ execute permissions
134-
install -m755 -D "${TEMPLATES_DIR}/run.sh" "${DJANGO_WORKLOAD_ROOT}/bin/run.sh"
134+
# Copy bin directory from srcs
135+
mkdir -p "${DJANGO_WORKLOAD_ROOT}/bin"
136+
cp -r "${DJANGO_PKG_ROOT}/srcs/bin/"* "${DJANGO_WORKLOAD_ROOT}/bin/"
137+
chmod +x "${DJANGO_WORKLOAD_ROOT}/bin/"*.sh
135138

136139
# 2. Install JDK
137140
JDK_NAME=java-1.8.0-openjdk-devel
@@ -177,8 +180,8 @@ if ! [ -d Python-3.10.2 ]; then
177180
wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz
178181
tar -xzf Python-3.10.2.tgz
179182
cd Python-3.10.2
180-
./configure --enable-optimizations
181-
make altinstall
183+
./configure --enable-optimizations --prefix="$(pwd)/python-build" --enable-shared LN="ln -s"
184+
make install
182185
cd ../
183186
fi
184187

@@ -187,54 +190,41 @@ if ! [ -d "cinder" ]; then
187190
git clone -b cinder/3.10 https://github.com/facebookincubator/cinder.git
188191
pushd cinder
189192
mkdir -p cinder-build
190-
./configure --prefix="$(pwd)/cinder-build" --enable-optimizations
193+
./configure --prefix="$(pwd)/cinder-build" --enable-optimizations --enable-shared LN="ln -s"
191194
make -j
192195
make install
193196
popd
194197
fi
195198

196199
# Create virtual environments for both CPython and Cinder
197200
# Create CPython virtual env
198-
[ ! -d venv_cpython ] && Python-3.10.2/python -m venv venv_cpython
201+
CPYTHON_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build"
202+
export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib"
203+
[ ! -d venv_cpython ] && "${CPYTHON_INSTALL_PREFIX}/bin/python3.10" -m venv venv_cpython
199204

200205
# Create Cinder virtual env
201-
[ ! -d venv_cinder ] && "${DJANGO_SERVER_ROOT}/cinder/cinder-build/bin/python3" -m venv venv_cinder
206+
CINDER_INSTALL_PREFIX="${DJANGO_SERVER_ROOT}/cinder/cinder-build"
207+
export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib"
208+
[ ! -d venv_cinder ] && "${CINDER_INSTALL_PREFIX}/bin/python3" -m venv venv_cinder
202209

203210
# Install packages in both virtual environments
204211
# First, CPython environment
205212
set +u
206213
# shellcheck disable=SC1091
207214
source ./venv_cpython/bin/activate
208215
set -u
209-
if ! [ -f setup.py.bak ]; then
210-
#sed -i 's/django-cassandra-engine/django-cassandra-engine >= 1.6, < 1.9/' setup.py
211-
sed -i '/Django/s/.*//' setup.py
212-
sed -i "/uwsgi/s/.*/ 'uwsgi',/" setup.py
213-
cp setup.py setup.py.bak
214-
fi
215-
216-
cp setup.py.bak setup.py
217216

217+
export LD_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib"
218+
export CMAKE_LIBRARY_PATH="${CPYTHON_INSTALL_PREFIX}/lib"
219+
export CPATH="${DJANGO_SERVER_ROOT}/Python-3.10.2/python-build/include:${DJANGO_SERVER_ROOT}/Python-3.10.2/Include"
218220
# Install dependencies using third_party pip dependencies
219221
pip3.10 install "django-statsd-mozilla" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}"
220222
pip3.10 install "numpy>=1.19" --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}"
221223
pip3.10 install -e . --no-index --find-links file://"${DJANGO_WORKLOAD_DEPS}"
222224

223-
# Configure Django and uWSGI
224-
cp "${TEMPLATES_DIR}/cluster_settings.py" "${DJANGO_SERVER_ROOT}/cluster_settings.py.template" || exit 1
225-
cp "${TEMPLATES_DIR}/uwsgi.ini" "${DJANGO_SERVER_ROOT}/uwsgi.ini" || exit 1
226-
cp "${TEMPLATES_DIR}/urls_template.txt" "${DJANGO_REPO_ROOT}/client/urls_template.txt" || exit 1
227-
228-
# Install the modified run-siege script
229-
cp "${TEMPLATES_DIR}/run-siege" "${DJANGO_REPO_ROOT}/client/run-siege" || exit 1
225+
# No need to copy configuration files as they are already in the srcs directory
230226

231-
# Patch for MLP and icache buster
232-
# cltorres: Disable MLP patch. MLP implemented in Python does not work as intented due to bytecode abstraction
233-
# git apply --check "${TEMPLATES_DIR}/django_mlp.patch" && git apply "${TEMPLATES_DIR}/django_mlp.patch"
234-
pushd "${DJANGO_REPO_ROOT}"
235-
git apply --check "${TEMPLATES_DIR}/django_genurl.patch" && git apply "${TEMPLATES_DIR}/django_genurl.patch"
236-
git apply --check "${TEMPLATES_DIR}/django_libib.patch" && git apply "${TEMPLATES_DIR}/django_libib.patch"
237-
popd # ${DJANGO_REPO_ROOT}
227+
# No need to apply patches as the code in srcs already has the desired changes
238228

239229
# Build oldisim icache buster library
240230
set +u
@@ -255,34 +245,18 @@ if [ ! -f "${OUT}/django-workload/django-workload/libicachebuster.so" ]; then
255245
rm -rfv build/
256246
fi
257247

258-
pushd "${BENCHPRESS_ROOT}"
259-
# Patch for Java
260-
git apply --check "${TEMPLATES_DIR}/cassandra-env.patch" && git apply "${TEMPLATES_DIR}/cassandra-env.patch"
261-
git apply --check "${TEMPLATES_DIR}/jvm_options.patch" && git apply "${TEMPLATES_DIR}/jvm_options.patch"
248+
# Configure Java options directly
262249
# shellcheck disable=SC2016
263250
echo 'JVM_OPTS="$JVM_OPTS -Xss512k"' >> "${DJANGO_WORKLOAD_ROOT}/apache-cassandra/conf/cassandra-env.sh"
264-
# Patch for gen-urls-file
265-
git apply --check "${TEMPLATES_DIR}/gen-urls-file.patch" && git apply "${TEMPLATES_DIR}/gen-urls-file.patch"
266-
popd
267-
268-
pushd "${DJANGO_SERVER_ROOT}/django_workload"
269-
# Patch for URLs
270-
git apply --check "${TEMPLATES_DIR}/urls.patch" && git apply "${TEMPLATES_DIR}/urls.patch"
271251

272-
# Apply Memcache tuning
273-
git apply --check "${TEMPLATES_DIR}/0002-Memcache-Tuning.patch" && git apply "${TEMPLATES_DIR}/0002-Memcache-Tuning.patch"
274-
# Apply db caching
275-
git apply --check "${TEMPLATES_DIR}/0003-bundle_tray_caching.patch" && git apply "${TEMPLATES_DIR}/0003-bundle_tray_caching.patch"
276-
# Remove duplicate middleware classes
277-
git apply --check "${TEMPLATES_DIR}/0004-del_dup_middleware_classes.patch" && git apply "${TEMPLATES_DIR}/0004-del_dup_middleware_classes.patch"
278-
# Enable Session, Authentication and Message middleware
279-
git apply --check "${TEMPLATES_DIR}/0005-django_middleware_settings.patch" && git apply "${TEMPLATES_DIR}/0005-django_middleware_settings.patch"
280-
popd
252+
# No need to copy template files as they are already in the srcs directory
281253

282254
deactivate
283255

284256
# Now install packages in Cinder environment
285257
pushd "${DJANGO_SERVER_ROOT}" # Make sure we're in the right directory
258+
export LD_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib"
259+
export CMAKE_LIBRARY_PATH="${CINDER_INSTALL_PREFIX}/lib"
286260
export CPATH="${DJANGO_SERVER_ROOT}/cinder/cinder-build/include:${DJANGO_SERVER_ROOT}/cinder/Include"
287261
source ./venv_cinder/bin/activate
288262
set -u

0 commit comments

Comments
 (0)