Skip to content

Commit 5b9a282

Browse files
joshuacurtissjoshuacurtiss
committed
feat: Speed up and streamline backup/restore process
Use standard `docker cp` tooling to transfer tar archives of the volumes, instead of running tar within the container. Also, if pbzip2, pigz, or pixz exist on the system, they will be used, instead of their standard compression tool equivalents that are only single-threaded (bzip2, gzip, xz). If bsdtar is available on the system, it will use that to filter the tar stream of the paths that don't need to be included in the backup. However, if bsdtar is not available, we also handle removing these paths during the restore process as well. So, we always ensure the caches and sessions are eliminated during restore. Closes #6.
1 parent 3254bfe commit 5b9a282

3 files changed

Lines changed: 130 additions & 108 deletions

File tree

‎lib/mdl-common.sh‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,18 @@ function support_long_options() {
8787
fi
8888
}
8989

90+
function calc_compression_tool() {
91+
local ext=${1##*.}
92+
local cmd
93+
[[ $ext == bz2 ]] && cmd=bzip2 && command -v pbzip2 &>/dev/null && cmd=pbzip2
94+
[[ $ext == gz ]] && cmd=gzip && command -v pigz &>/dev/null && cmd=pigz
95+
[[ $ext == xz ]] && cmd=xz && command -v pixz &>/dev/null && cmd=pixz
96+
echo "$cmd"
97+
}
98+
9099
# Receives file path and decompresses it. Can detect bzip2, gzip and xz files. If none of
91-
# those extensions match the filename, it throws an error. After successful decompression,
100+
# those extensions match the filename, it throws an error. It will always try to use the
101+
# parallel processing version of the command if available. After successful decompression,
92102
# the original file is deleted unless you specify `--keep`.
93103
#
94104
# Parameters:
@@ -99,14 +109,11 @@ function decompress() {
99109
local file_path=$1
100110
local out=$2
101111
local ext=${file_path##*.}
102-
local cmd
112+
local cmd=$(calc_compression_tool "$ext")
103113
# Check options
104114
[[ $* =~ -k || $* =~ --keep ]] && keep=true || keep=false
105115
# File path is required
106116
[[ -z $file_path ]] && return 1
107-
[[ $ext == bz2 ]] && cmd=bzip2
108-
[[ $ext == gz ]] && cmd=gzip
109-
[[ $ext == xz ]] && cmd=xz
110117
if [[ -n $cmd ]]; then
111118
# If they didn't provide an explicit output path, use file path sans extension
112119
[[ -z $out ]] && out=${file_path%".$ext"}

‎libexec/mdl-backup.sh‎

Lines changed: 83 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,19 @@ label=$default_label
1919
# Declarations
2020
source_host=
2121
source_type=
22-
compress_flag=
2322
ssh_args=
2423
container_tool=${MDL_CONTAINER_TOOL[*]}
2524
verbose=false
2625
dry_run=false
2726

27+
# Functions
28+
function ssh_wrap() {
29+
echo "${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}$1${source_host:+"\\\"\""}"
30+
}
31+
function eval_ssh_wrap() {
32+
eval "$(ssh_wrap "$1")"
33+
}
34+
2835
# Help
2936
display_help() {
3037
cat <<EOF
@@ -140,8 +147,16 @@ shift $((OPTIND - 1))
140147
# Calculations
141148
#
142149

150+
# Compression extension
151+
compress_ext=''
152+
case "$compress_arg" in
153+
bzip2) compress_ext='.bz2' ;;
154+
gzip) compress_ext='.gz' ;;
155+
xz) compress_ext='.xz' ;;
156+
esac
157+
143158
# Figure out the compression flag to send to tar
144-
[[ $compress_arg != none ]] && compress_flag="--$compress_arg"
159+
compression_tool=$(calc_compression_tool "$compress_ext")
145160

146161
#
147162
# Pre-environment Validation
@@ -177,7 +192,6 @@ if [[ $modules =~ fastdb ]]; then
177192
fi
178193
fi
179194

180-
181195
# Check necessary utilities
182196
cmds=(tar "${MDL_CONTAINER_TOOL[0]}")
183197
[[ $compress_arg != none ]] && cmds+=("$compress_arg")
@@ -241,42 +255,39 @@ for mname in $mnames; do
241255
ping -c 1 "$source_host_server" &> /dev/null && source_host_reachable=true || source_host_reachable=false
242256
fi
243257

244-
# If "container", the container must be active in order to dump the database for the "db" module.
258+
# If "container", the containers must be active in order to backup data for their module.
245259
source_db_container=''
260+
source_moodle_container=''
246261
if $is_container; then
247-
if [[ " $modules " == *" db "* ]]; then
248-
source_db_container_cmd="
249-
${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
250-
$container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep mariadb | head -1
251-
${source_host:+"\\\"\""}
252-
"
253-
source_db_container=$(eval "$source_db_container_cmd")
262+
if [[ " $modules " == *" db "* || $modules =~ fastdb ]]; then
263+
# Find the database container name
264+
source_db_container=$(eval_ssh_wrap "$container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep mariadb | head -1")
254265
[[ -z $source_db_container ]] && echo "${red}The database container cannot be found. Start the environment to perform a backup." >&2 && exit 1
266+
# Determine db path if not provided (usually won't be for container mode)
267+
if [[ -z $source_db_path && $modules =~ fastdb ]]; then
268+
source_db_path=$(eval_ssh_wrap "$container_tool inspect '$source_db_container' | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains(\"db\"))) | .Destination'")
269+
[[ -z $source_db_path ]] && echo "${red}Could not determine ${ul}db$rmul directory for $ul$mname$rmul!$norm" >&2 && exit 1
270+
fi
255271
fi
256-
if [[ $modules =~ data || $modules =~ src || $modules =~ fastdb ]]; then
257-
vols_cmd="
258-
${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
259-
$container_tool volume ls -q --filter 'label=com.docker.compose.project=$mname'
260-
${source_host:+"\\\"\""}
261-
"
262-
vols=$(eval "$vols_cmd")
263-
[[ $modules =~ data ]] && source_data_volume=$(grep data <<< "$vols")
264-
[[ $modules =~ src ]] && source_src_volume=$(grep src <<< "$vols")
265-
[[ $modules =~ fastdb ]] && source_db_volume=$(grep db <<< "$vols")
272+
if [[ $modules =~ data || $modules =~ src ]]; then
273+
# Find the moodle container name
274+
source_moodle_container=$(eval_ssh_wrap "$container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep moodle | head -1")
275+
[[ -z $source_moodle_container ]] && echo "${red}The Moodle container cannot be found. Start the environment to perform a backup." >&2 && exit 1
276+
# Determine src/data paths if not provided (usually won't be for container mode)
277+
if [[ -z $source_src_path && $modules =~ src ]]; then
278+
source_src_path=$(eval_ssh_wrap "$container_tool inspect '$source_moodle_container' | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains(\"src\"))) | .Destination'")
279+
[[ -z $source_src_path ]] && echo "${red}Could not determine ${ul}src$rmul directory for $ul$mname$rmul!$norm" >&2 && exit 1
280+
fi
281+
if [[ -z $source_data_path && $modules =~ data ]]; then
282+
source_data_path=$(eval_ssh_wrap "$container_tool inspect '$source_moodle_container' | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains(\"data\"))) | .Destination'")
283+
[[ -z $source_data_path ]] && echo "${red}Could not determine ${ul}data$rmul directory for $ul$mname$rmul!$norm" >&2 && exit 1
284+
fi
266285
fi
267286
fi
268287

269288
# Password mask
270289
source_db_password_mask=${source_db_password//?/*}
271290

272-
# Compression extension
273-
compress_ext=''
274-
case "$compress_arg" in
275-
bzip2) compress_ext='.bz2' ;;
276-
gzip) compress_ext='.gz' ;;
277-
xz) compress_ext='.xz' ;;
278-
esac
279-
280291
# Targets
281292
[[ $modules =~ data ]] && data_target="$MDL_BACKUP_DIR/${mname}_${label}_data.tar$compress_ext"
282293
[[ $modules =~ src ]] && src_target="$MDL_BACKUP_DIR/${mname}_${label}_src.tar$compress_ext"
@@ -287,25 +298,24 @@ for mname in $mnames; do
287298
echo -e "$ul$bold$action_word $mname environment$norm"
288299
if $verbose; then
289300
echo
290-
echo "$bold Type:$norm $source_type"
291-
[[ -n $source_host ]] && echo "$bold Host:$norm $source_host ($($source_host_reachable && echo "${green}reachable" || echo "${red}unreachable!")$norm)"
292-
[[ -n $source_src_path ]] && echo "$bold 'src' Path:$norm $source_src_path"
293-
[[ -n $source_data_path ]] && echo "$bold 'data' Path:$norm $source_data_path"
294-
[[ -n $source_src_volume ]] && echo "$bold 'src' Volume:$norm $source_src_volume"
295-
[[ -n $source_data_volume ]] && echo "$bold 'data' Volume:$norm $source_data_volume"
296-
[[ -n $source_db_volume ]] && echo "$bold 'db' Volume:$norm $source_db_volume"
297-
[[ -n $source_db_container ]] && echo "$bold DB Container:$norm $source_db_container"
298-
[[ -n $source_db_name ]] && echo "$bold DB Name:$norm $source_db_name"
299-
[[ -n $source_db_username ]] && echo "$bold DB Username:$norm $source_db_username"
300-
[[ -n $source_db_password ]] && echo "$bold DB Password:$norm $source_db_password_mask"
301+
echo "$bold Type:$norm $source_type"
302+
[[ -n $source_host ]] && echo "$bold Host:$norm $source_host ($($source_host_reachable && echo "${green}reachable" || echo "${red}unreachable!")$norm)"
303+
[[ -n $source_moodle_container ]] && echo "$bold Mdl Container:$norm $source_moodle_container"
304+
[[ -n $source_src_path ]] && echo "$bold 'src' Path:$norm $source_src_path"
305+
[[ -n $source_data_path ]] && echo "$bold 'data' Path:$norm $source_data_path"
306+
[[ -n $source_db_container ]] && echo "$bold DB Container:$norm $source_db_container"
307+
[[ -n $source_db_path ]] && echo "$bold 'db' Path:$norm $source_db_path"
308+
[[ -n $source_db_name ]] && echo "$bold DB Name:$norm $source_db_name"
309+
[[ -n $source_db_username ]] && echo "$bold DB Username:$norm $source_db_username"
310+
[[ -n $source_db_password ]] && echo "$bold DB Password:$norm $source_db_password_mask"
301311
echo
302-
[[ -n $label ]] && echo "$bold Label:$norm $label"
303-
echo "$bold Modules:$norm $modules"
304-
echo "$bold Compress:$norm $compress_arg"
305-
[[ -n $src_target ]] && echo "$bold 'src' Path:$norm $src_target"
306-
[[ -n $data_target ]] && echo "$bold 'data' Path:$norm $data_target"
307-
[[ -n $db_target ]] && echo "$bold DB Path:$norm $db_target"
308-
[[ -n $fastdb_target ]] && echo "$bold Fast DB Path:$norm $fastdb_target"
312+
[[ -n $label ]] && echo "$bold Label:$norm $label"
313+
echo "$bold Modules:$norm $modules"
314+
echo "$bold Compress:$norm $compress_arg"
315+
[[ -n $src_target ]] && echo "$bold 'src' Path:$norm $src_target"
316+
[[ -n $data_target ]] && echo "$bold 'data' Path:$norm $data_target"
317+
[[ -n $db_target ]] && echo "$bold DB Path:$norm $db_target"
318+
[[ -n $fastdb_target ]] && echo "$bold Fast DB Path:$norm $fastdb_target"
309319
echo
310320
fi
311321

@@ -315,42 +325,28 @@ for mname in $mnames; do
315325
fi
316326

317327
# MODULE: data
318-
# shellcheck disable=SC2034
319-
data_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
320-
$is_container && data_cmd="$data_cmd $container_tool run --rm --name '${mname}_worker_bk_data' -v '$source_data_volume':/data $MDL_SHELL_IMAGE"
321-
data_cmd="$data_cmd \
322-
tar c $compress_flag \
323-
--exclude='./trashdir' \
324-
--exclude='./temp' \
325-
--exclude='./sessions' \
326-
--exclude='./localcache' \
327-
--exclude='./cache' \
328-
--exclude='./moodle-cron.log' \
329-
-C $($is_container && echo /data || echo "$source_data_path") . \
328+
$is_container && data_cmd=$(ssh_wrap "$container_tool cp $source_moodle_container:$source_data_path/. -")
329+
$is_container || data_cmd=$(ssh_wrap "tar c -C '$source_data_path' .")
330+
command -v bsdtar &>/dev/null && data_cmd="$data_cmd | bsdtar cf - \
331+
--exclude localcache \
332+
--exclude cache \
333+
--exclude sessions \
334+
--exclude temp \
335+
--exclude trashdir \
336+
--exclude moodle-cron.log \
337+
@- \
330338
"
331-
data_cmd=$data_cmd${source_host:+"\\\"\""}
332339

333340
# MODULE: src
341+
$is_container && src_cmd=$(ssh_wrap "$container_tool cp $source_moodle_container:$source_src_path/. -")
334342
# shellcheck disable=SC2034
335-
src_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
336-
$is_container && src_cmd="$src_cmd $container_tool run --rm --name '${mname}_worker_bk_src' -v '$source_src_volume':/src $MDL_SHELL_IMAGE"
337-
src_cmd="$src_cmd \
338-
tar c $compress_flag \
339-
-C $($is_container && echo /src || echo "$source_src_path") . \
340-
"
341-
src_cmd=$src_cmd${source_host:+"\\\"\""}
343+
$is_container || src_cmd=$(ssh_wrap "tar c -C '$source_src_path' .")
342344

343345
# MODULE: fastdb
344346
# shellcheck disable=SC2034
345-
fastdb_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
346-
$is_container && fastdb_cmd="$fastdb_cmd $container_tool run --rm --name '${mname}_worker_bk_fastdb' -v '$source_db_volume':/db $MDL_SHELL_IMAGE"
347-
$is_container && fastdb_cmd="$fastdb_cmd tar c $compress_flag -C /db ."
348-
fastdb_cmd=$fastdb_cmd${source_host:+"\\\"\""}
347+
$is_container && fastdb_cmd=$(ssh_wrap "$container_tool cp $source_db_container:$source_db_path/. -")
349348

350349
# MODULE: db
351-
# TODO: When piping to compression program, a failed status of mysqldump will be lost.
352-
# shellcheck disable=SC2034
353-
db_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
354350
$is_container && db_cmd="$db_cmd $container_tool exec '$source_db_container'"
355351
db_cmd="$db_cmd \
356352
mysqldump \
@@ -360,22 +356,21 @@ for mname in $mnames; do
360356
-C -Q -e --create-options \
361357
$source_db_name \
362358
"
363-
db_cmd=$db_cmd${source_host:+"\\\"\""}
364-
[[ $compress_arg != none ]] && db_cmd="$db_cmd | $compress_arg -cq9"
359+
db_cmd=$(ssh_wrap "$db_cmd")
365360

366361
# Actually EXECUTE the commands
367362
pids=()
363+
$verbose && echo
368364
for t in $valid_modules; do
369365
if [[ " $modules " == *" $t "* ]]; then
370-
targ_var=${t}_target; targ=${!targ_var}
371-
cmd_var=${t}_cmd; cmd=${!cmd_var}
366+
targ_var="${t}_target"; targ="${!targ_var}"
367+
cmd_var="${t}_cmd"; cmd="${!cmd_var}"
368+
[[ -n $compression_tool ]] && cmd="$cmd | $compression_tool -cq9"
372369
echo "$mname $t: $targ"
373-
# In verbose mode, output the command, but mask the password and eliminate whitespace by echoing with word splitting.
374-
# shellcheck disable=2086
375-
$verbose && echo ${cmd//password=\'$source_db_password\'/password=\'$source_db_password_mask\'}
376-
if $dry_run; then
377-
echo -e "${red}Not executed. This is a dry run.$norm\n"
378-
else
370+
# In verbose mode, output the command, but mask password. Eliminate whitespace with `xargs`.
371+
# Ref: https://stackoverflow.com/questions/369758/how-to-trim-whitespace-from-a-bash-variable
372+
$verbose && echo "${bold}Command:$norm ${cmd//password=\'$source_db_password\'/password=\'$source_db_password_mask\'}" | xargs
373+
if ! $dry_run; then
379374
cmd="$cmd > '$targ'"
380375
# Execute the command, and handle success/fail scenarios
381376
eval "$cmd" && success=true || success=false
@@ -384,13 +379,14 @@ for mname in $mnames; do
384379
rm "$targ"
385380
echo "Removed $(basename "$targ") because the $t backup failed." >&2
386381
fi
387-
fi
388-
fi &
389-
pids+=($!)
382+
fi &
383+
pids+=($!)
384+
fi
390385
done
391386

392387
# TODO: It'd be nice if we check if any of the steps failed, and exit non-zero if so.
393388
wait "${pids[@]}"
389+
$dry_run && echo -e "${red}Commands not executed. This is a dry run.$norm\n"
394390
echo "$action_word of $mname environment is complete!"
395391

396392
# Unset environment variables

0 commit comments

Comments
 (0)