Skip to content

README.md

Latest commit

 

History

History
301 lines (252 loc) · 6.19 KB

README.md

File metadata and controls

301 lines (252 loc) · 6.19 KB
Raw

FALL3D Workflow

The cloud directory contains information and scripts used to submit the workflow to the cloud environment (Hetzner).

output stores the simulation output.

BUILD.md describes how the FALL3D model was compiled on each environment.

Running FALL3D+CWL+MPI on the laptop or in the cloud environment

baseCommand: mpirun, no containers

SIM=workflow-base
ENV=laptop  # or cloud
JOBFILE="arguments_etna_laptop.yml"  # or arguments_etna_cloud.yml
CONTAINER=none

LOGDIR="logs/cwltool/$ENV/$CONTAINER"
mkdir -p "$LOGDIR"

CMD="cwltool \
  --no-container \
  --debug \
  --strict-memory-limit \
  --strict-cpu-limit \
  fall3d-what-if-volcanos.0.2.1.cwl#demo-etna \
  $JOBFILE"

echo "COMMAND: $CMD" > "$LOGDIR/$SIM.cmd"
START=$(date +%s)
bash -c "$CMD" > "$LOGDIR/$SIM.log" 2>&1
EXIT=$?
END=$(date +%s)
DURATION=$((END - START))
echo "$DURATION" > "$LOGDIR/$SIM.time"
echo "$EXIT" > "$LOGDIR/$SIM.exit"

baseCommand: mpirun, with Singularity

SIM=workflow-base
ENV=laptop  # or cloud
JOBFILE="arguments_etna_container.yml"
CONTAINER=singularity

LOGDIR="logs/cwltool/$ENV/$CONTAINER"
mkdir -p "$LOGDIR"

CMD="cwltool \
  --singularity \
  --debug \
  --strict-memory-limit \
  --strict-cpu-limit \
  fall3d-what-if-volcanos.0.2.1.cwl#demo-etna \
  $JOBFILE"

echo "COMMAND: $CMD" > "$LOGDIR/$SIM.cmd"
START=$(date +%s)
bash -c "$CMD" > "$LOGDIR/$SIM.log" 2>&1
EXIT=$?
END=$(date +%s)
DURATION=$((END - START))
echo "$DURATION" > "$LOGDIR/$SIM.time"
echo "$EXIT" > "$LOGDIR/$SIM.exit"

MPIRequirement, no containers

SIM=workflow-req
ENV=laptop  # or cloud
JOBFILE="arguments_etna_laptop.yml"  # or arguments_etna_cloud.yml
CONTAINER=none
MPICONFIGFILE="mpi-config-file-laptop.yml"  # or mpi-config-file-cloud.yml

LOGDIR="logs/cwltool/$ENV/$CONTAINER"
mkdir -p "$LOGDIR"

CMD="cwltool \
  --no-container \
  --enable-ext \
  --mpi-config-file $MPICONFIGFILE \
  --debug \
  --strict-memory-limit \
  --strict-cpu-limit \
  fall3d-what-if-volcanos.0.2.1_mpirequirement.cwl#demo-etna \
  $JOBFILE"

echo "COMMAND: $CMD" > "$LOGDIR/$SIM.cmd"
START=$(date +%s)
bash -c "$CMD" > "$LOGDIR/$SIM.log" 2>&1
EXIT=$?
END=$(date +%s)
DURATION=$((END - START))
echo "$DURATION" > "$LOGDIR/$SIM.time"
echo "$EXIT" > "$LOGDIR/$SIM.exit"

MPIRequirement, with Singularity

SIM=workflow-req
ENV=laptop  # or cloud
JOBFILE="arguments_etna_container.yml"
CONTAINER=singularity
MPICONFIGFILE="mpi-config-file-laptop.yml"  # or mpi-config-file-cloud.yml

LOGDIR="logs/cwltool/$ENV/$CONTAINER"
mkdir -p "$LOGDIR"

CMD="cwltool \
  --singularity \
  --enable-ext \
  --mpi-config-file $MPICONFIGFILE \
  --debug \
  --strict-memory-limit \
  --strict-cpu-limit \
  fall3d-what-if-volcanos.0.2.1_mpirequirement.cwl#demo-etna \
  $JOBFILE"

echo "COMMAND: $CMD" > "$LOGDIR/$SIM.cmd"
START=$(date +%s)
bash -c "$CMD" > "$LOGDIR/$SIM.log" 2>&1
EXIT=$?
END=$(date +%s)
DURATION=$((END - START))
echo "$DURATION" > "$LOGDIR/$SIM.time"
echo "$EXIT" > "$LOGDIR/$SIM.exit"

Running FALL3D+CWL+MPI on LUMI

baseCommand: mpirun, no containers

SIM=workflow-base
ENV=lumi
CONTAINER=none
JOBNAME="${SIM}_${ENV}_${CONTAINER}"

LOGDIR="logs/cwltool/$ENV/$CONTAINER"
mkdir -p "$LOGDIR"

CMD="cwltool \
  --no-container \
  --debug \
  --strict-memory-limit \
  --strict-cpu-limit \
  fall3d-what-if-volcanos.0.2.1.cwl#demo-etna \
  arguments_etna_lumi.yml"

echo "COMMAND: $CMD" > "$LOGDIR/$SIM.cmd"

sbatch \
  --partition=small \
  --account=[REDACTED] \
  --time=00:30:00 \
  --job-name="$JOBNAME" \
  --nodes=2 \
  --ntasks-per-node=32 \
  --cpus-per-task=1 \
  --export=ALL \
  --wrap="
START=\$(date +%s)
$CMD > '$LOGDIR/$SIM.log' 2>&1
EXIT=\$?
END=\$(date +%s)
echo \$((END-START)) > '$LOGDIR/$SIM.time'
echo \$EXIT > '$LOGDIR/$SIM.exit'
exit \$EXIT
"

baseCommand: mpirun, with Singularity

SIM=workflow-base
ENV=lumi
CONTAINER=singularity
JOBNAME="${SIM}_${ENV}_${CONTAINER}"

LOGDIR="logs/cwltool/$ENV/$CONTAINER"
mkdir -p "$LOGDIR"

CMD="cwltool \
  --singularity \
  --debug \
  --strict-memory-limit \
  --strict-cpu-limit \
  fall3d-what-if-volcanos.0.2.1.cwl#demo-etna \
  arguments_etna_container.yml"

echo "COMMAND: $CMD" > "$LOGDIR/$SIM.cmd"

sbatch \
  --partition=small \
  --account=[REDACTED] \
  --time=00:30:00 \
  --job-name="$JOBNAME" \
  --nodes=2 \
  --ntasks-per-node=32 \
  --cpus-per-task=1 \
  --export=ALL \
  --wrap="
START=\$(date +%s)
$CMD > '$LOGDIR/$SIM.log' 2>&1
EXIT=\$?
END=\$(date +%s)
echo \$((END-START)) > '$LOGDIR/$SIM.time'
echo \$EXIT > '$LOGDIR/$SIM.exit'
exit \$EXIT

MPIRequirement, no containers

SIM=workflow-req
ENV=lumi
CONTAINER=none
JOBNAME="${SIM}_${ENV}_${CONTAINER}"

LOGDIR="logs/cwltool/$ENV/$CONTAINER"
mkdir -p "$LOGDIR"

CMD="cwltool \
  --no-container \
  --enable-ext \
  --mpi-config-file mpi-config-file-lumi.yml \
  --debug \
  --strict-memory-limit \
  --strict-cpu-limit \
  fall3d-what-if-volcanos.0.2.1_mpirequirement.cwl#demo-etna \
  arguments_etna_lumi.yml"

echo "COMMAND: $CMD" > "$LOGDIR/$SIM.cmd"

sbatch \
  --partition=small \
  --account=[REDACTED] \
  --time=00:30:00 \
  --job-name="$JOBNAME" \
  --nodes=2 \
  --ntasks-per-node=32 \
  --cpus-per-task=1 \
  --export=ALL \
  --wrap="
START=\$(date +%s)
$CMD > '$LOGDIR/$SIM.log' 2>&1
EXIT=\$?
END=\$(date +%s)
echo \$((END-START)) > '$LOGDIR/$SIM.time'
echo \$EXIT > '$LOGDIR/$SIM.exit'
exit \$EXIT
"

MPIRequirement, with Singularity

SIM=workflow-req
ENV=lumi
CONTAINER=singularity
JOBNAME="${SIM}_${ENV}_${CONTAINER}"

LOGDIR="logs/cwltool/$ENV/$CONTAINER"
mkdir -p "$LOGDIR"

CMD="cwltool \
  --singularity \
  --enable-ext \
  --mpi-config-file mpi-config-file-lumi.yml \
  --debug \
  --strict-memory-limit \
  --strict-cpu-limit \
  fall3d-what-if-volcanos.0.2.1_mpirequirement.cwl#demo-etna \
  arguments_etna_container.yml"

echo "COMMAND: $CMD" > "$LOGDIR/$SIM.cmd"

sbatch \
  --partition=small \
  --account=[REDACTED] \
  --time=00:30:00 \
  --job-name="$JOBNAME" \
  --nodes=2 \
  --ntasks-per-node=32 \
  --cpus-per-task=1 \
  --export=ALL \
  --wrap="
START=\$(date +%s)
$CMD > '$LOGDIR/$SIM.log' 2>&1
EXIT=\$?
END=\$(date +%s)
echo \$((END-START)) > '$LOGDIR/$SIM.time'
echo \$EXIT > '$LOGDIR/$SIM.exit'
exit \$EXIT