Merge pull request #635 from broadinstitute/upgrade-wdl-1.1

Upgrade WDL from 1.0 to 1.1

Author: dpark01

github_actions_ci/build-dx.sh

@@ -2,7 +2,7 @@
 set -e -o pipefail
 
 if [ -z "$DX_API_TOKEN" ]; then
-  echo "ERROR: DX_API_TOKEN is not set, this is needed to build dxWDL workflows."
+  echo "ERROR: DX_API_TOKEN is not set, this is needed to build DNAnexus workflows."
   exit 1
 fi
 
@@ -14,8 +14,8 @@ source dx-toolkit/environment
 dx login --token "$DX_API_TOKEN" --noprojects
 dx select $DX_PROJECT
 
-# compile with dxWDL
-COMPILE_SUCCESS="dxWDL-compile_all-success.txt"
+# compile with dxCompiler
+COMPILE_SUCCESS="dxCompiler-compile_all-success.txt"
 touch $COMPILE_SUCCESS
 for workflow in pipes/WDL/workflows/*.wdl; do
   if [ -n "$(grep DX_SKIP_WORKFLOW $workflow)" ]; then
@@ -34,7 +34,7 @@ for workflow in pipes/WDL/workflows/*.wdl; do
     extras_json="pipes/dnax/dx-extras.json"
     CMD_DEFAULTS+=" -extras $extras_json"
 
-	  dx_id=$(java -jar dxWDL.jar compile \
+	  dx_id=$(java -jar dxCompiler.jar compile \
       $workflow $CMD_DEFAULTS -f -verbose \
       -leaveWorkflowsOpen \
       -imports pipes/WDL/tasks/ \
@@ -50,27 +50,42 @@ for workflow in pipes/WDL/workflows/*.wdl; do
   fi
 done
 
-# build consolidate_run_tarballs (native DNAnexus applet) applet
-pushd pipes/dnax/dx-launcher
-cp consolidate_run_tarballs.yml consolidate_run_tarballs_dxapp.yml
-consolidate_tarballs_dx_id=$(./dx-yml-build consolidate_run_tarballs_dxapp.yml -a --destination /build/$VERSION/ | jq -r ".id")
-popd
-echo -e "consolidate_run_tarballs\t$dx_id" >> $COMPILE_SUCCESS
-
 # Special case: build demux launchers (native DNAnexus applets), embedding the
-# demux workflow ID as a default input 
+# demux workflow ID as a default input. Skip if no demux workflows were compiled.
 demux_workflows_to_build="demux_plus demux_only"
+any_demux_compiled=false
 for wf_name in $(echo "${demux_workflows_to_build}"); do
-  echo "Building applet ${wf_name}..."
-  demux_workflow_id=$(grep "^${wf_name}\s" $COMPILE_SUCCESS | cut -f 2)
+  if grep -q "^${wf_name}\s" $COMPILE_SUCCESS; then
+    any_demux_compiled=true
+  fi
+done
+
+if [ "$any_demux_compiled" = true ]; then
+  # build consolidate_run_tarballs (native DNAnexus applet) applet
   pushd pipes/dnax/dx-launcher
-  sed "s/DEFAULT_DEMUX_WORKFLOW_ID/$demux_workflow_id/" demux_launcher.yml \
-    | sed "s/DEFAULT_DEMUX_WORKFLOW_NAME/${wf_name}_launcher/" \
-    | sed "s/DEFAULT_CONSOLIDATE_RUN_TARBALLS_APPLET_ID/$consolidate_tarballs_dx_id/" > "${wf_name}_dxapp.yml"
-  dx_id=$(./dx-yml-build ${wf_name}_dxapp.yml -a --destination /build/$VERSION/ | jq -r ".id")
+  cp consolidate_run_tarballs.yml consolidate_run_tarballs_dxapp.yml
+  consolidate_tarballs_dx_id=$(./dx-yml-build consolidate_run_tarballs_dxapp.yml -a --destination /build/$VERSION/ | jq -r ".id")
   popd
-  echo -e "${wf_name}_launcher\t$dx_id" >> $COMPILE_SUCCESS
-done
+  echo -e "consolidate_run_tarballs\t$consolidate_tarballs_dx_id" >> $COMPILE_SUCCESS
+
+  for wf_name in $(echo "${demux_workflows_to_build}"); do
+    demux_workflow_id=$(grep "^${wf_name}\s" $COMPILE_SUCCESS | cut -f 2)
+    if [ -z "$demux_workflow_id" ]; then
+      echo "Skipping applet ${wf_name}_launcher (${wf_name} was not compiled)"
+      continue
+    fi
+    echo "Building applet ${wf_name}..."
+    pushd pipes/dnax/dx-launcher
+    sed "s/DEFAULT_DEMUX_WORKFLOW_ID/$demux_workflow_id/" demux_launcher.yml \
+      | sed "s/DEFAULT_DEMUX_WORKFLOW_NAME/${wf_name}_launcher/" \
+      | sed "s/DEFAULT_CONSOLIDATE_RUN_TARBALLS_APPLET_ID/$consolidate_tarballs_dx_id/" > "${wf_name}_dxapp.yml"
+    dx_id=$(./dx-yml-build ${wf_name}_dxapp.yml -a --destination /build/$VERSION/ | jq -r ".id")
+    popd
+    echo -e "${wf_name}_launcher\t$dx_id" >> $COMPILE_SUCCESS
+  done
+else
+  echo "Skipping consolidate_run_tarballs and demux launchers (no demux workflows were compiled)"
+fi
 
 # the presence of this file in the project denotes successful build
 dx upload --brief --no-progress --destination /build/$VERSION/ $COMPILE_SUCCESS

github_actions_ci/install-wdl.sh

@@ -12,10 +12,34 @@ fetch_jar_from_github () {
 	ln -s $_jar_fname $_tool_name.jar
 }
 
-fetch_jar_from_github broadinstitute cromwell womtool 88
-fetch_jar_from_github broadinstitute cromwell cromwell 88
-fetch_jar_from_github dnanexus dxWDL dxWDL v1.50
-fetch_jar_from_github dnanexus dxCompiler dxCompiler 2.11.6
+# Cromwell & womtool: pull the latest daily "develop" build from Docker Hub and copy the
+# jars out of the image. This is the same artifact Terra runs, so CI tests against what
+# Terra actually uses rather than a stale pinned GitHub release (Cromwell no longer cuts
+# GitHub releases -- they only publish daily builds to Docker Hub). The daily build
+# publishes only Docker images -- no standalone jars are hosted anywhere -- so we
+# exfiltrate the jars here rather than running cromwell/womtool inside docker (which would
+# break docker-in-docker task execution).
+extract_jar_from_docker () {
+	_image=$1   # e.g. broadinstitute/cromwell:develop
+	_src=$2     # path inside image, e.g. /app/cromwell.jar
+	_dest=$3    # output filename in cwd, e.g. cromwell.jar
+	echo "Pulling $_image to extract $_dest"
+	docker pull --quiet "$_image"
+	# --user keeps host file ownership as the runner (not root); cp -L dereferences the
+	# versioned symlink (docker cp would copy a broken symlink since the jar filename
+	# embeds a daily-changing git hash); readlink logs the exact build to CI output.
+	docker run --rm \
+		--user "$(id -u):$(id -g)" \
+		-v "$PWD:/out" \
+		--entrypoint /bin/bash \
+		"$_image" \
+		-c "echo -n 'extracted '; readlink -f '$_src'; cp -L '$_src' '/out/$_dest'"
+}
+
+extract_jar_from_docker broadinstitute/cromwell:develop /app/cromwell.jar cromwell.jar
+extract_jar_from_docker broadinstitute/womtool:develop /app/womtool.jar womtool.jar
+
+fetch_jar_from_github dnanexus dxCompiler dxCompiler 2.15.0
 
 TGZ=dx-toolkit-v0.311.0-ubuntu-20.04-amd64.tar.gz
 echo "Fetching $TGZ"

github_actions_ci/tests-dx.sh

@@ -11,7 +11,7 @@ if [ -n "$DX_API_TOKEN" ]; then
   dx select $DX_PROJECT
 fi
 
-COMPILE_SUCCESS="dxWDL-compile_all-success.txt"
+COMPILE_SUCCESS="dxCompiler-compile_all-success.txt"
 if [ ! -f $COMPILE_SUCCESS ]; then
   dx download --no-progress /build/$VERSION/$COMPILE_SUCCESS
 fi
@@ -79,22 +79,25 @@ else
 fi
 
 # Special case: run test for the demux_(plus|only)_launcher native applet (which invokes
-# the demux_(plus|only) WDL workflow)
-demux_launcher_id=$(grep "^${demux_name}_launcher\s" $COMPILE_SUCCESS | cut -f 2)
+# the demux_(plus|only) WDL workflow). Skip if the launcher was not built.
+demux_launcher_id=$(grep "^${demux_name}_launcher\s" $COMPILE_SUCCESS | cut -f 2 || true)
+demux_workflow_id=$(grep "^${demux_name}\s" $COMPILE_SUCCESS | cut -f 2 || true)
 
-demux_workflow_id=$(grep "^${demux_name}\s" $COMPILE_SUCCESS | cut -f 2)
-
-timeout_args=$(dx_run_timeout_args $demux_workflow_id $demux_launcher_id)
-dx_job_id=$(dx run "${demux_launcher_id}" \
-  -y --brief \
-  -i upload_sentinel_record=record-Bv8qkgQ0jy198GK0QVz2PV8Y \
-  -i demux_workflow_id=${demux_workflow_id} \
-  --name "$VERSION ${demux_name}_launcher" \
-  -i folder=/tests/$VERSION/${demux_name}_launcher \
-  --extra-args $timeout_args \
-  )
-echo "Launched ${demux_name}_launcher as $dx_job_id"
-echo -e "${demux_name}_launcher\t$demux_launcher_id\t$dx_job_id" >> $TEST_LAUNCH_ALL
+if [ -n "$demux_launcher_id" -a -n "$demux_workflow_id" ]; then
+  timeout_args=$(dx_run_timeout_args $demux_workflow_id $demux_launcher_id)
+  dx_job_id=$(dx run "${demux_launcher_id}" \
+    -y --brief \
+    -i upload_sentinel_record=record-Bv8qkgQ0jy198GK0QVz2PV8Y \
+    -i demux_workflow_id=${demux_workflow_id} \
+    --name "$VERSION ${demux_name}_launcher" \
+    -i folder=/tests/$VERSION/${demux_name}_launcher \
+    --extra-args $timeout_args \
+    )
+  echo "Launched ${demux_name}_launcher as $dx_job_id"
+  echo -e "${demux_name}_launcher\t$demux_launcher_id\t$dx_job_id" >> $TEST_LAUNCH_ALL
+else
+  echo "Skipping ${demux_name}_launcher test (launcher was not built)"
+fi
 
 # the presence of this file in the project denotes all tests launched
 dx upload --brief --no-progress --destination /build/$VERSION/ $TEST_LAUNCH_ALL

pipes/WDL/tasks/tasks_16S_amplicon.wdl

@@ -1,4 +1,4 @@
-version 1.0
+version 1.1
 
 task qiime_import_from_bam {
     meta {

pipes/WDL/tasks/tasks_assembly.wdl

@@ -1,4 +1,4 @@
-version 1.0
+version 1.1
 
 task assemble {
     input {

pipes/WDL/tasks/tasks_demux.wdl

@@ -1,4 +1,4 @@
-version 1.0
+version 1.1
 
 task merge_tarballs {
   input {

pipes/WDL/tasks/tasks_interhost.wdl

@@ -1,4 +1,4 @@
-version 1.0
+version 1.1
 
 task subsample_by_cases {
     meta {

pipes/WDL/tasks/tasks_intrahost.wdl

@@ -1,4 +1,4 @@
-version 1.0
+version 1.1
 
 task polyphonia_detect_cross_contamination {
   input {

pipes/WDL/tasks/tasks_megablast.wdl

@@ -1,4 +1,4 @@
-version 1.0
+version 1.1
 
 task trim_rmdup_subsamp {
     meta {

pipes/WDL/tasks/tasks_metagenomics.wdl

@@ -1,4 +1,4 @@
-version 1.0
+version 1.1
 
 task krakenuniq {
   meta {