Merge pull request ASAP-CRN#46 from ASAP-CRN/chore/replace-gsutils-with-gcloud

Replace gsutil with gcloud storage

Author: kfang4

docker/util/scripts/upload_outputs

@@ -68,9 +68,8 @@ for output_file in "${OUTPUT_FILES[@]}"; do
 	echo "${output_file}.meta.tsv" >> output_files.txt
 done
 
-gsutil \
-	-u "${BILLING_PROJECT}" \
-	-m cp \
+gcloud storage cp \
+	--billing-project="${BILLING_PROJECT}" \
 	-I \
 	"${DESTINATION_PATH}" \
 < output_files.txt

util/transfer_raw_data

@@ -43,16 +43,16 @@ gsync() {
   source_path="${1#/}/"
   destination_path="${2#/}/"
 
-  gsutil -m rsync \
-    -d \
+  gcloud storage rsync \
+    --delete-unmatched-destination-objects \
     -r \
     ${DRY_RUN_ARG} \
     "${source_path}" \
     "${destination_path}"
 }
 
 # Default to dry run if promotion is not selected
-DRY_RUN_ARG="-n"
+DRY_RUN_ARG="--dry-run"
 FOLDER_TYPE=""
 PREFIX=""
 DELETE=""
@@ -146,7 +146,7 @@ for team in "${TEAMS[@]}"; do
     response=$(echo "$response" | tr '[:upper:]' '[:lower:]')
     if [[ "$response" == "yes" ]]; then
       log "Starting deletion..."
-      gsutil -m rm -r "$raw_bucket"
+      gcloud storage rm -r "$raw_bucket"
     elif [[ "$response" == "no" ]]; then
       log "Deletion canceled"
     else

wdl/tasks/upload_final_outputs.wdl

@@ -18,28 +18,27 @@ task upload_final_outputs {
 
 		echo -e "filename\tmd5_hash\ttimestamp\tworkflow\tworkflow_version\tworkflow_release" > MANIFEST.tsv
 		mkdir metadata
-		gsutil -u ~{billing_project} -m cp -I ./metadata/ \
+		gcloud storage cp --billing-project=~{billing_project} -I ./metadata/ \
 		< metadata_paths.txt
 
 		find metadata -type f -exec cat {} \; \
 		>> MANIFEST.tsv
 
 		while read -r staging_data_bucket || [[ -n "${staging_data_bucket}" ]]; do
 			# Remove files currently existing at the target path, if they exist
-			if gsutil -u ~{billing_project} ls "${staging_data_bucket}/~{staging_data_path}/**"; then
-				gsutil -u ~{billing_project} \
-					-m rm \
+			if gcloud storage ls --billing-project=~{billing_project} "${staging_data_bucket}/~{staging_data_path}/**"; then
+				gcloud storage rm --billing-project=~{billing_project} \
 					"${staging_data_bucket}/~{staging_data_path}/**"
 			fi
 
 			# Copy files to the staging data path
-			gsutil -u ~{billing_project} -m cp \
+			gcloud storage cp --billing-project=~{billing_project} \
 				-I \
 				"${staging_data_bucket}/~{staging_data_path}/" \
 			< ~{write_lines(output_file_paths)}
 
 			# Upload the manifest to the staging data path
-			gsutil -u ~{billing_project} -m cp \
+			gcloud storage cp --billing-project=~{billing_project} \
 				MANIFEST.tsv \
 				"${staging_data_bucket}/~{staging_data_path}/"