Merge remote-tracking branch 'upstream/main'

Author: github-actions[bot]

.gitignore

@@ -1,2 +1,5 @@
 # Mac/OSX system file
 .DS_Store
+
+# Python
+**/__pycache__/

util/common.py

@@ -353,14 +353,16 @@ def compare_md5_hashes(results, staging, same_files):
 ##############################################################
 ##### PROMOTE WORKFLOW OUTPUTS - STAGING TO PROD SECTION #####
 ##############################################################
-def gcopy(source_path, destination_path):
+def gcopy(source_path, destination_path, recursive=False):
 	command = [
 		"gcloud",
 		"storage",
 		"cp",
 		source_path,
 		destination_path
 	]
+	if recursive:
+		command.insert(3, "--recursive")
 	result = subprocess.run(command, check=True, capture_output=True, text=True)
 	logging.info(result.stdout)
 	logging.error(result.stderr)

util/promote_staging_data

@@ -155,6 +155,7 @@ def main(args):
 
 			production_workflow_bucket = f"gs://asap-curated-{team}-{args.source}-{dataset}/{args.workflow_name}"
 			production_workflow_version_bucket = f"gs://asap-curated-{team}-{args.source}-{dataset}/{args.workflow_name}/archive/workflow_version/{args.workflow_version}"
+			production_workflow_metadata_path = f"{production_workflow_version_bucket}/workflow_metadata/{formatted_time}"
 
 			dev_workflow_metadata_path = f"{staging_dev_bucket}/{args.workflow_name}/archive/workflow_version/{args.workflow_version}/workflow_metadata/{formatted_time}"
 			uat_workflow_metadata_path = f"{staging_uat_bucket}/{args.workflow_name}/archive/workflow_version/{args.workflow_version}/workflow_metadata/{formatted_time}"
@@ -200,6 +201,12 @@ def main(args):
 			logging.info(f"\tProduction archive/workflow_version bucket path:\t[{production_workflow_version_bucket}]")
 			# Dry and real run will only work if staging data gets promoted to production in previous step
 			gsync_del(production_workflow_bucket, production_workflow_version_bucket, args.workflow_name, dry_run)
+
+			if dry_run:
+				logging.info(f"Would copy {uat_workflow_metadata_path} to {production_workflow_metadata_path}")
+			else:
+				# Promote combined manifest and data promotion report from staging to production
+				gcopy(uat_workflow_metadata_path, production_workflow_metadata_path, recursive=True)
 		else:
 			logging.error(f"Data cannot be promoted for [{team}-{args.source}-{dataset}]; exiting")
 			sys.exit(1)