Release v1.15.0

.flake8

@@ -3,5 +3,7 @@ exclude =
     .git,
     __pycache__,
     build,
+    venv,
+    .venv
     env
 max-complexity = 10

docs/examples/example1.csv

@@ -1,4 +1,4 @@
 project_name, modality0.capsule_id, modality0, modality0.input_source, modality1, modality1.input_source, s3_bucket, subject_id, platform, acq_datetime, job_type
 Ephys Platform, , ecephys, dir/data_set_1, ,, default, 123454, ecephys, 2020-10-10 14:10:10, ecephys
-Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, behavior, 10/13/2020 1:10:10 PM,
-Behavior Platform, , behavior-videos, dir/data_set_2, behavior, dir/data_set_3, private, 123456, behavior, 10/13/2020 1:10:10 PM,
+Behavior Platform, 1f999652-00a0-4c4b-99b5-64c2985ad070, behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, behavior, 10/13/2020 1:10:10 PM, default
+Behavior Platform, , behavior-videos, dir/data_set_2, behavior, dir/data_set_3, private, 123456, behavior, 10/13/2020 1:10:10 PM, default

docs/examples/remove_source_folders.py

@@ -0,0 +1,80 @@
+"""
+WARNING: This deletes raw data. Please use caution.
+This example demonstrates how to remove the source folders after the data
+has been uploaded to S3.
+"""
+
+from datetime import datetime
+
+import requests
+from aind_data_schema_models.modalities import Modality
+from aind_data_schema_models.platforms import Platform
+
+from aind_data_transfer_service.models.core import (
+    SubmitJobRequestV2,
+    Task,
+    UploadJobConfigsV2,
+)
+
+# The job_type contains the default settings for compression and Code Ocean
+# pipelines.
+job_type = "ecephys"
+
+acq_datetime = datetime(2023, 4, 3, 18, 17, 7)
+
+remove_source_folders = Task(skip_task=False)
+
+ecephys_task = Task(
+    job_settings={
+        "input_source": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "ecephys/655019_2023-04-03_18-17-07"
+        )
+    }
+)
+
+modality_transformation_settings = {"ecephys": ecephys_task}
+
+gather_preliminary_metadata = Task(
+    job_settings={
+        "metadata_dir": (
+            "/allen/aind/scratch/svc_aind_upload/test_data_sets/"
+            "ecephys/655019_2023-04-03_18-17-07"
+        )
+    }
+)
+
+
+upload_job_configs_v2 = UploadJobConfigsV2(
+    job_type=job_type,
+    project_name="Ephys Platform",
+    platform=Platform.ECEPHYS,
+    modalities=[Modality.ECEPHYS],
+    subject_id="655019",
+    acq_datetime=acq_datetime,
+    tasks={
+        "modality_transformation_settings": modality_transformation_settings,
+        "gather_preliminary_metadata": gather_preliminary_metadata,
+        "remove_source_folders": remove_source_folders
+    },
+)
+
+submit_request_v2 = SubmitJobRequestV2(
+    upload_jobs=[upload_job_configs_v2],
+)
+
+post_request_content = submit_request_v2.model_dump(
+    mode="json", exclude_none=True
+)
+
+# Please use the production endpoint for submitting jobs and the dev endpoint
+# for running tests.
+# endpoint = "http://aind-data-transfer-service"
+endpoint = "http://aind-data-transfer-service-dev"  # For testing
+
+submit_job_response = requests.post(
+    url=f"{endpoint}/api/v2/submit_jobs",
+    json=post_request_content,
+)
+print(submit_job_response.status_code)
+print(submit_job_response.json())

docs/source/Contributing.rst

@@ -71,6 +71,7 @@ To run uvicorn locally:
    export ENV_NAME='local'
    export AWS_DEFAULT_REGION='us-west-2'
    export AIND_AIRFLOW_PARAM_PREFIX='/aind/dev/airflow/variables/job_types'
+   export AIND_SSO_SECRET_NAME='/aind/dev/data_transfer_service/sso/secrets'
    uvicorn aind_data_transfer_service.server:app --host 0.0.0.0 --port 5000 --reload
 
 You can now access aind-data-transfer-service at

pyproject.toml

@@ -53,7 +53,8 @@ server = [
     'wtforms',
     'requests==2.25.0',
     'openpyxl',
-    'python-logging-loki'
+    'python-logging-loki',
+    'authlib'
 ]
 
 [tool.setuptools.packages.find]

src/aind_data_transfer_service/__init__.py

@@ -1,7 +1,7 @@
 """Init package"""
 import os
 
-__version__ = "1.14.0"
+__version__ = "1.15.0"
 
 # Global constants
 OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open")

src/aind_data_transfer_service/configs/job_upload_template.py

@@ -2,24 +2,24 @@
 
 import datetime
 from io import BytesIO
-from typing import Any, Dict, List
+from typing import Any, ClassVar, Dict, List
 
 from aind_data_schema_models.modalities import Modality
 from aind_data_schema_models.platforms import Platform
 from openpyxl import Workbook
 from openpyxl.styles import Font
 from openpyxl.utils import get_column_letter
 from openpyxl.worksheet.datavalidation import DataValidation
+from pydantic import BaseModel
 
 
-# TODO: convert to pydantic model
-class JobUploadTemplate:
+class JobUploadTemplate(BaseModel):
     """Class to configure and create xlsx job upload template"""
 
-    FILE_NAME = "job_upload_template.xlsx"
-    NUM_TEMPLATE_ROWS = 20
-    XLSX_DATETIME_FORMAT = "YYYY-MM-DDTHH:mm:ss"
-    HEADERS = [
+    FILE_NAME: ClassVar[str] = "job_upload_template.xlsx"
+    _NUM_TEMPLATE_ROWS: ClassVar[int] = 20
+    _XLSX_DATETIME_FORMAT: ClassVar[str] = "YYYY-MM-DDTHH:mm:ss"
+    _HEADERS: ClassVar[List[str]] = [
         "job_type",
         "project_name",
         "platform",
@@ -31,7 +31,7 @@ class JobUploadTemplate:
         "modality1",
         "modality1.input_source",
     ]
-    SAMPLE_JOBS = [
+    _SAMPLE_JOBS: ClassVar[List[List[Any]]] = [
         [
             "default",
             "Behavior Platform",
@@ -68,8 +68,8 @@ class JobUploadTemplate:
         ],
     ]
 
-    @property
-    def validators(self) -> List[Dict[str, Any]]:
+    @classmethod
+    def _get_validators(cls) -> List[Dict[str, Any]]:
         """
         Returns
         -------
@@ -82,36 +82,36 @@ def validators(self) -> List[Dict[str, Any]]:
                 "name": "platform",
                 "type": "list",
                 "options": list(Platform.abbreviation_map.keys()),
-                "column_indexes": [self.HEADERS.index("platform")],
+                "column_indexes": [cls._HEADERS.index("platform")],
             },
             {
                 "name": "modality",
                 "type": "list",
                 "options": list(Modality.abbreviation_map.keys()),
                 "column_indexes": [
-                    self.HEADERS.index("modality0"),
-                    self.HEADERS.index("modality1"),
+                    cls._HEADERS.index("modality0"),
+                    cls._HEADERS.index("modality1"),
                 ],
             },
             {
                 "name": "datetime",
                 "type": "date",
-                "column_indexes": [self.HEADERS.index("acq_datetime")],
+                "column_indexes": [cls._HEADERS.index("acq_datetime")],
             },
         ]
 
-    @property
-    def excel_sheet_filestream(self) -> BytesIO:
+    @classmethod
+    def create_excel_sheet_filestream(cls) -> BytesIO:
         """Create job template as xlsx filestream"""
         xl_io = BytesIO()
         workbook = Workbook()
         workbook.iso_dates = True
         worksheet = workbook.active
-        worksheet.append(self.HEADERS)
-        for job in self.SAMPLE_JOBS:
+        worksheet.append(cls._HEADERS)
+        for job in cls._SAMPLE_JOBS:
             worksheet.append(job)
         # data validators
-        for validator in self.validators:
+        for validator in cls._get_validators():
             dv_type = validator["type"]
             dv_name = validator["name"]
             dv_params = {
@@ -127,17 +127,17 @@ def excel_sheet_filestream(self) -> BytesIO:
                 dv_params["prompt"] = f"Select a {dv_name} from the dropdown"
             elif dv_type == "date":
                 dv_params["prompt"] = "Provide a {} using {}".format(
-                    dv_name, self.XLSX_DATETIME_FORMAT
+                    dv_name, cls._XLSX_DATETIME_FORMAT
                 )
             dv = DataValidation(**dv_params)
             for i in validator["column_indexes"]:
                 col = get_column_letter(i + 1)
-                col_range = f"{col}2:{col}{self.NUM_TEMPLATE_ROWS}"
+                col_range = f"{col}2:{col}{cls._NUM_TEMPLATE_ROWS}"
                 dv.add(col_range)
                 if dv_type != "date":
                     continue
                 for (cell,) in worksheet[col_range]:
-                    cell.number_format = self.XLSX_DATETIME_FORMAT
+                    cell.number_format = cls._XLSX_DATETIME_FORMAT
             worksheet.add_data_validation(dv)
         # formatting
         bold = Font(bold=True)

src/aind_data_transfer_service/models/core.py

@@ -290,11 +290,15 @@ def check_duplicate_upload_jobs(self, info: ValidationInfo):
         # check against any jobs in the context
         current_jobs = (info.context or dict()).get("current_jobs", list())
         for job in current_jobs:
-            prefix = job.get("s3_prefix")
-            if (
-                prefix is not None
-                and prefix in jobs_map
-                and json.dumps(job, sort_keys=True) in jobs_map[prefix]
-            ):
-                raise ValueError(f"Job is already running/queued for {prefix}")
+            jobs_to_check = job.get("upload_jobs", [job])
+            for j in jobs_to_check:
+                prefix = j.get("s3_prefix")
+                if (
+                    prefix is not None
+                    and prefix in jobs_map
+                    and json.dumps(j, sort_keys=True) in jobs_map[prefix]
+                ):
+                    raise ValueError(
+                        f"Job is already running/queued for {prefix}"
+                    )
         return self