Merge pull request #433 from ghga-de/develop

Prepare Release 0.9.8

Author: lkuchenb

datameta/api/download.py

@@ -45,24 +45,22 @@ def get_file_url(request: Request) -> HTTPTemporaryRedirect:
     )
 
     # retrieve URL:
-    url = storage.get_download_url(
+    url, expires_at = storage.get_download_url(
         request=request,
         db_file=db_file,
         expires_after=expires_after
     )
 
-    if redirect:
-        return HTTPTemporaryRedirect(url, json_body = {
+    response = {
             'fileId' : get_identifier(db_file),
             'fileUrl' : f"{request.host_url}{url}",
-            'expires' : expires_after
-            })
+            'expires' : expires_at.isoformat() + "+00:00"
+            }
+
+    if redirect:
+        return HTTPTemporaryRedirect(url, json_body=response)
     else:
-        return HTTPOk(json_body = {
-            'fileId' : get_identifier(db_file),
-            'fileUrl' : f"{request.host_url}{url}",
-            'expires' : expires_after
-            })
+        return HTTPOk(json_body=response)
 
 
 @view_config(
@@ -83,7 +81,7 @@ def download_by_token(request: Request) -> HTTPOk:
     db_token = db.query(models.DownloadToken).filter(
         and_(
             models.DownloadToken.value == hashed_token,
-            models.DownloadToken.expires > datetime.now()
+            models.DownloadToken.expires > datetime.utcnow()
         )
     ).one_or_none()
 

datameta/api/metadatasets.py

@@ -378,14 +378,14 @@ def set_metadata_via_service(request: Request) -> MetaDataSetResponse:
             .options(joinedload(Service.users))\
             .one_or_none()
 
-    service_metadata = { mdatum.name : mdatum for mdatum in service.target_metadata }
-
-    service_records = { rec.metadatum.name : rec for rec in metadataset.metadatumrecords if rec.metadatum.name in service_metadata }
-
     # Return 404 if a resource could not be found
     if metadataset is None or service is None:
         raise HTTPNotFound()
 
+    service_metadata = { mdatum.name : mdatum for mdatum in service.target_metadata }
+
+    service_records = { rec.metadatum.name : rec for rec in metadataset.metadatumrecords if rec.metadatum.name in service_metadata }
+
     # Return 403 if the user has no permission to execute this service or the
     # service has already been executed for this metadataset
     if service.id in (sexec.service_id for sexec in metadataset.service_executions):
@@ -413,9 +413,6 @@ def set_metadata_via_service(request: Request) -> MetaDataSetResponse:
     # Validate submission access to the specified files
     validation.validate_submission_access(db, db_files, {}, auth_user)
 
-    # Validate the associations between files and records
-    fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { metadataset.site_id : metadataset })
-
     # Validate the provided records
     validate_metadataset_record(service_metadata, records, return_err_message=False, rendered=False)
 
@@ -426,9 +423,21 @@ def set_metadata_via_service(request: Request) -> MetaDataSetResponse:
         db_rec.value = record_value
         db.add(db_rec)
 
+    # Validate the associations between files and records
+    fnames, ref_fnames, val_errors = validation.validate_submission_association(db_files, { metadataset.site_id : metadataset }, ignore_submitted_metadatasets=True)
+
+    # If there were any validation errors, return 400
+    if val_errors:
+        entities, fields, messages = zip(*val_errors)
+        raise errors.get_validation_error(messages=messages, fields=fields, entities=entities)
+
+    # Given that validation hasn't failed, we know that file names are unique. Flatten the dict.
+    fnames = { k : v[0] for k, v in fnames.items() }
+
     # Associate the files with the metadata
     for fname, mdatrec in ref_fnames.items():
         mdatrec.file = fnames[fname]
+        db.add(mdatrec)
 
     # Create a service execution
     sexec = ServiceExecution(

datameta/api/openapi.yaml

@@ -15,7 +15,7 @@
 openapi: 3.0.0
 info:
   description: DataMeta
-  version: 0.26.0
+  version: 0.27.0
   title: DataMeta
 
 servers:
@@ -1276,7 +1276,8 @@ components:
         fileUrl:
           type: string
         expires:
-          type: integer
+          type: string
+          format: date-time
       required:
         - fileId
         - fileUrl

datameta/storage.py

@@ -164,7 +164,7 @@ def _get_download_url_local(request: Request, db_file: models.File, expires_afte
 
     token = security.generate_token()
     token_hash = security.hash_token(token)
-    expires = datetime.now() + timedelta(minutes = float(expires_after))
+    expires = datetime.utcnow() + timedelta(minutes = float(expires_after))
 
     db = request.dbsession
     download_token = models.DownloadToken(
@@ -174,7 +174,7 @@ def _get_download_url_local(request: Request, db_file: models.File, expires_afte
     )
     db.add(download_token)
 
-    return f"{base_url}/download/{token}"
+    return f"{base_url}/download/{token}", expires
 
 
 def _get_download_url_s3(request: Request, db_file: models.File, expires_after: Optional[int] = None):

datameta/validation.py

@@ -64,7 +64,7 @@ def validate_submission_access(db, db_files, db_msets, auth_user):
         raise errors.get_validation_error(messages=messages, fields=fields, entities=entities)
 
 
-def validate_submission_association(db_files, db_msets):
+def validate_submission_association(db_files, db_msets, ignore_submitted_metadatasets = False):
     """Validates a submission with regard to
 
     - All submitted files being associated to metadata
@@ -82,7 +82,8 @@ def validate_submission_association(db_files, db_msets):
     # Collect files which already have other metadata associated
     errors += [ (db_file, None, "Already submitted") for file_id, db_file in db_files.items() if db_file.metadatumrecord is not None ]
     # Collect metadatasets that were already submitted
-    errors += [ (db_mset, None, "Already submitted") for mset_id, db_mset in db_msets.items() if db_mset.submission_id is not None ]
+    if not ignore_submitted_metadatasets:
+        errors += [ (db_mset, None, "Already submitted") for mset_id, db_mset in db_msets.items() if db_mset.submission_id is not None ]
 
     # Collect the file names of the provided files
     f_names_obj = defaultdict(list)

setup.py

@@ -60,7 +60,7 @@
 
 setup(
     name                   = 'datameta',
-    version                = '0.9.7',
+    version                = '0.9.8',
     description            = 'DataMeta - submission server for data and associated metadata',
     long_description       = README + '\n\n' + CHANGES,
     author                 = 'Leon Kuchenbecker',