Merge branch 'master' into improved-multilangual

Author: hcvdwerf

.github/workflows/test.yml

@@ -68,12 +68,13 @@ jobs:
         pip install -r ckanext-harvest/requirements.txt
         git clone https://github.com/ckan/ckanext-scheming
         pip install -e ckanext-scheming
-        pip install git+https://github.com/ckan/ckanext-fluent.git@4e9340a#egg=ckanext-fluent
+        git clone https://github.com/ckan/ckanext-fluent
+        pip install -e ckanext-fluent
         git clone https://github.com/ckan/ckanext-dataset-series
         pip install -e ckanext-dataset-series
     - name: Setup extension
       run: |
         ckan -c test.ini db init
         ckan -c test.ini db pending-migrations --apply
     - name: Run tests
-      run: pytest --ckan-ini=test.ini --cov=ckanext.dcat --cov-report=term-missing --cov-append --disable-warnings ckanext/dcat/tests
+      run: pytest --ckan-ini=test.ini --cov=ckanext.dcat --cov-report=term-missing --cov-append --disable-warnings ckanext/dcat/tests

ckanext/dcat/harvesters/rdf.py

@@ -210,18 +210,39 @@ def gather_stage(self, harvest_job):
                 return []
 
             try:
-                source_dataset = model.Package.get(harvest_job.source.id)                
-                
-                series_ids, series_mapping = self._parse_and_collect(
-                    parser.dataset_series(),
-                    source_dataset,
-                    harvest_job,
-                    guids_in_source,
-                    is_series=True,
-                    collect_series_mapping=True
-                )
-                object_ids += series_ids
-                object_ids += self._parse_and_collect(parser.datasets(series_mapping), source_dataset, harvest_job, guids_in_source, is_series=False)
+
+                source_dataset = model.Package.get(harvest_job.source.id)
+
+                for dataset in parser.datasets():
+                    if not dataset.get('name'):
+                        dataset['name'] = self._gen_new_name(dataset['title'])
+                    if dataset['name'] in self._names_taken:
+                        suffix = len([i for i in self._names_taken if i.startswith(dataset['name'] + '-')]) + 1
+                        dataset['name'] = '{}-{}'.format(dataset['name'], suffix)
+                    self._names_taken.append(dataset['name'])
+
+                    # Unless already set by the parser, get the owner organization (if any)
+                    # from the harvest source dataset
+                    if not dataset.get('owner_org'):
+                        if source_dataset.owner_org:
+                            dataset['owner_org'] = source_dataset.owner_org
+
+                    # Try to get a unique identifier for the harvested dataset
+                    guid = self._get_guid(dataset, source_url=source_dataset.url)
+
+                    if not guid:
+                        self._save_gather_error('Could not get a unique identifier for dataset: {0}'.format(dataset),
+                                                harvest_job)
+                        continue
+
+                    dataset['extras'].append({'key': 'guid', 'value': guid})
+                    guids_in_source.append(guid)
+
+                    obj = HarvestObject(guid=guid, job=harvest_job,
+                                        content=json.dumps(dataset))
+
+                    obj.save()
+                    object_ids.append(obj.id)
             except Exception as e:
                 self._save_gather_error('Error when processsing dataset: %r / %s' % (e, traceback.format_exc()),
                                         harvest_job)
@@ -401,70 +422,3 @@ def import_stage(self, harvest_object):
             model.Session.commit()
 
         return True
-        
-    def _parse_and_collect(
-        self,
-        items,
-        source_dataset,
-        harvest_job,
-        guids_in_source,
-        is_series=False,
-        collect_series_mapping=False
-    ):
-        object_ids = []
-        label = "dataset series" if is_series else "dataset"
-        series_mapping = {} if collect_series_mapping else None
-
-        for item in items:
-            original_title = item.get("title", label)
-            if not item.get("name"):
-                item["name"] = self._gen_new_name(original_title)
-
-            if item["name"] in self._names_taken:
-                suffix = len([i for i in self._names_taken if i.startswith(item["name"] + "-")]) + 1
-                item["name"] = f"{item['name']}-{suffix}"
-
-            self._names_taken.append(item["name"])
-
-            if not item.get("owner_org") and source_dataset.owner_org:
-                item["owner_org"] = source_dataset.owner_org
-
-            guid = self._get_guid(item, source_url=source_dataset.url)
-            if not guid:
-                self._save_gather_error(f"Could not get a unique identifier for {label}: {item}", harvest_job)
-                continue
-
-            item.setdefault("extras", []).append({"key": "guid", "value": guid})
-            guids_in_source.append(guid)
-
-            obj = HarvestObject(guid=guid, job=harvest_job, content=json.dumps(item))
-            obj.save()
-            object_ids.append(obj.id)
-
-            # Store mapping of RDF URI to dataset name if requested
-            if collect_series_mapping:
-                series_uri = item.get("uri") or item.get("identifier")
-                if series_uri:
-                    # Try to find an existing active dataset series by 'guid' match
-                    existing = model.Session.query(model.Package).\
-                        join(model.PackageExtra).\
-                        filter(model.PackageExtra.key == 'guid').\
-                        filter(model.PackageExtra.value == series_uri).\
-                        filter(model.Package.type == 'dataset_series').\
-                        filter(model.Package.state == 'active').\
-                        first()
-
-                    if existing:
-                        item["name"] = existing.name
-
-                    series_mapping[str(series_uri)] = {
-                        "id": existing.id if existing else item.get("id"),
-                        "name": item["name"]
-                    }
-
-
-        if collect_series_mapping:
-            return object_ids, series_mapping
-
-        return object_ids
-

ckanext/dcat/helpers.py

@@ -72,7 +72,7 @@ def structured_data(dataset_dict, profiles=None):
     return _get_serialization(dataset_dict, profiles, "jsonld")
 
 
-def croissant(dataset_dict, profiles=None):
+def croissant(dataset_dict, profiles=None, jsonld_context=None):
     """
     Returns a string containing the Croissant ML representation of the given
     dataset using the `croissant` profile.
@@ -82,8 +82,10 @@ def croissant(dataset_dict, profiles=None):
     if not profiles:
         profiles = config.get("ckanext.dcat.croissant.profiles", ["croissant"])
 
-    frame = {"@context": JSONLD_CONTEXT, "@type": "sc:Dataset"}
+    context = jsonld_context or JSONLD_CONTEXT
+
+    frame = {"@context": context, "@type": "sc:Dataset"}
 
     return _get_serialization(
-        dataset_dict, profiles, "jsonld", context=JSONLD_CONTEXT, frame=frame
+        dataset_dict, profiles, "jsonld", context=context, frame=frame
     )

ckanext/dcat/processors.py

@@ -119,16 +119,6 @@ def _datasets(self):
         for dataset in self.g.subjects(RDF.type, DCAT.Dataset):
             yield dataset
 
-    def _dataset_series(self):
-        '''
-        Generator that returns all DCAT dataset series on the graph
-
-        Yields rdflib.term.URIRef objects that can be used on graph lookups
-        and queries
-        '''
-        for dataset_series in self.g.subjects(RDF.type, DCAT.DatasetSeries):
-            yield dataset_series
-
     def next_page(self):
         '''
         Returns the URL of the next page or None if there is no next page
@@ -183,7 +173,7 @@ def supported_formats(self):
                        for plugin
                        in rdflib.plugin.plugins(kind=rdflib.parser.Parser)])
 
-    def datasets(self, series_mapping=None):
+    def datasets(self):
         '''
         Generator that returns CKAN datasets parsed from the RDF graph
 
@@ -203,39 +193,6 @@ def datasets(self, series_mapping=None):
                 )
                 profile.parse_dataset(dataset_dict, dataset_ref)
 
-            # Add in_series if present in RDF and mapped
-            in_series = []
-            for series_ref in self.g.objects(dataset_ref, DCAT.inSeries):
-                key = str(series_ref)
-                if series_mapping and key in series_mapping:
-                    in_series.append(series_mapping[key]["id"])
-
-            if in_series:
-                dataset_dict["in_series"] = in_series
-
-            yield dataset_dict
-
-               
-    def dataset_series(self):
-        '''
-        Generator that returns CKAN dataset series parsed from the RDF graph
-
-        Each dataset series is passed to all the loaded profiles before being
-        yielded, so it can be further modified by each one of them.
-
-        Returns a dataset series dict that can be passed to eg `package_create`
-        or `package_update`
-        '''
-        for dataset_ref in self._dataset_series():
-            dataset_dict = {}
-            for profile_class in self._profiles:
-                profile = profile_class(
-                    self.g,
-                    dataset_type=self.dataset_type,
-                    compatibility_mode=self.compatibility_mode
-                )
-                profile.parse_dataset(dataset_dict, dataset_ref)
-
             yield dataset_dict
 
 

ckanext/dcat/profiles/croissant.py

@@ -24,6 +24,7 @@
 
 JSONLD_CONTEXT = {
     "@vocab": "https://schema.org/",
+    "@language": config.get("ckan.locale_default"),
     "sc": "https://schema.org/",
     "cr": "http://mlcommons.org/croissant/",
     "rai": "http://mlcommons.org/croissant/RAI/",

ckanext/dcat/profiles/euro_dcat_ap_3.py

@@ -30,17 +30,6 @@ def parse_dataset(self, dataset_dict, dataset_ref):
         # DCAT AP v2 scheming fields
         dataset_dict = self._parse_dataset_v2_scheming(dataset_dict, dataset_ref)
 
-
-        # Check if it's a dataset series
-        if (dataset_ref, RDF.type, DCAT.DatasetSeries) in self.g:
-            dataset_dict["type"] = "dataset_series"
-
-            # Example defaulting logic (adjust based on RDF vocab if you have it)
-            if "series_order_field" not in dataset_dict:
-                dataset_dict["series_order_field"] = "metadata_created"
-            if "series_order_type" not in dataset_dict:
-                dataset_dict["series_order_type"] = "date"
-
         # DCAT AP v3: hasVersion
         values = self._object_value_list(dataset_ref, DCAT.hasVersion)
         if values:

ckanext/dcat/profiles/euro_health_dcat_ap.py

@@ -183,6 +183,7 @@ def _parse_retention_period(self, subject_ref):
 
         return [retention_dict] if retention_dict else []
 
+
     def graph_from_dataset(self, dataset_dict, dataset_ref):
         super().graph_from_dataset(dataset_dict, dataset_ref)
         for prefix, namespace in namespaces.items():

ckanext/dcat/schemas/health_dcat_ap.yaml

@@ -733,8 +733,6 @@ resource_fields:
 
     - field_name: rights
       label: Rights
-      form_snippet: markdown.html
-      display_snippet: markdown.html
       preset: multiple_text
       validators: ignore_missing scheming_multiple_text
 

ckanext/dcat/tests/test_blueprints.py

@@ -66,6 +66,28 @@ def test_dataset_default(self, app):
         assert dcat_dataset['title'] == dataset['title']
         assert dcat_dataset['notes'] == dataset['notes']
 
+    def test_dataset_default_private(self, app):
+        user = factories.UserWithToken()
+        org = factories.Organization(users=[{"name": user["name"], "capacity": "admin"}])
+        dataset = factories.Dataset(
+            notes='Test dataset',
+            owner_org=org['id'],
+            private=True
+        )
+
+        url = url_for('dcat.read_dataset', _id=dataset['name'], _format='rdf')
+
+
+        # Unauthenticated request
+        response = app.get(url)
+        assert response.status_code == 403
+
+        # Authenticated request
+        headers = {"Authorization": user["token"]}
+        response = app.get(url, headers=headers)
+
+        assert response.headers['Content-Type'] == 'application/rdf+xml'
+
     def test_dataset_xml(self, app):
 
         dataset = factories.Dataset(
@@ -612,8 +634,8 @@ def test_croissant_metadata_embedded(self, app):
         response = app.get(url)
 
         assert '<script type="application/ld+json">' in response.body
-        assert '"description": "test description"' in response.body
-        assert '"conformsTo": "http://mlcommons.org/croissant/1.0"' in response.body
+        assert '"@value": "test description"' in response.body
+        assert '"@value": "http://mlcommons.org/croissant/1.0"' in response.body
 
     @pytest.mark.ckan_config('ckan.plugins', 'dcat croissant')
     def test_croissant_metadata_endpoint(self, app):
@@ -627,8 +649,8 @@ def test_croissant_metadata_endpoint(self, app):
         response = app.get(url)
         croissant_dict = json.loads(response.body)
 
-        assert croissant_dict["description"] == "test description"
-        assert croissant_dict["conformsTo"] == "http://mlcommons.org/croissant/1.0"
+        assert croissant_dict["description"] == {"@value": "test description"}
+        assert croissant_dict["conformsTo"] == {"@value": "http://mlcommons.org/croissant/1.0"}
 
 
 @pytest.mark.usefixtures("with_plugins", "clean_db", "clean_index")