Add collapse_comments option and extract includedActivities fields in ecospold2 extractor

- New `collapse_comments` kwarg (default True) on `extract()` and `extract_activity()`;
  when False, `comment` is a dict keyed by source field instead of a flat string
- Always extract `included_activities_start` and `included_activities_end` as top-level
  dataset fields from activityDescription/activity
- Fix pre-existing double-space in collapsed comment separator (": " not ":  ")
- Update existing tests for new fields and corrected spacing; add test for dict mode

Author: cmutel

bw2io/extractors/ecospold2.py

@@ -113,6 +113,7 @@ def extract(
         db_name: str,
         use_mp: bool = True,
         cache: bool = False,
+        collapse_comments: bool = True,
     ):
         """
         Extract data from all ecospold2 files in a directory.
@@ -128,6 +129,11 @@ def extract(
         cache : bool, optional
             Cache extracted datasets as `.json.gz` files alongside the source `.spold`
             files for faster re-imports (default is False).
+        collapse_comments : bool, optional
+            If True (default), combine all comment fields into a single string. If False,
+            return ``comment`` as a dict with keys ``general``, ``included activities
+            start``, ``included activities end``, ``geography``, ``technology``, and
+            ``time period`` (only non-empty keys are included).
 
         Returns
         -------
@@ -168,7 +174,7 @@ def extract(
                         pool.apply_async(
                             Ecospold2DataExtractor.extract_activity,
                             args=(dirpath, x, db_name),
-                            kwds={"cache": cache},
+                            kwds={"cache": cache, "collapse_comments": collapse_comments},
                             callback=lambda _: pb.update(1),
                         )
                         for x in filelist
@@ -177,7 +183,11 @@ def extract(
         else:
             data = []
             for filename in tqdm(filelist):
-                data.append(cls.extract_activity(dirpath, filename, db_name, cache=cache))
+                data.append(
+                    cls.extract_activity(
+                        dirpath, filename, db_name, cache=cache, collapse_comments=collapse_comments
+                    )
+                )
 
         return data
 
@@ -218,7 +228,7 @@ def condense_multiline_comment(cls, element):
             return ""
 
     @classmethod
-    def extract_activity(cls, dirpath, filename, db_name, cache: bool = False):
+    def extract_activity(cls, dirpath, filename, db_name, cache: bool = False, collapse_comments: bool = True):
         """
         Extract and return the data of an activity from an XML file with the given
         `filename` in the directory with the path `dirpath`.
@@ -268,44 +278,60 @@ def extract_activity(cls, dirpath, filename, db_name, cache: bool = False):
         else:
             stem = root.childActivityDataset
 
-        comments = [
-            cls.condense_multiline_comment(
-                getattr2(stem.activityDescription.activity, "generalComment")
-            ),
-            (
-                "Included activities start: ",
-                getattr2(stem.activityDescription.activity, "includedActivitiesStart"),
-            ),
-            (
-                "Included activities end: ",
-                getattr2(stem.activityDescription.activity, "includedActivitiesEnd"),
-            ),
-            (
-                "Geography: ",
-                cls.condense_multiline_comment(
-                    getattr2(stem.activityDescription.geography, "comment")
-                ),
-            ),
-            (
-                "Technology: ",
-                cls.condense_multiline_comment(
-                    getattr2(stem.activityDescription.technology, "comment")
-                ),
-            ),
-            (
-                "Time period: ",
-                cls.condense_multiline_comment(
-                    getattr2(stem.activityDescription.timePeriod, "comment")
-                ),
-            ),
-        ]
-        comment = "\n".join(
-            [
-                (" ".join(str(i) for i in x) if isinstance(x, tuple) else x)
-                for x in comments
-                if (x[1] if isinstance(x, tuple) else x)
-            ]
+        def _text(obj):
+            if isinstance(obj, dict):
+                return ""
+            try:
+                return obj.text or ""
+            except Exception:
+                return ""
+
+        general_comment = cls.condense_multiline_comment(
+            getattr2(stem.activityDescription.activity, "generalComment")
+        )
+        included_start = _text(
+            getattr2(stem.activityDescription.activity, "includedActivitiesStart")
+        )
+        included_end = _text(
+            getattr2(stem.activityDescription.activity, "includedActivitiesEnd")
+        )
+        geo_comment = cls.condense_multiline_comment(
+            getattr2(stem.activityDescription.geography, "comment")
         )
+        tech_comment = cls.condense_multiline_comment(
+            getattr2(stem.activityDescription.technology, "comment")
+        )
+        time_comment = cls.condense_multiline_comment(
+            getattr2(stem.activityDescription.timePeriod, "comment")
+        )
+
+        if collapse_comments:
+            parts = [general_comment]
+            if included_start:
+                parts.append("Included activities start: " + included_start)
+            if included_end:
+                parts.append("Included activities end: " + included_end)
+            if geo_comment:
+                parts.append("Geography: " + geo_comment)
+            if tech_comment:
+                parts.append("Technology: " + tech_comment)
+            if time_comment:
+                parts.append("Time period: " + time_comment)
+            comment = "\n".join(x for x in parts if x)
+        else:
+            comment = {}
+            if general_comment:
+                comment["general"] = general_comment
+            if included_start:
+                comment["included activities start"] = included_start
+            if included_end:
+                comment["included activities end"] = included_end
+            if geo_comment:
+                comment["geography"] = geo_comment
+            if tech_comment:
+                comment["technology"] = tech_comment
+            if time_comment:
+                comment["time period"] = time_comment
 
         classifications = [
             (el.classificationSystem.text, el.classificationValue.text)
@@ -315,6 +341,8 @@ def extract_activity(cls, dirpath, filename, db_name, cache: bool = False):
 
         data = {
             "comment": comment,
+            "included_activities_start": included_start,
+            "included_activities_end": included_end,
             "classifications": classifications,
             "activity type": ACTIVITY_TYPES[
                 int(stem.activityDescription.activity.get("specialActivityType") or 0)

tests/ecospold2/ecospold2_extractor.py

@@ -18,7 +18,9 @@ def test_extraction_without_synonyms():
         "ei",
     )
     expected = {
-        "comment": "Things and stuff and whatnot\na Kikki comment\nIncluded activities start:  Includes start stuff\nIncluded activities end:  Includes end stuff\nTechnology:  typical technology for ze Germans!",
+        "comment": "Things and stuff and whatnot\na Kikki comment\nIncluded activities start: Includes start stuff\nIncluded activities end: Includes end stuff\nTechnology: typical technology for ze Germans!",
+        "included_activities_start": "Includes start stuff",
+        "included_activities_end": "Includes end stuff",
         "classifications": [
             ("EcoSpold01Categories", "construction materials/concrete"),
             (
@@ -157,7 +159,9 @@ def test_extraction_with_synonyms():
         "ei",
     )
     expected = {
-        "comment": "Things and stuff and whatnot\na Kikki comment\nIncluded activities end:  Includes some stuff\nTechnology:  typical technology for ze Germans!",
+        "comment": "Things and stuff and whatnot\na Kikki comment\nIncluded activities end: Includes some stuff\nTechnology: typical technology for ze Germans!",
+        "included_activities_start": "",
+        "included_activities_end": "Includes some stuff",
         "classifications": [
             ("EcoSpold01Categories", "construction materials/concrete"),
             (
@@ -366,3 +370,19 @@ def test_cache_written_and_read(tmp_path):
 
     cached = Ecospold2DataExtractor.extract_activity(tmp_path, SPOLD, "ei", cache=True)
     assert cached == {"name": "from cache"}
+
+
+def test_collapse_comments_false():
+    data = Ecospold2DataExtractor.extract(
+        FIXTURES / SPOLD,
+        "ei",
+        collapse_comments=False,
+    )
+    comment = data[0]["comment"]
+    assert isinstance(comment, dict)
+    assert comment["general"] == "Things and stuff and whatnot\na Kikki comment"
+    assert comment["included activities start"] == "Includes start stuff"
+    assert comment["included activities end"] == "Includes end stuff"
+    assert comment["technology"] == "typical technology for ze Germans!"
+    assert "geography" not in comment
+    assert "time period" not in comment