Merge pull request #25 from BramVanroy/24-wrong-column-names-in-output

improve CoNLL-U fields

Author: Bram Vanroy

HISTORY.md

@@ -1,5 +1,16 @@
 # History
 
+## 3.4.0 (April 2nd, 2023)
+
+User @matgrioni rightfully pointed out that the default fields in the library are not the real CoNLL-U fields. They
+should be in all caps, and for xpostag it should be XPOS and for upostag UPOS. Additionally, the user asked for more
+control over these fields. This release accommodates that request.
+
+- **[formatter]** Breaking change: `CONLL_FIELD_NAMES` now is in line with the CoNLL-U descriptor field names: 
+"ID", "FORM", "LEMMA", "UPOS", "XPOS", "FEATS", "HEAD", "DEPREL", "DEPS", "MISC". These are the new default fields.
+- **[formatter]** New parameter: `field_names` that allows you to override the fields that were described above. 
+Simply add a dictionary of `{"default field name": "new field name"}`, e.g. `{"UPOS": "upostag"}
+
 ## 3.3.0 (January 17th, 2023)
 
 Since spaCy 3.2.0, the data that is passed to a spaCy pipeline has become more strict. This means that passing 

README.md

@@ -131,6 +131,9 @@ To illustrate, here is an advanced example, showing the more complex options:
 -  `conversion_maps`: a two-level dictionary that looks like `{field_name: {tag_name: replacement}}`. In 
    other words, you can specify in which field a certain value should be replaced by another. This is especially useful
    when you are not satisfied with the tagset of a model and wish to change some tags to an alternative0. 
+- `field_names`: allows you to change the default CoNLL-U field names to your own custom names. Similar to the 
+   conversion map above, you should use any of the default field names as keys and add your own key as value. 
+   Possible keys are : "ID", "FORM", "LEMMA", "UPOS", "XPOS", "FEATS", "HEAD", "DEPREL", "DEPS", "MISC".
 
 The example below
 
@@ -168,13 +171,23 @@ The snippets above will output a pandas DataFrame by using `._.pandas` rather th
 `._.conll_pd`, and all occurrences of `nsubj` in the deprel field are replaced by `subj`.
 
 ```
-   id     form   lemma upostag xpostag                                       feats  head deprel deps           misc
+   ID     FORM   LEMMA    UPOS    XPOS                                       FEATS  HEAD DEPREL DEPS           MISC
 0   1        I       I    PRON     PRP  Case=Nom|Number=Sing|Person=1|PronType=Prs     2   subj    _              _
 1   2     like    like    VERB     VBP                     Tense=Pres|VerbForm=Fin     0   ROOT    _              _
 2   3  cookies  cookie    NOUN     NNS                                 Number=Plur     2   dobj    _  SpaceAfter=No
 3   4        .       .   PUNCT       .                              PunctType=Peri     2  punct    _  SpaceAfter=No
 ```
 
+Another initialization example that would replace the column names "UPOS" with "upostag" amd "XPOS" with "xpostag":
+
+```python
+import spacy
+
+
+nlp = spacy.load("en_core_web_sm")
+config = {"field_names": {"UPOS": "upostag", "XPOS": "xpostag"}}
+nlp.add_pipe("conll_formatter", config=config, last=True)
+```
 
 #### Reading CoNLL into a spaCy object
 

spacy_conll/formatter.py

@@ -6,39 +6,46 @@
 from spacy.tokens import Doc, Span, Token
 from spacy_conll.utils import PD_AVAILABLE, merge_dicts_strict
 
-
 if PD_AVAILABLE:
     import pandas as pd
 
 CONLL_FIELD_NAMES = [
-    "id",
-    "form",
-    "lemma",
-    "upostag",
-    "xpostag",
-    "feats",
-    "head",
-    "deprel",
-    "deps",
-    "misc",
+    "ID",
+    "FORM",
+    "LEMMA",
+    "UPOS",
+    "XPOS",
+    "FEATS",
+    "HEAD",
+    "DEPREL",
+    "DEPS",
+    "MISC",
 ]
 
 
 @Language.factory(
     "conll_formatter",
-    default_config={"conversion_maps": None, "ext_names": None, "include_headers": False, "disable_pandas": False},
+    default_config={
+        "conversion_maps": None,
+        "ext_names": None,
+        "field_names": None,
+        "include_headers": False,
+        "disable_pandas": False
+    },
 )
 def create_conll_formatter(
-    nlp: Language,
-    name: str,
-    conversion_maps: Optional[Dict[str, Dict[str, str]]] = None,
-    ext_names: Optional[Dict[str, str]] = None,
-    include_headers: bool = False,
-    disable_pandas: bool = False,
+        nlp: Language,
+        name: str,
+        conversion_maps: Optional[Dict[str, Dict[str, str]]] = None,
+        ext_names: Optional[Dict[str, str]] = None,
+        field_names: Dict[str, str] = None,
+        include_headers: bool = False,
+        disable_pandas: bool = False,
 ):
     return ConllFormatter(
         conversion_maps=conversion_maps,
         ext_names=ext_names if ext_names else {},
+        field_names=field_names if field_names else {},
         include_headers=include_headers,
         disable_pandas=disable_pandas,
     )
@@ -77,8 +84,11 @@ class ConllFormatter:
     on the first level, and the conversion map on the second.
     E.g. {'lemma': {'-PRON-': 'PRON'}} will map the lemma '-PRON-' to 'PRON'
     :param ext_names: dictionary containing names for the custom spaCy extensions. You can rename the following
-    extensions: conll, conll_pd, conll_str.
-    E.g. {'conll': 'conll_dict', 'conll_pd': 'conll_pandas'} will rename the properties accordingly
+    extensions (use as keys): 'conll', 'conll_pd', 'conll_str'. E.g. {'conll': 'conll_dict', 'conll_pd': 'conll_pandas'}
+     will rename the properties accordingly
+    :param field_names: dictionary containing names for custom field names in case you do not want to use default
+     CoNLL-U field names. You can rename the following fields (use as keys): 'ID', 'FORM', 'LEMMA', 'UPOS', 'XPOS',
+     'FEATS', 'HEAD', 'DEPREL', 'DEPS', 'MISC'. E.g. {'UPOS': 'upostag'} will rename the field name UPOS accordingly.
     :param include_headers: whether to include the CoNLL headers in the conll_str string output. These consist
     of two lines containing the sentence id and the text as per the CoNLL format
     https://universaldependencies.org/format.html#sentence-boundaries-and-comments.
@@ -88,13 +98,16 @@ class ConllFormatter:
 
     conversion_maps: Optional[Dict[str, Dict[str, str]]] = None
     ext_names: Dict[str, str] = field(default_factory=dict)
+    field_names: Dict[str, str] = field(default_factory=dict)
     include_headers: bool = False
     disable_pandas: bool = False
 
     def __post_init__(self):
         # Set custom attribute names so that users can access them with their own preference
         default_ext_names = {"conll_str": "conll_str", "conll": "conll", "conll_pd": "conll_pd"}
         self.ext_names = merge_dicts_strict(default_ext_names, self.ext_names)
+        default_field_names = {fname: fname for fname in CONLL_FIELD_NAMES}
+        self.field_names = merge_dicts_strict(default_field_names, self.field_names)
 
         # Initialize extensions
         self._set_extensions()
@@ -197,7 +210,7 @@ def _set_token_conll(self, token: Token, token_idx: int = 1) -> Token:
         )
 
         # turn field name values (keys) and token values (values) into dict
-        token_conll_d = OrderedDict(zip(CONLL_FIELD_NAMES, token_conll))
+        token_conll_d = OrderedDict(zip(list(self.field_names.values()), token_conll))
 
         # convert properties if needed
         if self.conversion_maps:

tests/conftest.py

@@ -63,16 +63,22 @@ def spacy_conllparser():
 
 @pytest.fixture
 def spacy_ext_names():
-    return init_parser("en_core_web_sm", "spacy",
-                       ext_names={"conll": "conllu", "conll_str": "conll_text", "conll_pd": "pandas"}
-                       )
+    return init_parser(
+        "en_core_web_sm", "spacy",
+        ext_names={"conll": "conllu", "conll_str": "conll_text", "conll_pd": "pandas"}
+    )
 
 
 @pytest.fixture
 def spacy_conversion_map():
     return init_parser("en_core_web_sm", "spacy", conversion_maps={"lemma": {"-PRON-": "PRON"}})
 
 
+@pytest.fixture
+def spacy_field_names():
+    return init_parser("en_core_web_sm", "spacy", field_names={"UPOS": "upostag"})
+
+
 @pytest.fixture
 def spacy_disabled_pandas():
     return init_parser("en_core_web_sm", "spacy", disable_pandas=True)
@@ -112,6 +118,11 @@ def spacy_ext_names_doc(spacy_ext_names):
     return spacy_ext_names(single_sent())
 
 
+@pytest.fixture
+def spacy_fields_names_doc(spacy_field_names):
+    return spacy_field_names(single_sent())
+
+
 @pytest.fixture
 def spacy_conversion_map_doc(spacy_conversion_map):
     return spacy_conversion_map(single_sent())

tests/test_conversion_maps.py

@@ -1,7 +1,7 @@
 def test_conversion_map_pronoun(spacy_conversion_map_doc):
     pronoun = spacy_conversion_map_doc[0]
     # Verify that -PRON- was changed to PRON
-    assert pronoun._.conll["lemma"] == "he"
+    assert pronoun._.conll["LEMMA"] == "he"
     # lemma is the third column
     assert pronoun._.conll_str.split("\t")[2] == "he"
-    assert pronoun._.conll_pd["lemma"] == "he"
+    assert pronoun._.conll_pd["LEMMA"] == "he"

tests/test_field_names.py

@@ -0,0 +1,14 @@
+def test_fields_names_doc_pd(spacy_fields_names_doc):
+    assert spacy_fields_names_doc._.conll_pd.columns[3] == "upostag"
+
+
+def test_fields_names_sents_conll(spacy_fields_names_doc):
+    for sent in spacy_fields_names_doc.sents:
+        assert sent._.conll_pd.columns[3] == "upostag"
+
+
+def test_fields_names_token_conll(spacy_fields_names_doc):
+    for token in spacy_fields_names_doc:
+        assert "UPOS" not in token._.conll
+        assert "upostag" in token._.conll
+        assert token._.conll_pd.index[3] == "upostag"