feat: add method to get field names on latest index

Useful for very quickly retrieving a basic list of all fields from the latest index. Returns field type information, but type counts are all set to 1.

Author: alycejenni

splitgill/manager.py

@@ -4,7 +4,7 @@
 
 from cytoolz.dicttoolz import get_in
 from elasticsearch import Elasticsearch
-from elasticsearch_dsl import Search
+from elasticsearch_dsl import Index, Search
 from elasticsearch_dsl.query import Query
 from pymongo import ASCENDING, DESCENDING, IndexModel, MongoClient
 from pymongo.collection import Collection
@@ -735,3 +735,27 @@ def get_parsed_fields(
         # descending frequency (so most frequent fields first)
         parsed_fields.sort(key=lambda f: f.count, reverse=True)
         return parsed_fields
+
+    def get_field_names(self) -> List[str]:
+        """
+        Retrieves a list of field names from the latest index mapping.
+
+        Does not take any version or query parameters; simply returns all the "data."
+        fields available on the index, along with their available types. All relevant
+        type counts are set to 1 to enable use of e.g. .is_text(). Use get_data_fields
+        or get_parsed_fields if you need accurate counts, or to filter by version or
+        query.
+        """
+        latest_index = Index(self.indices.latest, using=self._client.elasticsearch)
+        mapping = latest_index.get_mapping()
+        parsed_fields = []
+        for field_path, field_props in get_in(
+            [self.indices.latest, 'mappings', 'properties', 'data', 'properties'],
+            mapping.body,
+            default={},
+        ).items():
+            parsed_field = ParsedField(field_path)
+            for type_name in field_props['properties'].keys():
+                parsed_field.add(type_name, 1)
+            parsed_fields.append(parsed_field)
+        return parsed_fields

tests/test_manager.py

@@ -1476,3 +1476,29 @@ def test_resync_arcs(splitgill: SplitgillClient):
     assert count == 2400
     assert r_5_count == 3
     assert r_780_count == 2
+
+
+def test_get_field_names(splitgill: SplitgillClient):
+    database = splitgill.get_database('test')
+    records = [
+        Record.new({'a': 1}),
+        Record.new({'a': 2}),
+        Record.new({'b': 3}),
+        Record.new({'b': 'x'}),
+        Record.new({'b': 5}),
+        Record.new({'c': 'y'}),
+    ]
+    database.ingest(records, commit=True)
+    database.sync()
+
+    field_names = database.get_field_names()
+    assert len(field_names) == 4
+    expected_fields = [
+        pf('_id', 3, t=1),
+        pf('a', 4, t=1, n=1),
+        pf('b', 4, t=1, n=1),
+        pf('c', 3, t=1),
+    ]
+    for f in expected_fields:
+        f.type_counts[ParsedType.UNPARSED] = 1
+    assert field_names == expected_fields