chg: consistent gps output, with original data in json column

Author: nicokant

src/gps_logger_parser/gps/columns.py

@@ -18,7 +18,7 @@ class GPSHarmonizedColumn(str, Enum):
     PDOP = "pdop"
     SATELLITES_COUNT = "satellites_count"
     TEMPERATURE = "temperature"
-    SOLAR_I_MA = "solar_I_mA"
+    SOLAR_I_MA = "solar_i_ma"
     BAT_SOC_PCT = "bat_soc_pct"
     RING_NR = "ring_nr"
     TRIP_NR = "trip_nr"
@@ -30,7 +30,6 @@ class GPSHarmonizedColumn(str, Enum):
     GPSHarmonizedColumn.TIMESTAMP: "datetime64[ns]",
     GPSHarmonizedColumn.LATITUDE: "float64",
     GPSHarmonizedColumn.LONGITUDE: "float64",
-    GPSHarmonizedColumn.GEOMETRY: "geoarrow.point",
     GPSHarmonizedColumn.ALTITUDE: "float64",
     GPSHarmonizedColumn.SPEED_KM_H: "float64",
     GPSHarmonizedColumn.TYPE: "object",
@@ -44,4 +43,5 @@ class GPSHarmonizedColumn(str, Enum):
     GPSHarmonizedColumn.BAT_SOC_PCT: "float64",
     GPSHarmonizedColumn.RING_NR: "object",
     GPSHarmonizedColumn.TRIP_NR: "Int64",
+    GPSHarmonizedColumn.GEOMETRY: "object",
 }

src/gps_logger_parser/gps/ecotone.py

@@ -1,12 +1,46 @@
+import geoarrow.pyarrow as ga
+import numpy as np
+import pandas as pd
+
 from ..parser_base import CSVParser
+from .columns import GPSHarmonizedColumn
 from .mixin import GPSHarmonizationMixin
 
 
 class EcotoneParser(GPSHarmonizationMixin, CSVParser):
     DATATYPE = "gps_ecotone"
-    FIELDS = [x for x in range(0, 9)]
+    FIELDS = [
+        "logger_id",
+        "day",
+        "month",
+        "year",
+        "hours",
+        "minutes",
+        "meters_north",
+        "meters_east",
+        "HDOP",
+    ]
     SEPARATOR = ";"
     HEADER = 0
+    MAPPINGS = {
+        GPSHarmonizedColumn.ID: "logger_id",
+        GPSHarmonizedColumn.TIMESTAMP: None,
+        GPSHarmonizedColumn.LATITUDE: None,
+        GPSHarmonizedColumn.LONGITUDE: None,
+        GPSHarmonizedColumn.ALTITUDE: None,
+        GPSHarmonizedColumn.SPEED_KM_H: None,
+        GPSHarmonizedColumn.TYPE: None,
+        GPSHarmonizedColumn.DISTANCE: None,
+        GPSHarmonizedColumn.COURSE: None,
+        GPSHarmonizedColumn.HDOP: "HDOP",
+        GPSHarmonizedColumn.PDOP: None,
+        GPSHarmonizedColumn.SATELLITES_COUNT: None,
+        GPSHarmonizedColumn.TEMPERATURE: None,
+        GPSHarmonizedColumn.SOLAR_I_MA: None,
+        GPSHarmonizedColumn.BAT_SOC_PCT: None,
+        GPSHarmonizedColumn.RING_NR: None,
+        GPSHarmonizedColumn.TRIP_NR: None,
+    }
 
     def _check_headers(self, header):
         if len(header) != len(self.FIELDS):
@@ -15,6 +49,56 @@ def _check_headers(self, header):
                 f"{len(header)} != {len(self.FIELDS)}"
             )
 
+    def harmonize_data(self, data):
+
+        # Call parent harmonization — applies MAPPINGS, enforces GPS schema,
+        # creates geometry, and drops raw source columns
+        result = super().harmonize_data(data)
+
+        result["timestamp"] = pd.to_datetime(
+            data["year"].astype(str)
+            + "/"
+            + data["month"].astype(str)
+            + "/"
+            + data["day"].astype(str)
+            + " "
+            + data["hours"].astype(str)
+            + ":"
+            + data["minutes"].astype(str),
+            format="%Y/%m/%d %H:%M",
+            errors="raise",
+        )
+
+        if (
+            "meters_north" in data.columns
+            and "meters_east" in data.columns
+            and not data["meters_north"].isna().all()
+            and not data["meters_east"].isna().all()
+        ):
+            # Create point geometries from meters_north/meters_east coordinates
+            # GeoArrow expects (x, y) which is (longitude, latitude)
+            points = ga.make_point(
+                data["meters_east"].str[:-1].astype(float).values,
+                data["meters_north"].str[:-1].astype(float).values,
+                crs="EPSG:32633",  # UTM zone 33N
+            )
+            # Convert to pandas ArrowExtensionArray
+            result["geometry"] = pd.array(
+                points.to_pylist(), dtype=pd.ArrowDtype(points.type)
+            )
+        else:
+            # Create empty geometry column if lat/lon don't exist or are all null
+            empty_points = ga.make_point(
+                np.full(len(data), np.nan),
+                np.full(len(data), np.nan),
+                crs="EPSG:32633",  # UTM zone 33N
+            )
+            result["geometry"] = pd.array(
+                empty_points.to_pylist(), dtype=pd.ArrowDtype(empty_points.type)
+            )
+
+        return result
+
 
 PARSERS = [
     EcotoneParser,

src/gps_logger_parser/gps/ho11.py

@@ -52,6 +52,21 @@ def harmonize_data(self, data):
             errors="raise",
             format="%d.%m.%Y %H:%M:%S",
         )
+
+        for column in [
+            "Latitude",
+            "Longitude",
+            "Altitude",
+            "Speed",
+            "Course",
+            "Distance",
+        ]:
+            try:
+                data[column] = data[column].str.replace(",", ".", regex=False)
+                data[column] = pd.to_numeric(data[column], errors="coerce")
+            except AttributeError:
+                pass
+
         return super().harmonize_data(data)
 
 

src/gps_logger_parser/gps/jm.py

@@ -26,8 +26,6 @@ class GPS2JMParser7_5(GPSHarmonizationMixin, Parser):
     """
 
     DATATYPE = "gps_2jm"
-    # TODO: define fields
-    FIELDS = [str(x) for x in range(0, 13)]
     VERSION = "v7.5"
     SEPARATOR = " "
     ENDINGS = [
@@ -55,14 +53,14 @@ class GPS2JMParser7_5(GPSHarmonizationMixin, Parser):
         GPSHarmonizedColumn.TIMESTAMP: None,
         GPSHarmonizedColumn.LATITUDE: None,
         GPSHarmonizedColumn.LONGITUDE: None,
-        GPSHarmonizedColumn.ALTITUDE: None,
-        GPSHarmonizedColumn.SPEED_KM_H: None,
+        GPSHarmonizedColumn.ALTITUDE: "altitude",
+        GPSHarmonizedColumn.SPEED_KM_H: "speed",
         GPSHarmonizedColumn.TYPE: None,
-        GPSHarmonizedColumn.DISTANCE: None,
+        GPSHarmonizedColumn.DISTANCE: "distance",
         GPSHarmonizedColumn.COURSE: None,
         GPSHarmonizedColumn.HDOP: None,
         GPSHarmonizedColumn.PDOP: None,
-        GPSHarmonizedColumn.SATELLITES_COUNT: None,
+        GPSHarmonizedColumn.SATELLITES_COUNT: "satellite",
         GPSHarmonizedColumn.TEMPERATURE: None,
         GPSHarmonizedColumn.SOLAR_I_MA: None,
         GPSHarmonizedColumn.BAT_SOC_PCT: None,
@@ -71,56 +69,49 @@ class GPS2JMParser7_5(GPSHarmonizationMixin, Parser):
     }
 
     def harmonize_data(self, data):
-        # Combine start date with time from each row
-        # start_date is in format DD.MM.YYYY, time is HH:MM:SS
-        # Call parent harmonization first
-        data = super().harmonize_data(data)
+        # Call parent harmonization — applies MAPPINGS, enforces GPS schema,
+        # creates geometry, and drops raw source columns
+        result = super().harmonize_data(data)
 
         # Convert coordinates from degrees + decimal minutes to decimal degrees
+        # before calling super(), so the harmonized lat/lon are in decimal degrees.
         # Latitude: degrees + (minutes / 60), with direction sign
         lat_decimal_degrees = (
-            data["__original__latitude"].astype(int)
-            + data["__original__latitude_decimal"].astype(float) / 60
+            data["latitude"].astype(int) + data["latitude_decimal"].astype(float) / 60
         )
-        data["latitude"] = [
+        result["latitude"] = [
             signed(lat, direction)
-            for lat, direction in zip(
-                lat_decimal_degrees, data["__original__n"], strict=False
-            )
+            for lat, direction in zip(lat_decimal_degrees, data["n"], strict=False)
         ]
 
         # Longitude: degrees + (minutes / 60), with direction sign
         lon_decimal_degrees = (
-            data["__original__longitude"].astype(int)
-            + data["__original__longitude_decimal"].astype(float) / 60
+            data["longitude"].astype(int) + data["longitude_decimal"].astype(float) / 60
         )
-        data["longitude"] = [
+        result["longitude"] = [
             signed(lon, direction)
-            for lon, direction in zip(
-                lon_decimal_degrees, data["__original__e"], strict=False
-            )
+            for lon, direction in zip(lon_decimal_degrees, data["e"], strict=False)
         ]
 
-        # Then create the timestamp column after original columns are prefixed
+        # Build timestamp column before calling super()
         if hasattr(self, "start_date") and self.start_date:
-            # time column is now __original__time
-            data["timestamp"] = pd.to_datetime(
-                self.start_date + " " + data["__original__time"],
+            result["timestamp"] = pd.to_datetime(
+                self.start_date + " " + data["time"],
                 format="%d.%m.%Y %H:%M:%S",
-                errors="raise",
+                errors="coerce",
             )
         else:
             # Fallback if start_date not available
-            data["timestamp"] = pd.to_datetime(
-                data["__original__date"].astype(str) + " " + data["__original__time"],
+            result["timestamp"] = pd.to_datetime(
+                data["date"].astype(str) + " " + data["time"],
                 format="%d %H:%M:%S",
-                errors="raise",
+                errors="coerce",
             )
 
         # Recreate geometry column now that lat/lon are finalized
-        data = self._create_geometry_column(data)
+        result = self._create_geometry_column(result)
 
-        return data
+        return result
 
     def _fix_content(self, data):
         return data
@@ -284,17 +275,18 @@ class GPS2JMParser8Alternative(GPSHarmonizationMixin, Parser):
     }
 
     def harmonize_data(self, data):
-        # Combine UTC_date and UTC_time columns into timestamp
-        # Call parent harmonization first
-        data = super().harmonize_data(data)
 
-        # Then create timestamp from the prefixed original columns
-        data["timestamp"] = pd.to_datetime(
-            data["__original__UTC_date"] + " " + data["__original__UTC_time"],
+        # Call parent harmonization — applies MAPPINGS, enforces GPS schema,
+        # creates geometry, and drops raw source columns
+        result = super().harmonize_data(data)
+
+        # Build timestamp from raw UTC_date and UTC_time columns
+        result["timestamp"] = pd.to_datetime(
+            data["UTC_date"] + " " + data["UTC_time"],
             format="%d.%m.%Y %H:%M:%S",
             errors="raise",
         )
-        return data
+        return result
 
     def _fix_content(self, data: str):
         """

src/gps_logger_parser/gps/mixin.py

@@ -10,7 +10,7 @@
 import numpy as np
 import pandas as pd
 
-from .columns import GPS_HARMONIZED_COLUMN_TYPES, GPSHarmonizedColumn
+from .columns import GPS_HARMONIZED_COLUMN_TYPES
 
 
 class GPSHarmonizationMixin:
@@ -35,47 +35,11 @@ def harmonize_data(self, data):
         # First, apply standard column renaming
         data = super().harmonize_data(data)
 
-        # Ensure all GPS harmonized columns exist with correct types
-        for harmonized_col in GPSHarmonizedColumn:
-            col_name = harmonized_col.value
-            pd_dtype = GPS_HARMONIZED_COLUMN_TYPES[harmonized_col]
-
-            # Skip geometry column - we'll create it last from lat/lon
-            if col_name == "geometry":
-                continue
-
-            if col_name not in data.columns:
-                # Add column with null values and appropriate dtype
-                if pd_dtype == "float64":
-                    data[col_name] = np.nan
-                elif pd_dtype == "datetime64[ns]":
-                    data[col_name] = pd.NaT
-                else:
-                    data[col_name] = None
-
-            # Ensure column has the correct dtype
-            col_exists = col_name in data.columns
-            if col_exists and data[col_name].dtype != pd_dtype:
-                # Convert to correct dtype, handling potential type conversion issues
-                if pd_dtype == "object":
-                    if data[col_name].dtype != "object":
-                        data[col_name] = data[col_name].astype(str)
-                elif pd_dtype == "float64":
-                    data[col_name] = pd.to_numeric(data[col_name], errors="raise")
-                elif pd_dtype == "Int64":
-                    # Convert to numeric first, then to nullable Int64
-                    data[col_name] = pd.to_numeric(data[col_name], errors="raise")
-                    # Round to handle floating point values before conversion
-                    data[col_name] = data[col_name].round().astype("Int64")
-                elif pd_dtype == "datetime64[ns]":
-                    data[col_name] = pd.to_datetime(data[col_name], errors="raise")
-
         # Create geometry column from latitude and longitude (WGS84)
         # Note: This creates geometry from current lat/lon values
         # If parsers modify lat/lon after calling super(), they should
         # call _create_geometry_column() again
         data = self._create_geometry_column(data)
-
         return data
 
     def _create_geometry_column(self, data):
@@ -126,4 +90,7 @@ def get_harmonization_schema(self):
         """
         Return None - we use pandas types directly, not PyArrow schemas
         """
-        return None
+        return {
+            harmonized_col.value: pd_dtype
+            for harmonized_col, pd_dtype in GPS_HARMONIZED_COLUMN_TYPES.items()
+        }

src/gps_logger_parser/gps/pathtrack.py

@@ -56,7 +56,8 @@ class PathtrackParser(GPSHarmonizationMixin, Parser):
     def harmonize_data(self, data):
         # Combine date and time fields into timestamp
         data["timestamp"] = pd.to_datetime(
-            data["year"].astype(str)
+            "20"
+            + data["year"].astype(str)
             + "/"
             + data["month"].astype(str)
             + "/"
@@ -164,7 +165,8 @@ class CSVPathtrack(GPSHarmonizationMixin, CSVParser):
     def harmonize_data(self, data):
         # Combine date and time fields into a timestamp
         data["timestamp"] = pd.to_datetime(
-            data["year"].astype(str)
+            "20"  # There are just the last 2 digits of the year
+            + data["year"].astype(str)
             + "/"
             + data["month"].astype(str)
             + "/"
@@ -178,6 +180,19 @@ def harmonize_data(self, data):
             format="%Y/%m/%d %H:%M:%S",
             errors="raise",
         )
+
+        for column in [
+            "latitude",
+            "longitude",
+            "altitude",
+        ]:
+            try:
+                data[column] = data[column].str.replace(",", ".", regex=False)
+                data[column] = pd.to_numeric(data[column], errors="coerce")
+            except AttributeError:
+                # this fails only when the column is already numeric
+                pass
+
         return super().harmonize_data(data)