Skip to content

Commit 577ef6e

Browse files
nicokantnicokant
committed
feat: harmonize accelerometers and tdr
1 parent 9664a4a commit 577ef6e

12 files changed

Lines changed: 529 additions & 18 deletions

‎src/gps_logger_parser/accelerometer/__init__.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@
55

66
from ..helpers import stream_starts_with
77
from ..parser_base import Parsable, Parser
8+
from .columns import AccelerometerHarmonizedColumn
9+
from .mixin import AccelerometerHarmonizationMixin
810

911

1012
def skip(row):
1113
return "error"
1214

1315

14-
class AcceleratorParser(Parser):
16+
class AcceleratorParser(AccelerometerHarmonizationMixin, Parser):
1517
DATATYPE = "accelerometer"
1618
FIELDS = ["X", "Y", "Z"]
1719
HEAD = "ACCELERATION DATA"
@@ -22,6 +24,12 @@ class AcceleratorParser(Parser):
2224
FREQUENCY_REGEX = r"\s*(\d*)\smsec\/point"
2325
DELTA_ATTR = "milliseconds"
2426
MAX_READ = 30
27+
MAPPINGS = {
28+
AccelerometerHarmonizedColumn.TIMESTAMP: "datetime",
29+
AccelerometerHarmonizedColumn.X: "X",
30+
AccelerometerHarmonizedColumn.Y: "Y",
31+
AccelerometerHarmonizedColumn.Z: "Z",
32+
}
2533

2634
def __init__(self, parsable: Parsable):
2735
super().__init__(parsable)

‎src/gps_logger_parser/accelerometer/columns.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from enum import Enum
2+
3+
4+
class AccelerometerHarmonizedColumn(str, Enum):
5+
"""Enum of harmonized accelerometer column names"""
6+
7+
TIMESTAMP = "timestamp"
8+
X = "x"
9+
Y = "y"
10+
Z = "z"
11+
12+
13+
# Pandas dtype mapping for each harmonized column
14+
ACCELEROMETER_HARMONIZED_COLUMN_TYPES = {
15+
AccelerometerHarmonizedColumn.TIMESTAMP: "datetime64[ns]",
16+
AccelerometerHarmonizedColumn.X: "float64",
17+
AccelerometerHarmonizedColumn.Y: "float64",
18+
AccelerometerHarmonizedColumn.Z: "float64",
19+
}

‎src/gps_logger_parser/accelerometer/mixin.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
"""
2+
Accelerometer-specific harmonization mixin for parsers.
3+
4+
This module provides the AccelerometerHarmonizationMixin class that ensures all
5+
accelerometer harmonized columns exist with correct types according to the
6+
ACCELEROMETER_HARMONIZED_COLUMN_TYPES specification.
7+
"""
8+
9+
from .columns import ACCELEROMETER_HARMONIZED_COLUMN_TYPES
10+
11+
12+
class AccelerometerHarmonizationMixin:
13+
"""
14+
Mixin to provide accelerometer-specific harmonization functionality.
15+
16+
This mixin ensures that all accelerometer harmonized columns exist with
17+
correct types according to the ACCELEROMETER_HARMONIZED_COLUMN_TYPES
18+
specification.
19+
"""
20+
21+
def get_harmonization_schema(self) -> dict:
22+
return {
23+
harmonized_col.value: pd_dtype
24+
for harmonized_col, pd_dtype in (
25+
ACCELEROMETER_HARMONIZED_COLUMN_TYPES.items()
26+
)
27+
}

‎src/gps_logger_parser/tdr/__init__.py‎

Lines changed: 50 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
from ..helpers import stream_chunk_match, stream_starts_with
88
from ..parser_base import CSVParser, Parsable, Parser
9+
from .columns import TDRHarmonizedColumn
10+
from .mixin import TDRHarmonizationMixin
911

1012
ENDLINES = ["No Fast Log Data", "No Fast Data"]
1113

@@ -16,13 +18,19 @@ def skip(row):
1618
return "error"
1719

1820

19-
class TDRParser(Parser):
21+
class TDRParser(TDRHarmonizationMixin, Parser):
2022
DATATYPE = "tdr"
2123
FIELDS = ["Date/Time Stamp", "Pressure", "Temp"]
2224
SEPARATOR = ","
2325
ALLOWED_META = ["Resolution"]
2426
HEAD = "\n?Comment\\s:-"
2527
MAX_READ = 1000
28+
MAPPINGS = {
29+
TDRHarmonizedColumn.TIMESTAMP: "Date/Time Stamp",
30+
TDRHarmonizedColumn.PRESSURE: "Pressure",
31+
TDRHarmonizedColumn.TEMPERATURE: "Temp",
32+
TDRHarmonizedColumn.DEPTH_M: None,
33+
}
2634

2735
def __init__(self, parsable: Parsable):
2836
super().__init__(parsable)
@@ -69,7 +77,7 @@ def __init__(self, parsable: Parsable):
6977
convert_options=convert_options,
7078
).to_pandas()
7179

72-
for key, value in meta:
80+
for key, value in meta.items():
7381
self.data[key] = [value] * len(self.data)
7482

7583

@@ -79,9 +87,15 @@ class TDR2Parser(TDRParser):
7987
"Pressure",
8088
"Temp",
8189
]
90+
MAPPINGS = {
91+
TDRHarmonizedColumn.TIMESTAMP: "Time Stamp",
92+
TDRHarmonizedColumn.PRESSURE: "Pressure",
93+
TDRHarmonizedColumn.TEMPERATURE: "Temp",
94+
TDRHarmonizedColumn.DEPTH_M: None,
95+
}
8296

8397

84-
class PathtrackPressParser(Parser):
98+
class PathtrackPressParser(TDRHarmonizationMixin, Parser):
8599
DATATYPE = "tdr"
86100
DIVIDER = "*" * 85 + "\n"
87101
HEAD = DIVIDER + "PathTrack Raw Pressure Data File Downloaded from Base Station"
@@ -101,22 +115,29 @@ class PathtrackPressParser(Parser):
101115
)
102116
OUTLIERS = None
103117
SEPARATOR = ","
118+
MAPPINGS = {
119+
TDRHarmonizedColumn.TIMESTAMP: "timestamp",
120+
TDRHarmonizedColumn.PRESSURE: "depth_mbar_float",
121+
TDRHarmonizedColumn.TEMPERATURE: "temperature_float",
122+
TDRHarmonizedColumn.DEPTH_M: "depth_m_float",
123+
}
104124

105125
def harmonize_data(self, data):
106-
data["time"] = (
107-
data["hour"].astype(str)
108-
+ ":"
109-
+ data["minute"].astype(str)
110-
+ ":"
111-
+ data["second"].astype(str)
126+
data["timestamp"] = pd.to_datetime(
127+
{
128+
"year": 2000 + data["year"],
129+
"month": data["month"],
130+
"day": data["day"],
131+
"hour": data["hour"],
132+
"minute": data["minute"],
133+
"second": data["second"],
134+
}
112135
)
113-
data["date"] = (
114-
data["day"].astype(str)
115-
+ "/"
116-
+ data["month"].astype(str)
117-
+ ":"
118-
+ data["year"].astype(str)
136+
data["temperature_float"] = (
137+
data["temperature"] + data["temperature_decimal"] / 100
119138
)
139+
data["depth_mbar_float"] = data["depth_mbar"] + data["depth_mbar_decimal"] / 100
140+
data["depth_m_float"] = data["depth_m"] + data["depth_m_decimal"] / 100
120141
return super().harmonize_data(data)
121142

122143
def __init__(self, parsable: Parsable):
@@ -145,22 +166,34 @@ def __init__(self, parsable: Parsable):
145166
)
146167

147168

148-
class SimpleTDR(CSVParser):
169+
class SimpleTDR(TDRHarmonizationMixin, CSVParser):
149170
DATATYPE = "tdr"
150171
FIELDS = [
151172
"Time Stamp",
152173
"Pressure",
153174
"Temp",
154175
]
176+
MAPPINGS = {
177+
TDRHarmonizedColumn.TIMESTAMP: "Time Stamp",
178+
TDRHarmonizedColumn.PRESSURE: "Pressure",
179+
TDRHarmonizedColumn.TEMPERATURE: "Temp",
180+
TDRHarmonizedColumn.DEPTH_M: None,
181+
}
155182

156183

157-
class SimpleTDRVariantDate(CSVParser):
184+
class SimpleTDRVariantDate(TDRHarmonizationMixin, CSVParser):
158185
DATATYPE = "tdr"
159186
FIELDS = [
160187
"Date/Time Stamp",
161188
"Pressure",
162189
"Temp",
163190
]
191+
MAPPINGS = {
192+
TDRHarmonizedColumn.TIMESTAMP: "Date/Time Stamp",
193+
TDRHarmonizedColumn.PRESSURE: "Pressure",
194+
TDRHarmonizedColumn.TEMPERATURE: "Temp",
195+
TDRHarmonizedColumn.DEPTH_M: None,
196+
}
164197

165198

166199
PARSERS = [

‎src/gps_logger_parser/tdr/columns.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from enum import Enum
2+
3+
4+
class TDRHarmonizedColumn(str, Enum):
5+
"""Enum of harmonized TDR column names"""
6+
7+
TIMESTAMP = "timestamp"
8+
PRESSURE = "pressure"
9+
TEMPERATURE = "temperature"
10+
DEPTH_M = "depth_m"
11+
12+
13+
# Pandas dtype mapping for each harmonized column
14+
TDR_HARMONIZED_COLUMN_TYPES = {
15+
TDRHarmonizedColumn.TIMESTAMP: "datetime64[ns]",
16+
TDRHarmonizedColumn.PRESSURE: "float64",
17+
TDRHarmonizedColumn.TEMPERATURE: "float64",
18+
TDRHarmonizedColumn.DEPTH_M: "float64",
19+
}

‎src/gps_logger_parser/tdr/mixin.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
"""
2+
TDR-specific harmonization mixin for parsers.
3+
4+
This module provides the TDRHarmonizationMixin class that ensures all TDR
5+
harmonized columns exist with correct types according to the
6+
TDR_HARMONIZED_COLUMN_TYPES specification.
7+
"""
8+
9+
from .columns import TDR_HARMONIZED_COLUMN_TYPES
10+
11+
12+
class TDRHarmonizationMixin:
13+
"""
14+
Mixin to provide TDR-specific harmonization functionality.
15+
16+
This mixin ensures that all TDR harmonized columns exist with correct types
17+
according to the TDR_HARMONIZED_COLUMN_TYPES specification.
18+
"""
19+
20+
def get_harmonization_schema(self) -> dict:
21+
return {
22+
harmonized_col.value: pd_dtype
23+
for harmonized_col, pd_dtype in TDR_HARMONIZED_COLUMN_TYPES.items()
24+
}

‎tests/config.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,13 @@ files:
5151
type: gps_unknown
5252
An12_PJ.txt:
5353
type: gps_unknown
54+
A15153_08-06-2018.csv:
55+
type: tdr
56+
CG_DI_A08918_02-07-12.CSV:
57+
type: tdr
58+
Obs250823_130741_Tag47649Press.txt:
59+
type: tdr
60+
ES_RU_292616_TDR_red.csv:
61+
type: tdr
62+
ES-SK-F-19011-295827-20200702-X27_Acc.txt:
63+
type: accelerometer

‎tests/files/A15153_08-06-2018.csv‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
Data for Tag A15153
2+
File created using G5 Host version 7.3.0.0
3+
4+
Comment :- JA-A15153-080618-BF
5+
6+
The following data are the ID block contents
7+
**** Tag Details ****
8+
Firmware Version No,3
9+
Firmware Build Level,80
10+
11+
12+
The following data are the Lifetime notebook contents
13+
Tag ID,A15153
14+
Pressure Range ,10
15+
No of sensors ,2
16+
17+
18+
The following data are the Deployment notebook contents
19+
Start,Stop,Logging Rate,Sensors,Resolution,Fast Rate,Repeat Rate,Repeat Count
20+
31.05.2018 12:00:00,14.06.2018 12:00:00,1,PT,12,0,0,0,
21+
22+
Mission Parameters
23+
Dive Termination Percentage = 1,5%
24+
Dive Termination Depth = 1,5m
25+
Wet dry logging - Inhibited
26+
Logging of Dive data - Active
27+
28+
29+
The following data are the Daylog contents
30+
Tags Diary
31+
Shipped,09.05.2018 09:00:00,Host V1.0.1
32+
Deep Sleep,09.05.2018 09:00:00,
33+
Deep Sleep,09.05.2018 09:00:00,
34+
Clock Set,30.05.2018 23:15:00,
35+
Clock Set,30.05.2018 23:15:00,
36+
Deployment,30.05.2018 23:15:00,Host V7.3.0
37+
Deployment,30.05.2018 23:26:00,Host V7.3.0
38+
Total Days Alive = 9
39+
7786
40+
41+
Daylog data for last deployment
42+
Mission Day,Date,Max Temp,Min Temp,Max Depth,Min Depth,Batt Volts
43+
0,30.05.2018,28,469,25,906,0,15,0,06,3,15
44+
1,31.05.2018,36,406,16,484,10,34,-0,41,3,13
45+
2,01.06.2018,34,578,14,656,20,65,-0,29,3,12
46+
3,02.06.2018,34,844,16,281,9,18,-0,38,3,12
47+
4,03.06.2018,34,125,14,984,14,93,-0,29,3,12
48+
5,04.06.2018,33,844,10,188,10,56,-0,32,3,12
49+
6,05.06.2018,34,672,11,578,18,34,-0,19,3,11
50+
7,06.06.2018,34,984,9,156,20,40,-0,22,3,12
51+
8,07.06.2018,34,313,10,859,24,59,-0,10,3,12
52+
53+
54+
55+
Data Block 1
56+
Start Time = 31.05.2018 12:00:00
57+
Stop Time = 14.06.2018 12:00:00
58+
Logging Rate = 1
59+
Resolution = 12
60+
Data points available = 728520
61+
Time Stamp,Pressure,Temp
62+
31.05.2018 12:00:00,-0,26,34,313
63+
31.05.2018 12:00:01,-0,26,34,313
64+
31.05.2018 12:00:02,-0,26,34,313
65+
31.05.2018 12:00:03,-0,26,34,313
66+
31.05.2018 12:00:04,-0,26,34,313
67+
31.05.2018 12:00:05,-0,26,34,313
68+
31.05.2018 12:00:06,-0,26,34,313
69+
31.05.2018 12:00:07,-0,26,34,313
70+
31.05.2018 12:00:08,-0,26,34,313
71+
31.05.2018 12:00:09,-0,26,34,313
72+
31.05.2018 12:00:10,-0,26,34,313
73+
31.05.2018 12:00:11,-0,26,34,313
74+
31.05.2018 12:00:12,-0,26,34,313
75+
31.05.2018 12:00:13,-0,26,34,313
76+
31.05.2018 12:00:14,-0,26,34,313
77+
31.05.2018 12:00:15,-0,26,34,297
78+
31.05.2018 12:00:16,-0,26,34,297
79+
31.05.2018 12:00:17,-0,26,34,297
80+
31.05.2018 12:00:18,-0,29,34,297
81+
31.05.2018 12:00:19,-0,26,34,297
82+
31.05.2018 12:00:20,-0,26,34,297
83+
31.05.2018 12:00:21,-0,26,34,297
84+
31.05.2018 12:00:22,-0,29,34,297
85+
31.05.2018 12:00:23,-0,29,34,297
86+
31.05.2018 12:00:24,-0,26,34,297
87+
31.05.2018 12:00:25,-0,26,34,297
88+
31.05.2018 12:00:26,-0,26,34,297
89+
31.05.2018 12:00:27,-0,26,34,297
90+
31.05.2018 12:00:28,-0,26,34,297
91+
31.05.2018 12:00:29,-0,26,34,297
92+
31.05.2018 12:00:30,-0,26,34,266
93+
31.05.2018 12:00:31,-0,26,34,266
94+
31.05.2018 12:00:32,-0,26,34,266
95+
31.05.2018 12:00:33,-0,26,34,266
96+
31.05.2018 12:00:34,-0,26,34,266
97+
31.05.2018 12:00:35,-0,26,34,266
98+
31.05.2018 12:00:36,-0,29,34,266
99+
31.05.2018 12:00:37,-0,26,34,266
100+
31.05.2018 12:00:38,-0,26,34,266
101+
31.05.2018 12:00:39,-0,26,34,266
102+
31.05.2018 12:00:40,-0,26,34,266
103+
31.05.2018 12:00:41,-0,26,34,266
104+
31.05.2018 12:00:42,-0,26,34,266
105+
31.05.2018 12:00:43,-0,26,34,266
106+
31.05.2018 12:00:44,-0,26,34,266
107+
31.05.2018 12:00:45,-0,26,34,266
108+
31.05.2018 12:00:46,-0,26,34,266
109+
31.05.2018 12:00:47,-0,26,34,266
110+
31.05.2018 12:00:48,-0,26,34,266
111+
31.05.2018 12:00:49,-0,26,34,266

0 commit comments

Comments
 (0)