Skip to content

Commit 293a30d

Browse files
e-kotove-kotov
authored
Prevent aggregation errors because of NAs in the source data (#187)
* fix NA bug, add regression unit tests
1 parent 42f71dc commit 293a30d

29 files changed

Lines changed: 279 additions & 85 deletions

‎DESCRIPTION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
Package: spanishoddata
22
Title: Get Spanish Origin-Destination Data
3-
Version: 0.2.3
3+
Version: 0.2.3.9000
44
Authors@R: c(
55
person("Egor", "Kotov", , "kotov.egor@gmail.com", role = c("aut", "cre"),
66
comment = c(ORCID = "0000-0001-6690-5345")),

‎NEWS.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
# spanishoddata (development version)
2+
3+
## Bug fixes
4+
5+
* Fixed a bug ([#186](https://github.com/rOpenSpain/spanishoddata/issues/186)) where literal `'NA'` strings in the source MITMA CSV files caused DuckDB conversion errors when casting to enums or other non-string types. This fix handles `'NA'` values in columns like `renta`, `distancia`, `residencia`, `origen`, `destino`, `actividad`, and others across v1 and v2 data.
6+
17
# spanishoddata 0.2.3
28

39
## Bug fixes

‎inst/extdata/sql-queries/v1-nt-distritos-clean-csv-view-en.sql‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ CREATE OR REPLACE VIEW nt_csv_clean AS SELECT
22
fecha AS date,
33
CAST (CASE distrito
44
WHEN 'externo' THEN 'external'
5+
WHEN 'NA' THEN NULL
56
ELSE distrito
67
END AS ZONES_ENUM)
78
AS id,
8-
CAST(numero_viajes AS N_TRIPS_ENUM) AS n_trips,
9+
CAST (CASE numero_viajes WHEN 'NA' THEN NULL ELSE numero_viajes END AS N_TRIPS_ENUM) AS n_trips,
910
personas AS n_persons,
1011
CAST(year AS INTEGER) AS year,
1112
CAST(month AS INTEGER) AS month,

‎inst/extdata/sql-queries/v1-nt-distritos-clean-csv-view-es.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
CREATE OR REPLACE VIEW nt_csv_clean AS SELECT
22
fecha,
3-
CAST (distrito AS ZONES_ENUM) AS distrito,
4-
CAST(numero_viajes AS N_TRIPS_ENUM) AS numero_viajes,
3+
CAST (CASE distrito WHEN 'NA' THEN NULL ELSE distrito END AS ZONES_ENUM) AS distrito,
4+
CAST (CASE numero_viajes WHEN 'NA' THEN NULL ELSE numero_viajes END AS N_TRIPS_ENUM) AS numero_viajes,
55
personas,
66
CAST(year AS INTEGER) AS ano,
77
CAST(month AS INTEGER) AS mes,

‎inst/extdata/sql-queries/v1-nt-municipios-clean-csv-view-en.sql‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@ SELECT
1919
d.fecha AS date,
2020
CAST(CASE r.municipio_mitma
2121
WHEN 'externo' THEN 'external'
22+
WHEN 'NA' THEN NULL
2223
ELSE r.municipio_mitma
2324
END AS ZONES_ENUM) AS id,
24-
CAST(d.numero_viajes AS N_TRIPS_ENUM) AS n_trips,
25+
CAST (CASE d.numero_viajes WHEN 'NA' THEN NULL ELSE d.numero_viajes END AS N_TRIPS_ENUM) AS n_trips,
2526
SUM(d.personas) AS n_persons,
2627
CAST(d.year AS INTEGER) AS year,
2728
CAST(d.month AS INTEGER) AS month,

‎inst/extdata/sql-queries/v1-nt-municipios-clean-csv-view-es.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ FROM
1717
CREATE OR REPLACE VIEW nt_csv_clean AS
1818
SELECT
1919
d.fecha AS fecha,
20-
CAST(r.municipio_mitma AS ZONES_ENUM) AS municipio_mitma,
21-
CAST(d.numero_viajes AS N_TRIPS_ENUM) AS numero_viajes,
20+
CAST (CASE municipio_mitma WHEN 'NA' THEN NULL ELSE municipio_mitma END AS ZONES_ENUM) AS municipio,
21+
CAST (CASE numero_viajes WHEN 'NA' THEN NULL ELSE numero_viajes END AS N_TRIPS_ENUM) AS numero_viajes,
2222
SUM(d.personas) AS personas,
2323
CAST(d.year AS INTEGER) AS ano,
2424
CAST(d.month AS INTEGER) AS mes,

‎inst/extdata/sql-queries/v1-od-distritos-clean-csv-view-en.sql‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,30 @@ CREATE OR REPLACE VIEW od_csv_clean AS SELECT
33
periodo AS hour,
44
CAST (CASE origen
55
WHEN 'externo' THEN 'external'
6+
WHEN 'NA' THEN NULL
67
ELSE origen
78
END AS ZONES_ENUM)
89
AS id_origin,
910
CAST (CASE destino
1011
WHEN 'externo' THEN 'external'
12+
WHEN 'NA' THEN NULL
1113
ELSE destino
1214
END AS ZONES_ENUM)
1315
AS id_destination,
14-
CAST(distancia AS DISTANCE_ENUM) AS distance,
15-
CAST(CASE actividad_origen
16+
CAST (CASE distancia WHEN 'NA' THEN NULL ELSE distancia END AS DISTANCE_ENUM) AS distance,
17+
CAST (CASE actividad_origen
1618
WHEN 'casa' THEN 'home'
1719
WHEN 'otros' THEN 'other'
1820
WHEN 'trabajo' THEN 'work_or_study'
21+
WHEN 'NA' THEN NULL
1922
END AS ACTIV_ENUM) AS activity_origin,
20-
CAST(CASE actividad_destino
23+
CAST (CASE actividad_destino
2124
WHEN 'casa' THEN 'home'
2225
WHEN 'otros' THEN 'other'
2326
WHEN 'trabajo' THEN 'work_or_study'
27+
WHEN 'NA' THEN NULL
2428
END AS ACTIV_ENUM) AS activity_destination,
25-
CAST(residencia AS INE_PROV_CODE_ENUM) AS residence_province_ine_code,
29+
CAST (CASE residencia WHEN 'NA' THEN NULL ELSE residencia END AS INE_PROV_CODE_ENUM) AS residence_province_ine_code,
2630
CAST (CASE residencia
2731
WHEN '01' THEN 'Araba/Álava'
2832
WHEN '02' THEN 'Albacete'

‎inst/extdata/sql-queries/v1-od-distritos-clean-csv-view-es.sql‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
CREATE OR REPLACE VIEW od_csv_clean AS SELECT
22
fecha AS date,
33
periodo,
4-
CAST(origen AS ZONES_ENUM) AS origen,
5-
CAST(destino AS ZONES_ENUM) AS destino,
6-
CAST(CASE actividad_origen
4+
CAST (CASE origen WHEN 'NA' THEN NULL ELSE origen END AS ZONES_ENUM) AS origen,
5+
CAST (CASE destino WHEN 'NA' THEN NULL ELSE destino END AS ZONES_ENUM) AS destino,
6+
CAST (CASE actividad_origen
77
WHEN 'trabajo' THEN 'trabajo_estudio'
8+
WHEN 'NA' THEN NULL
9+
ELSE actividad_origen
810
END AS ACTIV_ENUM) AS actividad_origen,
9-
CAST(CASE actividad_destino
11+
CAST (CASE actividad_destino
1012
WHEN 'trabajo' THEN 'trabajo_estudio'
13+
WHEN 'NA' THEN NULL
14+
ELSE actividad_destino
1115
END AS ACTIV_ENUM) AS actividad_destino,
12-
CAST(distancia AS DISTANCE_ENUM) AS distancia,
13-
CAST(residencia AS INE_PROV_CODE_ENUM) AS residencia,
16+
CAST (CASE distancia WHEN 'NA' THEN NULL ELSE distancia END AS DISTANCE_ENUM) AS distancia,
17+
CAST (CASE residencia WHEN 'NA' THEN NULL ELSE residencia END AS INE_PROV_CODE_ENUM) AS residencia,
1418
CAST (CASE residencia
1519
WHEN '01' THEN 'Araba/Álava'
1620
WHEN '02' THEN 'Albacete'

‎inst/extdata/sql-queries/v1-od-municipios-clean-csv-view-en.sql‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,21 @@ CREATE OR REPLACE VIEW od_csv_clean AS
1818
SELECT
1919
d.fecha AS date,
2020
d.periodo AS hour,
21-
CAST(m1.municipio_mitma AS ZONES_ENUM) AS id_origin,
22-
CAST(m2.municipio_mitma AS ZONES_ENUM) AS id_destination,
21+
CAST (CASE m1.municipio_mitma WHEN 'NA' THEN NULL ELSE m1.municipio_mitma END AS ZONES_ENUM) AS id_origin,
22+
CAST (CASE m2.municipio_mitma WHEN 'NA' THEN NULL ELSE m2.municipio_mitma END AS ZONES_ENUM) AS id_destination,
2323
CAST(CASE d.actividad_origen
2424
WHEN 'casa' THEN 'home'
2525
WHEN 'otros' THEN 'other'
2626
WHEN 'trabajo' THEN 'work_or_study'
27+
WHEN 'NA' THEN NULL
2728
END AS ACTIV_ENUM) AS activity_origin,
2829
CAST(CASE d.actividad_destino
2930
WHEN 'casa' THEN 'home'
3031
WHEN 'otros' THEN 'other'
3132
WHEN 'trabajo' THEN 'work_or_study'
33+
WHEN 'NA' THEN NULL
3234
END AS ACTIV_ENUM) AS activity_destination,
33-
CAST(d.residencia AS INE_PROV_CODE_ENUM) AS residence_province_ine_code,
35+
CAST (CASE d.residencia WHEN 'NA' THEN NULL ELSE d.residencia END AS INE_PROV_CODE_ENUM) AS residence_province_ine_code,
3436
CAST(CASE d.residencia
3537
WHEN '01' THEN 'Araba/Álava'
3638
WHEN '02' THEN 'Albacete'
@@ -85,7 +87,7 @@ SELECT
8587
WHEN '51' THEN 'Ceuta'
8688
WHEN '52' THEN 'Melilla'
8789
END AS INE_PROV_NAME_ENUM) AS residence_province_name,
88-
CAST(d.distancia AS DISTANCE_ENUM) AS distance,
90+
CAST (CASE d.distancia WHEN 'NA' THEN NULL ELSE d.distancia END AS DISTANCE_ENUM) AS distance,
8991
SUM(d.viajes) AS n_trips,
9092
SUM(d.viajes_km) AS trips_total_length_km,
9193
CAST(d.year AS INTEGER) AS year,

‎inst/extdata/sql-queries/v1-od-municipios-clean-csv-view-es.sql‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,19 @@ FROM
1717
CREATE OR REPLACE VIEW od_csv_clean AS
1818
SELECT
1919
d.fecha AS fecha,
20-
CAST(m1.municipio_mitma AS ZONES_ENUM) AS origen,
21-
CAST(m2.municipio_mitma AS ZONES_ENUM) AS destino,
22-
CAST(CASE d.actividad_origen
20+
CAST (CASE m1.municipio_mitma WHEN 'NA' THEN NULL ELSE m1.municipio_mitma END AS ZONES_ENUM) AS origen,
21+
CAST (CASE m2.municipio_mitma WHEN 'NA' THEN NULL ELSE m2.municipio_mitma END AS ZONES_ENUM) AS destino,
22+
CAST (CASE d.actividad_origen
2323
WHEN 'trabajo' THEN 'trabajo_estudio'
24+
WHEN 'NA' THEN NULL
25+
ELSE d.actividad_origen
2426
END AS ACTIV_ENUM) AS actividad_origen,
25-
CAST(CASE d.actividad_destino
27+
CAST (CASE d.actividad_destino
2628
WHEN 'trabajo' THEN 'trabajo_estudio'
29+
WHEN 'NA' THEN NULL
30+
ELSE d.actividad_destino
2731
END AS ACTIV_ENUM) AS actividad_destino,
28-
CAST(d.residencia AS INE_PROV_CODE_ENUM) AS residencia,
32+
CAST (CASE d.residencia WHEN 'NA' THEN NULL ELSE d.residencia END AS INE_PROV_CODE_ENUM) AS residencia,
2933
CAST(CASE d.residencia
3034
WHEN '01' THEN 'Araba/Álava'
3135
WHEN '02' THEN 'Albacete'
@@ -81,7 +85,7 @@ SELECT
8185
WHEN '52' THEN 'Melilla'
8286
END AS INE_PROV_NAME_ENUM) AS residencia_nombre,
8387
d.periodo AS periodo,
84-
CAST(d.distancia AS DISTANCE_ENUM) AS distancia,
88+
CAST (CASE d.distancia WHEN 'NA' THEN NULL ELSE d.distancia END AS DISTANCE_ENUM) AS distancia,
8589
SUM(d.viajes) AS viajes,
8690
SUM(d.viajes_km) AS viajes_km,
8791
CAST(d.year AS INTEGER) AS ano,

0 commit comments

Comments
 (0)