aq-level2-validation/_02-data-checks.R at main · bcgov/aq-level2-validation · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
# _02-data-checks.R

source("_00-setup.R")

#### read in data ####
data<-readr::read_rds("unverified-data.rds")

#### data check - valid status ####
# all data should have validation status 0.
data |>
  dplyr::distinct(year=lubridate::year(DATE_PST),
                  STATION_NAME_FULL,
                  PARAMETER,
                  VALIDATION_STATUS) |>
  # dplyr::distinct(VALIDATION_STATUS)
  dplyr::filter(VALIDATION_STATUS!="Level 0")

#### data check - NA stations ####
#check if there are any stations with NA for STATION_NAME
data |>
  dplyr::filter(is.na(STATION_NAME))

#### data check - duplicates ####
# look for duplicates
duplicates<-data |>

  dplyr::group_by(DATE_PST,
                  STATION_NAME_FULL,
                  STATION_NAME,
                  PARAMETER,
                  VALIDATION_STATUS,
                  INSTRUMENT) |>

  dplyr::summarise(n = dplyr::n(),
                   .groups = "drop") |>

  dplyr::filter(n > 1L) |>

  dplyr::left_join(x=_,
                   y=data,
                   by=c("DATE_PST",
                        "STATION_NAME_FULL",
                        "STATION_NAME",
                        "PARAMETER",
                        "VALIDATION_STATUS",
                        "INSTRUMENT"))|>

  dplyr::arrange(DATE_PST,
                 STATION_NAME,
                 STATION_NAME_FULL,
                 PARAMETER) |>

  dplyr::select(DATE_PST,
                STATION_NAME,
                STATION_NAME_FULL,
                PARAMETER,
                INSTRUMENT,
                RAW_VALUE,
                everything())

#### data check - campbell loggers ####
# issue: when datalogger is switched to campbell ""_60|_MET_60| MET_60" is appended to STATION_NAME_FULL. we want same id's for same station
#fix: strip "_60|_MET_60| MET_60" from station names

data %<>%
  dplyr::mutate(STATION_NAME_FULL=stringr::str_remove(STATION_NAME_FULL,
                                                    "_60|_MET_60| MET_60"))


# save checked data
readr::write_rds(data,
                 "unverified-data-checked.rds")