Skip to content

Improve blueprints #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
oscgonfer wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Improve blueprints #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e3770f6
Rename metrics to channels
oscgonfer Apr 13, 2026
82b15d2
Add checks to blueprint
oscgonfer Apr 13, 2026
0d1050c
Add columns to flat values check
oscgonfer Apr 13, 2026
b3c9def
Minor function name change
oscgonfer Apr 13, 2026
0c811cd
More improvements on checks blueprint
oscgonfer Apr 13, 2026
11f6817
Test alternative gap method
oscgonfer Apr 13, 2026
43f6e28
Rename device.metrics to channels. Add checks to device.
oscgonfer Apr 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
234 changes: 233 additions & 1 deletion blueprints/sc_air.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"meta": {
"documentation": "https://docs.smartcitizen.me/"
},
"metrics": [
"channels": [
{
"name": "PT1000_POS",
"description": "PT1000 raw value",
Expand Down Expand Up @@ -307,5 +307,237 @@
"unit": "ppm",
"depends_on": ["CO2_BASELINE"]
}
],
"checks": [
{
"name": "GAPS",
"description": "Find gaps in columns",
"kwargs": {
"default_gap_size_minutes": 5,
"default_frequency_minutes": 1,
"gap_sizes": [
{
"columns": ["SEN5X_PM_1", "SEN5X_PM_10", "SEN5X_PM_25", "SEN5X_PM_40", "SEN5X_PN_05", "SEN5X_PN_1", "SEN5X_PN_10", "SEN5X_PN_25", "SEN5X_PN_40", "SEN5X_TPS", "SEN5X_TEMP", "SEN5X_HUM"],
"gap_size_minutes": 10
}
],
"frequencies": [
{
"columns": ["SEN5X_PM_1", "SEN5X_PM_10", "SEN5X_PM_25", "SEN5X_PM_40", "SEN5X_PN_05", "SEN5X_PN_1", "SEN5X_PN_10", "SEN5X_PN_25", "SEN5X_PN_40", "SEN5X_TPS", "SEN5X_TEMP", "SEN5X_HUM"],
"frequency_minutes": 5
}
]
},
"store_qc": true,
"clean": false,
"function": "find_gaps"
},
{
"name": "IMPLAUSIBLE",
"description": "Find implausible values in columns",
"kwargs": {
"implausible_values": [
{
"column": "CCS811_VOCS",
"limits": [0, 30000]
},
{
"column": "CCS811_ECO2",
"limits": [400, 30000]
},
{
"column": "NOISE_A",
"limits": [20, 99]
},
{
"column": "SCD30_CO2",
"limits": [300, 9500]
},
{
"column": "SCD30_HUM",
"limits": [20, 99]
},
{
"column": "SCD30_TEMP",
"limits": [-20, 50]
},
{
"column": "BATT",
"limits": [0, 100]
},
{
"column": "LPS33_PRESS",
"limits": [50, 110]
},
{
"column": "PRESS",
"limits": [50, 110]
},
{
"column": "PMS5003_PM_1",
"limits": [0, 500]
},
{
"column": "PMS5003_PM_25",
"limits": [0, 500]
},
{
"column": "PMS5003_PM_10",
"limits": [0, 500]
},
{
"column": "SEN5X_HUM",
"limits": [20, 99]
},
{
"column": "SEN5X_PM_1",
"limits": [0, 500]
},
{
"column": "SEN5X_PM_10",
"limits": [0, 500]
},
{
"column": "SEN5X_PM_25",
"limits": [0, 500]
},
{
"column": "SEN5X_PM_40",
"limits": [0, 500]
},
{
"column": "SEN5X_TEMP",
"limits": [-20, 50]
},
{
"column": "SFA30_HCHO",
"limits": [0, 4500]
},
{
"column": "SFA30_HUM",
"limits": [20, 99]
},
{
"column": "SFA30_TEMP",
"limits": [-20, 50]
},
{
"column": "ADC_48_0",
"limits": [0, 5.1]
},
{
"column": "ADC_48_1",
"limits": [0, 5.1]
},
{
"column": "ADC_48_2",
"limits": [0, 5.1]
},
{
"column": "ADC_48_3",
"limits": [0, 5.1]
},
{
"column": "ADC_49_0",
"limits": [0, 5.1]
},
{
"column": "ADC_49_1",
"limits": [0, 5.1]
},
{
"column": "ADC_49_2",
"limits": [0, 5.1]
},
{
"column": "ADC_49_3",
"limits": [0, 13]
},
{
"column": "HUM",
"limits": [20, 99]
},
{
"column": "TEMP",
"limits": [-40, 60]
},
{
"column": "NO2",
"limits": [0, 1000]
},
{
"column": "O3",
"limits": [0, 1000]
}
]
},
"store_qc": true,
"clean": true,
"function": "find_implausible_values"
},
{
"name": "FLAT",
"description": "Find flat values in columns",
"kwargs": {
"flat_window_minutes": 1440,
"limit_rolling_std": 0.00001,
"columns": [
"SEN5X_PM_1",
"SEN5X_PM_25",
"SEN5X_PM_40",
"SEN5X_PM_10",
"PMS5003_PM_1",
"PMS5003_PM_25",
"PMS5003_PM_10",
"PRESS",
"NOISE_A",
"TEMP",
"HUM",
"SCD30_CO2",
"SFA30_HCHO",
"ADC_48_2",
"ADC_48_3",
"ADC_49_0",
"ADC_49_1",
"ADC_49_2",
"ADC_49_3",
"LPS33_PRESS"
]
},
"store_qc": true,
"clean": false,
"function": "find_flat_values"
},
{
"name": "OUTLIERS",
"description": "Find outliers values in columns",
"kwargs": {
"columns": [
"SEN5X_PM_1",
"SEN5X_PM_25",
"SEN5X_PM_40",
"SEN5X_PM_10",
"PMS5003_PM_1",
"PMS5003_PM_25",
"PMS5003_PM_10",
"PRESS",
"NOISE_A",
"TEMP",
"HUM",
"SCD30_CO2",
"SFA30_HCHO",
"ADC_48_2",
"ADC_48_3",
"ADC_49_0",
"ADC_49_1",
"ADC_49_2",
"ADC_49_3",
"LPS33_PRESS"
]
},
"store_qc": true,
"clean": false,
"function": "find_outliers_isolation_forest"
}
]
}
66 changes: 4 additions & 62 deletions scdata/_config/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from pydantic import TypeAdapter
from typing import List

from scdata.models import Name, Blueprint, Metric
from scdata.models import Name, Blueprint
from scdata.tools.dictmerge import dict_fmerge
from scdata.tools.gets import get_json_from_url

Expand Down Expand Up @@ -76,7 +76,7 @@ class Config(object):
### -------------SMART CITIZEN-------------
### ---------------------------------------
# # Urls
_base_postprocessing_url = 'https://raw.githubusercontent.com/fablabbcn/smartcitizen-data/master/'
_base_postprocessing_url = 'https://raw.githubusercontent.com/fablabbcn/smartcitizen-data/enhancement/improve-blueprints/'
_default_file_type = 'json'

calibrations_urls = [
Expand Down Expand Up @@ -406,68 +406,10 @@ class Config(object):
### ------------VALUES-CHECK---------------
### ---------------------------------------

_default_sampling_rate = {
'AMS AS7731 - UVA': 1,
'AMS AS7731 - UVB': 1,
'AMS AS7731 - UVC': 1,
'LIGHT': 1,
'BATT': 1,
'NOISE_A': 1,
'SCD30_CO2': 1,
'SCD30_HUM': 1,
'SCD30_TEMP': 1,
'SD-card': 1,
'LPS33_PRESS': 1,
'PRESS': 1,
'PMS5003_PM_1': 5,
'PMS5003_PM_25': 5,
'PMS5003_PM_10': 5,
'PMS5003_PN_03': 5,
'PMS5003_PN_03': 5,
'PMS5003_PN_05':5,
'PMS5003_PN_1':5,
'PMS5003_PN_10':5,
'PMS5003_PN_25':5,
'PMS5003_PN_5':5,
'SEN5X_HUM': 5,
'SEN5X_PM_1': 5,
'SEN5X_PM_10': 5,
'SEN5X_PM_25': 5,
'SEN5X_PM_40': 5,
'SEN5X_PN_05': 5,
'SEN5X_PN_1': 5,
'SEN5X_PN_10': 5,
'SEN5X_PN_25': 5,
'SEN5X_PN_40': 5,
'SEN5X_TPS': 5,
'SEN5X_TEMP': 5,
'SFA30_HCHO': 1,
'SFA30_HUM': 1,
'SFA30_TEMP': 1,
'ADC_48_0': 1,
'ADC_48_1': 1,
'ADC_48_2': 1,
'ADC_48_3': 1,
'ADC_49_0': 1,
'ADC_49_1': 1,
'ADC_49_2': 1,
'ADC_49_3': 1,
'CCS811_VOCS': 1,
'CCS811_ECO2': 1,
'HUM': 1,
'TEMP': 1,
'RSSI': 1,
'NO2': 1,
'O3': 1,
'AS_TEMP': 1,
'AS_PH': 1,
'AS_COND': 1,
'AS_DO_SAT': 1,
'AS_DO': 1
}
_default_gap_size_minutes = 5

# TODO - Review limits
_default_unplausible_values = {
_default_implausible_values = {
'CCS811_VOCS': [0, 30000],
'CCS811_ECO2': [400, 30000],
'NOISE_A': [20, 99],
Expand Down
4 changes: 4 additions & 0 deletions scdata/device/check/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
from .flats import find_flat_values
from .implausible import find_implausible_values
from .gaps import find_gaps
from .outliers import find_outliers_isolation_forest
25 changes: 25 additions & 0 deletions scdata/device/check/flats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
from scdata.tools.custom_logger import logger
from scdata.device.process.error_codes import StatusCode, ProcessResult

def find_flat_values(dataframe, **kwargs):

flat_sensor_window = kwargs.get('flat_sensor_window', 1000)
limit_rolling_std = kwargs.get('limit_rolling_std', 1e-5)
columns = kwargs.get('columns', list(dataframe.columns))

df = dataframe.copy()
cols = []

logger.info(f'Flat window size: {flat_sensor_window} minutes. STD Limit: {limit_rolling_std}')
for col in columns:
if '__' in col: continue # Internal code for healthchecks
if col not in df.columns:
logger.warning(f'{col} not in columns. Skipping')
continue

logger.info (f'Calculating flat values for {col}')
df[f'__{col}'] = df[col].rolling(window=flat_sensor_window).std() < limit_rolling_std

cols.append(f'__{col}')

return ProcessResult(df[cols], StatusCode.SUCCESS)
Loading
Loading