Merge pull request #4698 from bcgov/4511-validate-config-before-saving-activity-data

chore: 4511 validate configuration exists for submitted data

Author: pbastia

bc_obps/common/exceptions.py

@@ -5,3 +5,13 @@ class UserError(Exception):
     """
 
     pass
+
+
+class InternalSystemError(Exception):
+    """
+    Base class for obps system-related errors.
+    These errors are typically used for unexpected conditions that indicate a problem in either
+    unforeseen manipulation of the system, or a bug in the code.
+    """
+
+    pass

bc_obps/reporting/service/report_validation/validators/__init__.py

@@ -9,6 +9,7 @@
     report_data_by_fuel_type_validator,
     report_emission_allocation_other_excluded_category,
     report_emission_allocation_validator,
+    reporting_configuration_validator,
     supplementary_report_attachments_confirmation,
     supplementary_report_version_change,
     report_regulated_product_presence,
@@ -41,6 +42,7 @@
     "report_emission_allocation_other_excluded_category",
     "report_data_by_fuel_type_validator",
     "report_regulated_product_presence",
+    "reporting_configuration_validator",
     "required_fields_report_operation_information",
     "required_fields_report_person_responsible",
     "required_fields_report_activity_data",

bc_obps/reporting/service/report_validation/validators/reporting_configuration_validator.py

@@ -0,0 +1,86 @@
+from common.exceptions import InternalSystemError
+from django.contrib.postgres.aggregates import ArrayAgg
+from django.db.models import OuterRef, QuerySet
+from reporting.models.configuration_element import ConfigurationElement
+from reporting.models.report_methodology import ReportMethodology
+from reporting.models.report_version import ReportVersion
+from reporting.service.report_validation.report_validation_error import ReportValidationError
+from reporting.service.report_validation.report_validation_tags import ValidationTags
+
+TAGS = [ValidationTags.REPORT_VALIDATION, ValidationTags.ON_SUBMIT]
+
+
+def annotated_report_methodology(report_version_id: int) -> QuerySet[ReportMethodology]:
+    config_element = ConfigurationElement.objects.filter(
+        activity_id=OuterRef("report_emission__report_source_type__report_activity__activity_id"),
+        source_type_id=OuterRef("report_emission__report_source_type__source_type_id"),
+        gas_type_id=OuterRef("report_emission__gas_type_id"),
+        methodology_id=OuterRef("methodology_id"),
+        valid_from__valid_from__year__lte=OuterRef("report_version__report__reporting_year_id"),
+        valid_to__valid_to__year__gte=OuterRef("report_version__report__reporting_year_id"),
+    ).values("id")[:1]
+
+    return ReportMethodology.objects.filter(
+        report_version_id=report_version_id,
+    ).annotate(
+        matching_config_id=config_element,
+    )
+
+
+def validate_configuration_elements_present(report_version: ReportVersion) -> dict[str, ReportValidationError]:
+
+    missing_config = (
+        annotated_report_methodology(report_version.id).filter(matching_config_id__isnull=True).values("id")  # type: ignore
+    )
+
+    if missing_config.exists():
+        raise InternalSystemError(
+            f"Missing configuration elements for report methodology IDs: {str.join(', ', [str(m['id']) for m in missing_config])}"
+        )
+
+    return {}
+
+
+def validate_reporting_fields(report_version: ReportVersion) -> dict[str, ReportValidationError]:
+    """
+    Validate that for each ReportMethodology record, the reported fields are
+    - either listed in its configuration ConfigurationElementReportingField models
+    - or are a Unit field for an available reporting field (i.e. fieldSlug + "FieldUnits")
+    """
+    methodology_records = annotated_report_methodology(report_version.id).annotate(
+        allowed_slugs=(
+            ConfigurationElement.objects.filter(id=OuterRef("matching_config_id"))
+            .annotate(slugs=ArrayAgg("reporting_fields__slug"))
+            .values("slugs")
+        )
+    )
+
+    for record in methodology_records:
+        reported_slugs = {r.replace("FieldUnits", "") for r in record.json_data.keys()}
+        allowed_slugs = set(record.allowed_slugs)
+
+        if not reported_slugs.issubset(allowed_slugs):
+            raise InternalSystemError(
+                f"ReportMethodology ID {record.id} has reporting fields {reported_slugs - allowed_slugs} which are not in the allowed fields {allowed_slugs} of its matching configuration element."
+            )
+
+    return {}
+
+
+def validate(report_version: ReportVersion) -> dict[str, ReportValidationError]:
+    """
+    Validator ensuring that all the activity data reported has a matching configuration
+    defined for the reporting year of the report.
+
+    For each ReportMethodology record:
+    - Collect ReportMethodology -> ReportEmission (Gas Type) -> ReportSourceType -> ReportActivity -> Report (Reporting Year)
+    - Validate that there is a ConfigurationElement record matching that combination
+    - Validate that the extra reporting fields reported are in that configuration element
+    """
+
+    validate_configuration_elements_present(report_version)
+    validate_reporting_fields(report_version)
+
+    # This validator only raises system errors
+    # Errors found by this validator are not expected to be fixable by users, but rather indicate a bug or malicious manipulation of data inputs.
+    return {}

bc_obps/reporting/tests/service/report_validation/test_report_validation_service.py

@@ -47,6 +47,7 @@ def test_initializes_with_the_proper_plugins(self):
             "reporting.service.report_validation.validators.report_emission_allocation_other_excluded_category",
             "reporting.service.report_validation.validators.report_data_by_fuel_type_validator",
             "reporting.service.report_validation.validators.report_regulated_product_presence",
+            "reporting.service.report_validation.validators.reporting_configuration_validator",
             "reporting.service.report_validation.validators.required_fields.required_fields_report_operation_information",
             "reporting.service.report_validation.validators.required_fields.required_fields_report_person_responsible",
             "reporting.service.report_validation.validators.required_fields.required_fields_report_activity_data",

bc_obps/reporting/tests/service/report_validation/validators/test_reporting_configuration_validator.py

@@ -0,0 +1,141 @@
+from common.exceptions import InternalSystemError
+from model_bakery.baker import make, make_recipe, prepare_recipe
+import pytest
+from reporting.models.configuration_element import ConfigurationElement
+from reporting.service.report_validation.report_validation_tags import ValidationTags
+from reporting.service.report_validation.validators import reporting_configuration_validator
+
+
+@pytest.mark.django_db
+class TestReportingConfigurationValidator:
+
+    validator_under_test = reporting_configuration_validator
+
+    def setup_method(self):
+        self.report_version = make_recipe(
+            "reporting.tests.utils.report_version", report__reporting_year__reporting_year=2018
+        )
+        self.configuration = make_recipe(
+            "reporting.tests.utils.configuration", valid_from="2018-01-01", valid_to="2018-12-31"
+        )
+
+    def test_has_the_proper_configuration(self):
+        assert self.validator_under_test.TAGS == [ValidationTags.REPORT_VALIDATION, ValidationTags.ON_SUBMIT]
+
+    def test_raises_if_no_configuration_element_present(self):
+        report_methodology = make_recipe(
+            "reporting.tests.utils.report_methodology",
+            report_version=self.report_version,
+        )
+
+        with pytest.raises(
+            InternalSystemError,
+            match=f"Missing configuration elements for report methodology IDs: {report_methodology.id}",
+        ):
+            self.validator_under_test.validate(self.report_version)
+
+    def test_raises_if_configuration_doesnt_match_reporting_year(self):
+        report_methodology = make_recipe(
+            "reporting.tests.utils.report_methodology",
+            report_version=self.report_version,
+        )
+
+        wrong_year_confguration = make_recipe(
+            "reporting.tests.utils.configuration", valid_from="2011-01-01", valid_to="2012-12-31"
+        )
+        make(
+            ConfigurationElement,
+            activity=report_methodology.report_emission.report_source_type.report_activity.activity,
+            source_type=report_methodology.report_emission.report_source_type.source_type,
+            gas_type=report_methodology.report_emission.gas_type,
+            methodology=report_methodology.methodology,
+            valid_from=wrong_year_confguration,
+            valid_to=wrong_year_confguration,
+        )
+
+        with pytest.raises(
+            InternalSystemError,
+            match=f"Missing configuration elements for report methodology IDs: {report_methodology.id}",
+        ):
+            self.validator_under_test.validate(self.report_version)
+
+    def test_raises_if_configuration_doesnt_match_activity(self):
+        report_methodology = make_recipe(
+            "reporting.tests.utils.report_methodology",
+            report_version=self.report_version,
+        )
+        wrong_activity = make_recipe("reporting.tests.utils.activity")
+
+        make(
+            ConfigurationElement,
+            activity=wrong_activity,
+            source_type=report_methodology.report_emission.report_source_type.source_type,
+            gas_type=report_methodology.report_emission.gas_type,
+            methodology=report_methodology.methodology,
+            valid_from=self.configuration,
+            valid_to=self.configuration,
+        )
+
+        with pytest.raises(
+            InternalSystemError,
+            match=f"Missing configuration elements for report methodology IDs: {report_methodology.id}",
+        ):
+            self.validator_under_test.validate(self.report_version)
+
+    def test_raises_if_reported_fields_not_allowed_in_configuration_element(self):
+        report_methodology = make_recipe(
+            "reporting.tests.utils.report_methodology",
+            report_version=self.report_version,
+            json_data={"allowedField": "test data", "unexpectedField": 123},
+        )
+
+        make(
+            ConfigurationElement,
+            activity=report_methodology.report_emission.report_source_type.report_activity.activity,
+            source_type=report_methodology.report_emission.report_source_type.source_type,
+            gas_type=report_methodology.report_emission.gas_type,
+            methodology=report_methodology.methodology,
+            valid_from=self.configuration,
+            valid_to=self.configuration,
+            reporting_fields=[prepare_recipe("reporting.tests.utils.reporting_field", slug="allowedField")],
+        )
+
+        with pytest.raises(
+            InternalSystemError,
+            match=f"ReportMethodology ID {report_methodology.id} has reporting fields"
+            " {'unexpectedField'} which are not in the allowed fields {'allowedField'} of its matching configuration element.",
+        ):
+            self.validator_under_test.validate(self.report_version)
+
+    def test_passes_if_no_report_methodlogy_for_that_report_version(self):
+        assert self.validator_under_test.validate(self.report_version) == {}
+
+    def test_passes_with_fieldunits_field_reported(self):
+        report_methodology = make_recipe(
+            "reporting.tests.utils.report_methodology",
+            report_version=self.report_version,
+            json_data={
+                "allowedField": "test data",
+                "allowedFieldFieldUnits": "test units",
+                "anotherAllowedReportingField": 111,
+            },
+        )
+
+        allowed_fields = [
+            prepare_recipe("reporting.tests.utils.reporting_field", slug="allowedField"),
+            prepare_recipe("reporting.tests.utils.reporting_field", slug="anotherAllowedReportingField"),
+            prepare_recipe("reporting.tests.utils.reporting_field", slug="unreportedField"),
+        ]
+
+        make(
+            ConfigurationElement,
+            activity=report_methodology.report_emission.report_source_type.report_activity.activity,
+            source_type=report_methodology.report_emission.report_source_type.source_type,
+            gas_type=report_methodology.report_emission.gas_type,
+            methodology=report_methodology.methodology,
+            valid_from=self.configuration,
+            valid_to=self.configuration,
+            reporting_fields=allowed_fields,
+        )
+
+        assert self.validator_under_test.validate(self.report_version) == {}

bc_obps/reporting/tests/utils/baker_recipes.py

@@ -38,6 +38,7 @@
 from reporting.models.report_raw_activity_data import ReportRawActivityData
 from reporting.models.report_source_type import ReportSourceType
 from reporting.models.report_unit import ReportUnit
+from reporting.models.reporting_field import ReportingField
 from reporting.models.reporting_year import ReportingYear
 from reporting.models.report import Report
 from reporting.models.report_version import ReportVersion
@@ -106,6 +107,7 @@ def json_seq(
     valid_from=seq(date(4001, 1, 1), increment_by=timedelta(days=8)),
     valid_to=seq(date(4001, 1, 7), increment_by=timedelta(days=8)),
 )
+reporting_field = Recipe(ReportingField)
 
 activity = Recipe(Activity)
 source_type = Recipe(SourceType)