Add validation for flow_both (#1281)

* Add validation flow_both

* Fix existing flow_both data in a few places

* Fix minimum data in the docs

* Add a comment

* Update docs/src/10-tutorials/17-workflow.md

Co-authored-by: Abel Soares Siqueira <nepper271@gmail.com>

---------

Co-authored-by: Abel Soares Siqueira <nepper271@gmail.com>

Author: gnawin

benchmark/EU/flow-both.csv

@@ -417,14 +417,18 @@ DuckDB.query(
     connection,
     "CREATE TABLE flow_both AS
     SELECT
-        from_asset,
-        to_asset,
-        year AS milestone_year,
-        year AS commission_year,
-        initial_export_units,
-        initial_import_units,
+        t_flow_yearly.from_asset,
+        t_flow_yearly.to_asset,
+        t_flow_yearly.year AS milestone_year,
+        t_flow_yearly.year AS commission_year,
+        t_flow_yearly.initial_export_units,
+        t_flow_yearly.initial_import_units,
     FROM t_flow_yearly
-    ORDER by from_asset, to_asset
+    LEFT JOIN flow
+      ON flow.from_asset = t_flow_yearly.from_asset
+      AND flow.to_asset = t_flow_yearly.to_asset
+    WHERE flow.is_transport = TRUE -- flow_both must only contain transport flows
+    ORDER by t_flow_yearly.from_asset, t_flow_yearly.to_asset
     "
 )
 ```

docs/src/10-tutorials/17-workflow.md

@@ -87,10 +87,10 @@ data = Dict(
     # Basic flow data
     "flow" => DataFrame(:from_asset => ["some_producer"], :to_asset => ["some_consumer"]),
     "flow_both" => DataFrame(
-        :from_asset => ["some_producer"],
-        :to_asset => ["some_consumer"],
-        :commission_year => [2030],
-        :milestone_year => [2030],
+        :from_asset => String[],
+        :to_asset => String[],
+        :commission_year => Int[],
+        :milestone_year => Int[],
     ),
     "flow_commission" => DataFrame(
         :from_asset => ["some_producer"],

docs/src/20-user-guide/50-schemas.md

@@ -36,6 +36,11 @@ function validate_data!(connection)
             _validate_only_transport_flows_are_investable!,
             false,
         ),
+        (
+            "flow_both only contain transport flows",
+            _validate_flow_both_table_does_not_contain_non_transport_flows!,
+            false,
+        ),
         ("group consistency between tables", _validate_group_consistency!, false),
         (
             "data consistency for simple investment",
@@ -190,6 +195,30 @@ function _validate_only_transport_flows_are_investable!(connection)
     return error_messages
 end
 
+function _validate_flow_both_table_does_not_contain_non_transport_flows!(connection)
+    # In principle, we should also check all transport flows are covered.
+    # But that is tested elsewhere, i.e., in _validate_simple_method_data_consistency!()
+    error_messages = String[]
+
+    for row in DuckDB.query(
+        connection,
+        "SELECT flow_both.from_asset, flow_both.to_asset, flow_both.milestone_year, flow_both.commission_year
+        FROM flow_both
+        LEFT JOIN flow
+            ON flow.from_asset = flow_both.from_asset
+            AND flow.to_asset = flow_both.to_asset
+        WHERE flow.is_transport = FALSE
+        ",
+    )
+        push!(
+            error_messages,
+            "Unexpected (flow=('$(row.from_asset)', '$(row.to_asset)'), milestone_year=$(row.milestone_year), commission_year=$(row.commission_year)) in 'flow_both' because 'flow_both' should only contain transport flows.",
+        )
+    end
+
+    return error_messages
+end
+
 function _validate_foreign_key!(
     connection,
     table_name,

src/data-validation.jl

@@ -41,10 +41,10 @@ const simplest_data = Dict(
     # Basic flow data
     "flow" => DataFrame(:from_asset => ["some_producer"], :to_asset => ["some_consumer"]),
     "flow_both" => DataFrame(
-        :from_asset => ["some_producer"],
-        :to_asset => ["some_consumer"],
-        :commission_year => [2030],
-        :milestone_year => [2030],
+        :from_asset => String[],
+        :to_asset => String[],
+        :commission_year => Int[],
+        :milestone_year => Int[],
     ),
     "flow_commission" => DataFrame(
         :from_asset => ["some_producer"],

test/data-simplest.jl

@@ -146,6 +146,48 @@ end
     end
 end
 
+@testset "Check flow_both does not only contain non-transport flows" begin
+    @testset "Using fake data" begin
+        flow = DataFrame(
+            :from_asset => ["A1", "A2"],
+            :to_asset => ["B", "B"],
+            :is_transport => [false, true],
+        )
+        flow_both = DataFrame(
+            :from_asset => ["A1", "A2"],
+            :to_asset => ["B", "B"],
+            :milestone_year => [1, 2],
+            :commission_year => [1, 2],
+        )
+        connection = DBInterface.connect(DuckDB.DB)
+        DuckDB.register_data_frame(connection, flow, "flow")
+        DuckDB.register_data_frame(connection, flow_both, "flow_both")
+
+        error_messages =
+            TEM._validate_flow_both_table_does_not_contain_non_transport_flows!(connection)
+        @test error_messages == [
+            "Unexpected (flow=('A1', 'B'), milestone_year=1, commission_year=1) in 'flow_both' because 'flow_both' should only contain transport flows.",
+        ]
+    end
+
+    @testset "Using Multi-year data" begin
+        connection = _multi_year_fixture()
+        DuckDB.query(
+            connection,
+            """
+            INSERT INTO flow_both (from_asset, to_asset, milestone_year, commission_year)
+            VALUES ('wind', 'demand', 2030, 2030);
+            """,
+        )
+
+        error_messages =
+            TEM._validate_flow_both_table_does_not_contain_non_transport_flows!(connection)
+        @test error_messages == [
+            "Unexpected (flow=('wind', 'demand'), milestone_year=2030, commission_year=2030) in 'flow_both' because 'flow_both' should only contain transport flows.",
+        ]
+    end
+end
+
 @testset "Check that foreign keys are valid" begin
     @testset "Using fake data" begin
         # Main table