Merge pull request #2595 from flow-php/type-collection-casting-bug

fix(flow-php/types): structure/list/map casting no longer fabricates missing data

Author: norberttech

documentation/components/libs/types.md

@@ -96,6 +96,15 @@ $variable = $input->get('some-input');
 $string = type_string()->cast($variable); 
 ```
 
+Casting structures, lists and maps:
+
+- a structure element that is absent, or present with `null`, throws `CastingException` when the element's type does
+  not accept `null` - `getPrevious()` returns a `MissingElementCastingException` whose `element` property names the
+  failing element
+- elements whose type accepts `null` (`type_optional(...)`, `type_union(..., type_null())`) cast `null` to `null`;
+  an absent optional element stays absent from the output
+- `type_list(...)->cast(null)` and `type_map(...)->cast(null)` throw `CastingException`
+- a JSON string payload is decoded and cast element-wise, exactly like an array payload
 
 ### Complex Types 
 

documentation/upgrading.md

@@ -45,6 +45,18 @@ Columns that may only become null in later batches, declare the schema explicitl
 | `detectType([[1.2], [4.0, 5]])` → `list<list<float>>`             | `list<array<mixed>>`  |
 | `detectType([[], [1, 2]])` → `list<array<mixed>>`                 | `list<list<integer>>` |
 
+### 5) `flow-php/types` - structure/list/map casting no longer fabricates missing data
+
+| Before                                                                                                                                 | After                     |
+|----------------------------------------------------------------------------------------------------------------------------------------|---------------------------|
+| `type_structure(['id' => type_integer(), 'name' => type_string()])->cast(['id' => 1])` → `['id' => 1, 'name' => '']`                   | throws `CastingException` |
+| same type, `->cast(['id' => 1, 'name' => null])` → `['id' => 1, 'name' => '']`                                                         | throws `CastingException` |
+| same type, `->cast([])`, `->cast(null)` → `['id' => 0, 'name' => '']`                                                                  | throws `CastingException` |
+| `type_structure(['id' => type_integer()], ['name' => type_string()])->cast(['id' => 1, 'name' => null])` → `['id' => 1, 'name' => '']` | throws `CastingException` |
+| `type_list(type_string())->cast(null)` → `['']`                                                                                        | throws `CastingException` |
+| `type_structure(['id' => type_integer()])->cast('{"id":"1"}')` → throws                                                                | `['id' => 1]`             |
+| `type_list(type_integer())->cast('["1","2"]')` → throws                                                                                | `[1, 2]`                  |
+
 ---
 
 ## Upgrading from 0.42.x to 0.43.x
@@ -2458,7 +2470,7 @@ After:
     ->run();
 ```
 
-### 4) ConfigBuilder::putInputIntoRows () output is now prefixed with _       (underscore)
+### 4) ConfigBuilder::putInputIntoRows () output is now prefixed with _        (underscore)
 
 In order to avoid collisions with datasets columns, additional columns created after using putInputIntoRows ()
 would now be prefixed with `_` (underscore) symbol.

src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/HTTP/Tests/Integration/PsrHttpClientPaginatedExtractorTest.php

@@ -8,6 +8,7 @@
 use Flow\ETL\Exception\RuntimeException;
 use Flow\ETL\Row\Entry\StructureEntry;
 use Flow\ETL\Tests\FlowTestCase;
+use Flow\Types\Exception\CastingException;
 use Http\Mock\Client;
 use Nyholm\Psr7\Factory\Psr17Factory;
 use Nyholm\Psr7\Response;
@@ -341,6 +342,24 @@ public function test_schema_typed_response_body(): void
         static::assertInstanceOf(StructureEntry::class, $rows[0]->first()->get('response_body'));
     }
 
+    public function test_schema_typed_response_body_with_missing_field(): void
+    {
+        $client = new Client(new Psr17Factory());
+        $client->addResponse(PaginationMother::jsonResponse(['login' => 'flow-php']));
+
+        $this->expectException(CastingException::class);
+
+        iterator_to_array(from_http_paginated(
+            $client,
+            PaginationMother::request('GET', 'https://api.example.com/orgs/flow-php'),
+            http_pagination_cursor('next', http_request_option_query('cursor')),
+            schema(structure_schema('response_body', type_structure([
+                'login' => type_string(),
+                'id' => type_integer(),
+            ]))),
+        )->extract(flow_context(config())));
+    }
+
     public function test_schema_typed_response_body_via_with_schema(): void
     {
         $client = new Client(new Psr17Factory());

src/adapter/etl-adapter-http/tests/Flow/ETL/Adapter/HTTP/Tests/Integration/PsrHttpClientStaticExtractorTest.php

@@ -7,6 +7,7 @@
 use Flow\ETL\Row\Entry\StructureEntry;
 use Flow\ETL\Rows;
 use Flow\ETL\Tests\FlowTestCase;
+use Flow\Types\Exception\CastingException;
 use Http\Mock\Client;
 use Nyholm\Psr7\Factory\Psr17Factory;
 use Nyholm\Psr7\Response;
@@ -93,6 +94,48 @@ public function test_http_extractor(): void
         static::assertSame('tomaszhanc', $tomekResponseBody['login']);
     }
 
+    public function test_schema_typed_response_body_with_empty_body(): void
+    {
+        $factory = new Psr17Factory();
+        $client = new Client($factory);
+        $client->addResponse(new Response(200, ['Content-Type' => 'application/json'], '{}'));
+
+        $this->expectException(CastingException::class);
+
+        from_static_http_requests(
+            $client,
+            [$factory->createRequest('GET', 'https://api.github.com/users/norberttech')],
+            schema(structure_schema('response_body', type_structure([
+                'login' => type_string(),
+                'id' => type_integer(),
+            ]))),
+        )
+            ->extract(flow_context(config()))
+            ->current();
+    }
+
+    public function test_schema_typed_response_body_with_missing_field(): void
+    {
+        $factory = new Psr17Factory();
+        $client = new Client($factory);
+        $client->addResponse(new Response(200, ['Content-Type' => 'application/json'], json_encode([
+            'login' => 'norberttech',
+        ], JSON_THROW_ON_ERROR)));
+
+        $this->expectException(CastingException::class);
+
+        from_static_http_requests(
+            $client,
+            [$factory->createRequest('GET', 'https://api.github.com/users/norberttech')],
+            schema(structure_schema('response_body', type_structure([
+                'login' => type_string(),
+                'id' => type_integer(),
+            ]))),
+        )
+            ->extract(flow_context(config()))
+            ->current();
+    }
+
     public function test_schema_typed_response_body(): void
     {
         $factory = new Psr17Factory();

src/core/etl/tests/Flow/ETL/Tests/Unit/Row/AdaptiveRowHydratorTest.php

@@ -7,6 +7,7 @@
 use Flow\ETL\Row\AdaptiveRowHydrator;
 use Flow\ETL\Row\RawRowValues;
 use Flow\ETL\Tests\FlowTestCase;
+use Flow\Types\Exception\CastingException;
 
 use function Flow\ETL\DSL\int_entry;
 use function Flow\ETL\DSL\int_schema;
@@ -15,6 +16,10 @@
 use function Flow\ETL\DSL\schema;
 use function Flow\ETL\DSL\str_entry;
 use function Flow\ETL\DSL\str_schema;
+use function Flow\ETL\DSL\structure_schema;
+use function Flow\Types\DSL\type_integer;
+use function Flow\Types\DSL\type_string;
+use function Flow\Types\DSL\type_structure;
 
 final class AdaptiveRowHydratorTest extends FlowTestCase
 {
@@ -45,4 +50,13 @@ public function test_cast_infers_rows_without_a_schema(): void
         static::assertSame(1, $rows->first()->valueOf('id'));
         static::assertSame('x', $rows->first()->valueOf('name'));
     }
+
+    public function test_cast_throws_on_missing_required_structure_element(): void
+    {
+        $this->expectException(CastingException::class);
+
+        (new AdaptiveRowHydrator())->cast([new RawRowValues(['data' => [
+            'id' => 1,
+        ]])], schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()]))));
+    }
 }

src/core/etl/tests/Flow/ETL/Tests/Unit/Row/NativeRowHydratorTest.php

@@ -319,6 +319,11 @@ public static function castable_datasets(): Generator
         ];
 
         yield 'empty cast batch' => [schema(int_schema('id')), []];
+
+        yield 'all-optional structure with no matching keys' => [
+            schema(structure_schema('st', type_structure([], ['b' => type_string()]))),
+            [new RawRowValues(['st' => ['other' => 1]])],
+        ];
     }
 
     /**
@@ -394,9 +399,19 @@ public static function throwing_cast_datasets(): Generator
             [new RawRowValues(['l' => [-3]])],
         ];
 
-        yield 'all-optional structure with no matching keys' => [
-            schema(structure_schema('st', type_structure([], ['b' => type_string()]))),
-            [new RawRowValues(['st' => ['other' => 1]])],
+        yield 'structure missing required element' => [
+            schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()]))),
+            [new RawRowValues(['data' => ['id' => 1]])],
+        ];
+
+        yield 'structure present-null required element' => [
+            schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()]))),
+            [new RawRowValues(['data' => ['id' => 1, 'name' => null]])],
+        ];
+
+        yield 'structure present-null optional element' => [
+            schema(structure_schema('data', type_structure(['id' => type_integer()], ['name' => type_string()]))),
+            [new RawRowValues(['data' => ['id' => 1, 'name' => null]])],
         ];
     }
 

src/core/etl/tests/Flow/ETL/Tests/Unit/Row/PhpRowHydratorTest.php

@@ -16,6 +16,7 @@
 use Flow\ETL\Row\RawRowValues;
 use Flow\ETL\Row\TypedRowValues;
 use Flow\ETL\Tests\FlowTestCase;
+use Flow\Types\Exception\CastingException;
 use Flow\Types\Value\Uuid;
 
 use function Flow\ETL\DSL\bool_schema;
@@ -51,6 +52,15 @@ public function test_absent_schema_column_is_filled_with_typed_null(): void
         static::assertSame(['id' => 1, 'name' => null], $rows->first()->toArray());
     }
 
+    public function test_cast_throws_on_missing_required_structure_element(): void
+    {
+        $this->expectException(CastingException::class);
+
+        (new PhpRowHydrator())->cast([new RawRowValues(['data' => [
+            'id' => 1,
+        ]])], schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()]))));
+    }
+
     public function test_casts_datetime_and_uuid_strings(): void
     {
         $rows = (new PhpRowHydrator())->cast(

src/extension/flow-php-ext/src/cast.rs

@@ -540,13 +540,19 @@ fn cast_value(kind: &CastKind, value: &Zval, ctx: &mut Ctx) -> Result<Option<Zva
             for element in elements {
                 let Some(item) = values.get(element.name.as_str()) else {
                     if element.required {
-                        // PHP feeds cast(null) to absent required elements
+                        // PHP throws MissingElementCastingException for absent required elements
                         return Ok(None);
                     }
 
                     continue;
                 };
 
+                if item.is_null() && !matches!(element.kind, CastKind::Optional(_)) {
+                    // PHP throws MissingElementCastingException for present-null elements
+                    // whose type rejects null - required and structure-level optional alike
+                    return Ok(None);
+                }
+
                 let Some(casted) = cast_value(&element.kind, item, ctx)? else {
                     return Ok(None);
                 };

src/extension/flow-php-ext/tests/phpt/026_row_hydrator_cast_parity.phpt

@@ -96,6 +96,10 @@ $datasets = [
             new RawRowValues(['id' => null], ['id' => Metadata::fromArray(['k' => 'v2'])]),
         ],
     ],
+    'all_optional_st' => [
+        schema(structure_schema('st', type_structure([], ['b' => type_string()]))),
+        [new RawRowValues(['st' => ['other' => 1]])],
+    ],
     'empty' => [schema(int_schema('id')), []],
 ];
 
@@ -135,6 +139,7 @@ uuid_json        cast:yes
 containers       cast:yes
 exotic_fallback  cast:yes
 fill_and_metadata cast:yes
+all_optional_st  cast:yes
 empty            cast:yes
 schema_mutation  cast:yes
 null_schema      cast:yes

src/extension/flow-php-ext/tests/phpt/027_cast_fallback_exceptions.phpt

@@ -10,8 +10,8 @@ use Flow\ETL\Row\NativeRowHydrator;
 use Flow\ETL\Row\PhpRowHydrator;
 use Flow\ETL\Row\RawRowValues;
 
-use function Flow\ETL\DSL\{schema, datetime_schema, date_schema, uuid_schema, json_schema, list_schema, map_schema, structure_schema};
-use function Flow\Types\DSL\{type_list, type_map, type_structure, type_integer, type_string, type_positive_integer};
+use function Flow\ETL\DSL\{schema, datetime_schema, date_schema, uuid_schema, json_schema, list_schema, map_schema};
+use function Flow\Types\DSL\{type_list, type_map, type_integer, type_string, type_positive_integer};
 
 $throwing = [
     'uuid invalid' => [schema(uuid_schema('u')), [new RawRowValues(['u' => 'not-a-uuid'])]],
@@ -38,10 +38,6 @@ $throwing = [
         schema(list_schema('l', type_list(type_positive_integer()))),
         [new RawRowValues(['l' => [-3]])],
     ],
-    'all-optional structure' => [
-        schema(structure_schema('st', type_structure([], ['b' => type_string()]))),
-        [new RawRowValues(['st' => ['other' => 1]])],
-    ],
 ];
 
 $php = new PhpRowHydrator();
@@ -86,4 +82,3 @@ string map int keys         exception:match aborted:yes
 list bad keys               exception:match aborted:yes
 positive int list string    exception:match aborted:yes
 positive int list negative  exception:match aborted:yes
-all-optional structure      exception:match aborted:yes