Skip to content

Commit 77844c9

Browse files
MrHDOLEKMrHDOLEK
authored
perf(flow-php/etl): native int/float fast-path in sum() with opt-in exact mode (#2522)
* perf(flow-php/etl): native int/float fast-path in sum() with opt-in exact mode Sum::aggregate() routed every addition through brick/math: a new Calculator instance per row, two string casts and two BigDecimal parses per addition. On a 1M-row aggregation that dominated the cost of the most common aggregating function. - sum() now accumulates through native int/float arithmetic and mirrors Calculator::add() by returning int whenever the running sum has no fractional part - sum(ref('x'), exact: true) opts back into arbitrary-precision decimal arithmetic - a shared Calculator instance lives in Config and is exposed through FlowContext::calculator(); Sum (exact mode) and Average pull it from the context instead of constructing one per row Sum::aggregate() micro-benchmark, 1M rows: int 3.57 -> 0.45 us/row (8x), float 5.07 -> 0.51 us/row (10x); exact mode keeps the previous characteristics. * refactor(flow-php/etl): remove explanatory comments from sum() fast-path * feat(flow-php/etl): allow per-row ScalarFunction as sum() exact flag The exact flag follows the ScalarFunction|bool parameter convention: it accepts a plain boolean, lit(...) or any scalar expression evaluated per row (e.g. a boolean column). Plain booleans keep the allocation-free fast path. * chore(flow-php/etl): regenerate dsl.json for the sum() exact parameter
1 parent 1b5c5f2 commit 77844c9

8 files changed

Lines changed: 156 additions & 9 deletions

File tree

‎src/core/etl/src/Flow/ETL/Config.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
namespace Flow\ETL;
66

7+
use Flow\Calculator\Calculator;
78
use Flow\ETL\Config\Cache\CacheConfig;
89
use Flow\ETL\Config\ConfigBuilder;
910
use Flow\ETL\Config\Sort\SortConfig;
@@ -36,6 +37,7 @@ public function __construct(
3637
public SortConfig $sort,
3738
private ?Analyze $analyze,
3839
public TelemetryConfig $telemetry,
40+
private Calculator $calculator = new Calculator(),
3941
) {}
4042

4143
public static function builder(): ConfigBuilder
@@ -53,6 +55,11 @@ public function analyze(): ?Analyze
5355
return $this->analyze;
5456
}
5557

58+
public function calculator(): Calculator
59+
{
60+
return $this->calculator;
61+
}
62+
5663
public function clock(): ClockInterface
5764
{
5865
return $this->clock;

‎src/core/etl/src/Flow/ETL/DSL/functions.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1824,9 +1824,9 @@ function window(): Window
18241824
}
18251825

18261826
#[DocumentationDSL(module: Module::CORE, type: DSLType::AGGREGATING_FUNCTION)]
1827-
function sum(EntryReference|string $ref): Sum
1827+
function sum(EntryReference|string $ref, ScalarFunction|bool $exact = false): Sum
18281828
{
1829-
return new Sum(is_string($ref) ? ref($ref) : $ref);
1829+
return new Sum(is_string($ref) ? ref($ref) : $ref, $exact);
18301830
}
18311831

18321832
#[DocumentationDSL(module: Module::CORE, type: DSLType::AGGREGATING_FUNCTION)]

‎src/core/etl/src/Flow/ETL/FlowContext.php‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
namespace Flow\ETL;
66

7+
use Flow\Calculator\Calculator;
78
use Flow\ETL\Config\Telemetry\TelemetryContext;
89
use Flow\ETL\ErrorHandler\ThrowError;
910
use Flow\ETL\Filesystem\FilesystemStreams;
@@ -37,6 +38,11 @@ public function cache(): Cache
3738
return $this->config->cache->cache;
3839
}
3940

41+
public function calculator(): Calculator
42+
{
43+
return $this->config->calculator();
44+
}
45+
4046
public function entryFactory(): EntryFactory
4147
{
4248
return $this->config->entryFactory();

‎src/core/etl/src/Flow/ETL/Function/Average.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public function aggregate(Row $row, FlowContext $context): void
4747

4848
if (is_numeric($value)) {
4949
// @mago-ignore analysis:possibly-invalid-argument
50-
$this->sum = (new Calculator())->add($this->sum, $value);
50+
$this->sum = $context->calculator()->add($this->sum, $value);
5151
$this->count++;
5252
}
5353
} catch (InvalidArgumentException $e) {
@@ -67,7 +67,7 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed
6767

6868
if (is_numeric($value)) {
6969
// @mago-ignore analysis:possibly-invalid-argument
70-
$sum = (new Calculator())->add($sum, $value);
70+
$sum = $context->calculator()->add($sum, $value);
7171
$count++;
7272
}
7373
} catch (InvalidArgumentException $e) {
@@ -79,7 +79,7 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed
7979
}
8080
}
8181

82-
return (new Calculator())->divide($sum, $count, $this->scale, $this->rounding);
82+
return $context->calculator()->divide($sum, $count, $this->scale, $this->rounding);
8383
}
8484

8585
public function over(Window $window): WindowFunction

‎src/core/etl/src/Flow/ETL/Function/Sum.php‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
namespace Flow\ETL\Function;
66

7-
use Flow\Calculator\Calculator;
87
use Flow\ETL\Exception\InvalidArgumentException;
98
use Flow\ETL\Exception\RuntimeException;
109
use Flow\ETL\FlowContext;
@@ -27,6 +26,7 @@ final class Sum implements AggregatingFunction, WindowFunction
2726

2827
public function __construct(
2928
private readonly Reference $ref,
29+
private readonly ScalarFunction|bool $exact = false,
3030
) {
3131
$this->sum = 0;
3232
$this->window = null;
@@ -38,7 +38,7 @@ public function aggregate(Row $row, FlowContext $context): void
3838
$value = $row->valueOf($this->ref);
3939

4040
if (is_int($value) || is_float($value) || is_string($value) && is_numeric($value)) {
41-
$this->sum = (new Calculator())->add($this->sum, $value);
41+
$this->sum = $this->add($this->sum, $value, $row, $context);
4242
}
4343
} catch (InvalidArgumentException $e) {
4444
$context->functions()->invalidResult(new InvalidArgumentException('Sum error: ' . $e->getMessage()));
@@ -54,7 +54,7 @@ public function apply(Row $row, Rows $partition, FlowContext $context): mixed
5454
$value = $partitionRow->valueOf($this->ref);
5555

5656
if (is_int($value) || is_float($value) || is_string($value) && is_numeric($value)) {
57-
$sum = (new Calculator())->add($sum, $value);
57+
$sum = $this->add($sum, $value, $partitionRow, $context);
5858
}
5959
} catch (InvalidArgumentException $e) {
6060
$context
@@ -104,4 +104,36 @@ public function window(): Window
104104

105105
return $this->window;
106106
}
107+
108+
/**
109+
* @param float|int|numeric-string $value
110+
*/
111+
private function add(float|int $sum, float|int|string $value, Row $row, FlowContext $context): float|int
112+
{
113+
if ($this->isExact($row, $context)) {
114+
return $context->calculator()->add($sum, $value);
115+
}
116+
117+
$result = $sum + $value;
118+
119+
if (
120+
is_float($result)
121+
&& floor($result) === $result
122+
&& $result >= (float) PHP_INT_MIN
123+
&& $result < (float) PHP_INT_MAX
124+
) {
125+
return (int) $result;
126+
}
127+
128+
return $result;
129+
}
130+
131+
private function isExact(Row $row, FlowContext $context): bool
132+
{
133+
if (is_bool($this->exact)) {
134+
return $this->exact;
135+
}
136+
137+
return (new Parameter($this->exact))->asBoolean($row, $context);
138+
}
107139
}

‎src/core/etl/tests/Flow/ETL/Tests/Unit/FlowContextTest.php‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Flow\ETL\Tests\Unit;
6+
7+
use Flow\Calculator\Calculator;
8+
use Flow\ETL\Tests\FlowTestCase;
9+
10+
use function Flow\ETL\DSL\config;
11+
use function Flow\ETL\DSL\flow_context;
12+
13+
final class FlowContextTest extends FlowTestCase
14+
{
15+
public function test_provides_shared_calculator_instance_from_config(): void
16+
{
17+
$context = flow_context(config());
18+
19+
static::assertInstanceOf(Calculator::class, $context->calculator());
20+
static::assertSame($context->calculator(), $context->calculator());
21+
}
22+
}

‎src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SumTest.php‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@
88
use Flow\ETL\Function\ExecutionMode;
99
use Flow\ETL\Tests\FlowTestCase;
1010

11+
use function Flow\ETL\DSL\bool_entry;
1112
use function Flow\ETL\DSL\config;
1213
use function Flow\ETL\DSL\float_entry;
1314
use function Flow\ETL\DSL\flow_context;
1415
use function Flow\ETL\DSL\int_entry;
16+
use function Flow\ETL\DSL\lit;
1517
use function Flow\ETL\DSL\ref;
1618
use function Flow\ETL\DSL\row;
1719
use function Flow\ETL\DSL\rows;
@@ -58,6 +60,48 @@ public function test_aggregation_sum_with_float_result(): void
5860
static::assertSame(360.25, $aggregator->result(flow_context(config())->entryFactory())->value());
5961
}
6062

63+
public function test_aggregation_sum_of_decimal_fractions(): void
64+
{
65+
$aggregator = sum(ref('value'));
66+
67+
$aggregator->aggregate(row(float_entry('value', 0.1)), flow_context());
68+
$aggregator->aggregate(row(float_entry('value', 0.2)), flow_context());
69+
70+
static::assertSame(0.3, $aggregator->result(flow_context(config())->entryFactory())->value());
71+
}
72+
73+
public function test_window_function_sum_of_decimal_fractions_uses_float_arithmetic_by_default(): void
74+
{
75+
$rows = rows(
76+
$row1 = row(int_entry('id', 1), float_entry('value', 0.1)),
77+
row(int_entry('id', 2), float_entry('value', 0.2)),
78+
);
79+
80+
$sum = sum(ref('value'))->over(window()->orderBy(ref('id')->desc()));
81+
82+
static::assertSame(0.1 + 0.2, $sum->apply($row1, $rows, flow_context()));
83+
}
84+
85+
public function test_aggregation_sum_of_floats_returns_int_when_sum_is_whole(): void
86+
{
87+
$aggregator = sum(ref('value'));
88+
89+
$aggregator->aggregate(row(float_entry('value', 2.5)), flow_context());
90+
$aggregator->aggregate(row(float_entry('value', 2.5)), flow_context());
91+
92+
static::assertSame(5, $aggregator->result(flow_context(config())->entryFactory())->value());
93+
}
94+
95+
public function test_exact_aggregation_sum_of_decimal_fractions(): void
96+
{
97+
$aggregator = sum(ref('value'), exact: true);
98+
99+
$aggregator->aggregate(row(float_entry('value', 0.1)), flow_context());
100+
$aggregator->aggregate(row(float_entry('value', 0.2)), flow_context());
101+
102+
static::assertSame(0.3, $aggregator->result(flow_context(config())->entryFactory())->value());
103+
}
104+
61105
public function test_window_function_sum_on_partitioned_rows(): void
62106
{
63107
$rows = rows(
@@ -73,6 +117,42 @@ public function test_window_function_sum_on_partitioned_rows(): void
73117
static::assertSame(15, $sum->apply($row1, $rows, flow_context()));
74118
}
75119

120+
public function test_window_function_sum_of_decimal_fractions_in_exact_mode(): void
121+
{
122+
$rows = rows(
123+
$row1 = row(int_entry('id', 1), float_entry('value', 0.1)),
124+
row(int_entry('id', 2), float_entry('value', 0.2)),
125+
);
126+
127+
$sum = sum(ref('value'), exact: true)->over(window()->orderBy(ref('id')->desc()));
128+
129+
static::assertSame(0.3, $sum->apply($row1, $rows, flow_context()));
130+
}
131+
132+
public function test_window_function_sum_with_exact_mode_from_column(): void
133+
{
134+
$rows = rows(
135+
$row1 = row(int_entry('id', 1), float_entry('value', 0.1), bool_entry('is_exact', true)),
136+
row(int_entry('id', 2), float_entry('value', 0.2), bool_entry('is_exact', true)),
137+
);
138+
139+
$sum = sum(ref('value'), exact: ref('is_exact'))->over(window()->orderBy(ref('id')->desc()));
140+
141+
static::assertSame(0.3, $sum->apply($row1, $rows, flow_context()));
142+
}
143+
144+
public function test_window_function_sum_with_exact_mode_from_literal(): void
145+
{
146+
$rows = rows(
147+
$row1 = row(int_entry('id', 1), float_entry('value', 0.1)),
148+
row(int_entry('id', 2), float_entry('value', 0.2)),
149+
);
150+
151+
$sum = sum(ref('value'), exact: lit(true))->over(window()->orderBy(ref('id')->desc()));
152+
153+
static::assertSame(0.3, $sum->apply($row1, $rows, flow_context()));
154+
}
155+
76156
public function test_window_function_sum_with_missing_reference_in_strict_mode(): void
77157
{
78158
$this->expectException(InvalidArgumentException::class);

‎web/landing/resources/dsl.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)