Skip to content

Commit d75bac6

Browse files
authored
Merge pull request #60 from thiagoluga/feat/dynamic-a3-sql-config-source
feat(sources): dynamic path A3 — SQL source from config
2 parents 827ff30 + a673970 commit d75bac6

12 files changed

Lines changed: 387 additions & 11 deletions

File tree

NeoReports.sln

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "03-async-job-hangfire", "sa
5757
EndProject
5858
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "04-dynamic-config-csv", "samples\04-dynamic-config-csv\04-dynamic-config-csv.csproj", "{90351642-8368-4814-A097-D7A1241A4C43}"
5959
EndProject
60+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "05-dynamic-config-sql", "samples\05-dynamic-config-sql\05-dynamic-config-sql.csproj", "{50A22538-2819-43C2-A70B-656DFA0A2759}"
61+
EndProject
6062
Global
6163
GlobalSection(SolutionConfigurationPlatforms) = preSolution
6264
Debug|Any CPU = Debug|Any CPU
@@ -343,6 +345,18 @@ Global
343345
{90351642-8368-4814-A097-D7A1241A4C43}.Release|x64.Build.0 = Release|Any CPU
344346
{90351642-8368-4814-A097-D7A1241A4C43}.Release|x86.ActiveCfg = Release|Any CPU
345347
{90351642-8368-4814-A097-D7A1241A4C43}.Release|x86.Build.0 = Release|Any CPU
348+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
349+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Debug|Any CPU.Build.0 = Debug|Any CPU
350+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Debug|x64.ActiveCfg = Debug|Any CPU
351+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Debug|x64.Build.0 = Debug|Any CPU
352+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Debug|x86.ActiveCfg = Debug|Any CPU
353+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Debug|x86.Build.0 = Debug|Any CPU
354+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Release|Any CPU.ActiveCfg = Release|Any CPU
355+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Release|Any CPU.Build.0 = Release|Any CPU
356+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Release|x64.ActiveCfg = Release|Any CPU
357+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Release|x64.Build.0 = Release|Any CPU
358+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Release|x86.ActiveCfg = Release|Any CPU
359+
{50A22538-2819-43C2-A70B-656DFA0A2759}.Release|x86.Build.0 = Release|Any CPU
346360
EndGlobalSection
347361
GlobalSection(SolutionProperties) = preSolution
348362
HideSolutionNode = FALSE
@@ -371,5 +385,6 @@ Global
371385
{9477CA22-21CE-453F-9DC9-B99D874B2CD0} = {22222222-2222-2222-2222-222222222222}
372386
{21CDAB18-7D2A-4926-B27C-B29F82C01ECD} = {44444444-4444-4444-4444-444444444444}
373387
{90351642-8368-4814-A097-D7A1241A4C43} = {44444444-4444-4444-4444-444444444444}
388+
{50A22538-2819-43C2-A70B-656DFA0A2759} = {44444444-4444-4444-4444-444444444444}
374389
EndGlobalSection
375390
EndGlobal

PLAN.md

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,9 +98,12 @@ destinations and jobs are untouched. See **D21**.
9898
`IWriterFactory`, `IDestinationFactory`). Filter is parsed but deferred to A4 (compiler
9999
rejects it explicitly). **Acceptance:** golden config → compiled, runnable report. ✅ 33
100100
green Core tests (+7). **Depends on:** A1.
101-
- [ ] **A3 — SQL source from config.** Keyset SQL source driven by config (connection
102-
name/string · sql · key · pageSize), materializing columns to `ReportRecord` by
103-
name/ordinal. **Acceptance:** Testcontainers E2E config→SQL→CSV. **Depends on:** A2, A3 reuses v1 keyset.
101+
- [x] **A3 — SQL source from config.** `SqlConfigSourceProvider` (`type: "sql"`) reads
102+
connectionString/sql/key/pageSize from the source properties and materializes
103+
`ReportRecord`s by schema-column name, reusing the v1 keyset engine (an additive internal
104+
materializer overload on `SqlKeysetSource<T>`); `AddSqlConfigSource()` DI helper.
105+
**Acceptance:** Testcontainers E2E config→SQL→CSV. ✅ 6 green SQL integration tests
106+
(4 typed + 2 dynamic). **Depends on:** A2, reuses v1 keyset.
104107
- [ ] **A4 — JsonLogic filter.** Compile a JsonLogic expression to
105108
`Func<ReportRecord,bool>` evaluated on the dynamic row. **Acceptance:** operator
106109
coverage + a filtered E2E. **Depends on:** A1.

samples/04-dynamic-config-csv/README.md

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,11 @@ What the JSON drives today:
1616
- the **columns / schema**: name, semantic type, header, format and culture;
1717
- the **outputs** and **destinations** selection by id (`"csv"`, `"local"`).
1818

19-
Standing in for not-yet-built pieces:
19+
This sample keeps an in-memory source so it runs with no database. The real **SQL** config source
20+
now exists (**A3**): register it with `services.AddSqlConfigSource()` and change the `source` section
21+
to `{ "type": "sql", "properties": { "connectionString": "...", "sql": "...", "key": "Id" } }` to read
22+
from SQL Server instead — nothing else in the config changes.
2023

21-
- the **SQL** config source arrives in **A3**, so an in-memory `IConfigSourceProvider`
22-
([`InMemorySalesSourceProvider`](InMemorySalesSourceProvider.cs)) supplies the rows;
23-
- binding format/destination **options** from config arrives later (**A5**), so the CSV and Local
24-
factories are pre-wired in DI — the JSON's `properties` under `outputs`/`destinations` are
25-
illustrative for now.
24+
Still standing in: binding format/destination **options** from config arrives later (**A5**), so the
25+
CSV and Local factories are pre-wired in DI — the JSON's `properties` under `outputs`/`destinations`
26+
are illustrative for now.
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<Nullable>enable</Nullable>
7+
<ImplicitUsings>enable</ImplicitUsings>
8+
<IsPackable>false</IsPackable>
9+
<GenerateDocumentationFile>false</GenerateDocumentationFile>
10+
</PropertyGroup>
11+
12+
<ItemGroup>
13+
<ProjectReference Include="..\..\src\NeoReports.Core\NeoReports.Core.csproj" />
14+
<ProjectReference Include="..\..\src\Sources\NeoReports.Sources.Sql\NeoReports.Sources.Sql.csproj" />
15+
<ProjectReference Include="..\..\src\Formats\NeoReports.Formats.Csv\NeoReports.Formats.Csv.csproj" />
16+
<ProjectReference Include="..\..\src\Destinations\NeoReports.Destinations.Local\NeoReports.Destinations.Local.csproj" />
17+
</ItemGroup>
18+
19+
<ItemGroup>
20+
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
21+
<PackageReference Include="Microsoft.Extensions.Logging" />
22+
<PackageReference Include="Microsoft.Extensions.Logging.Console" />
23+
</ItemGroup>
24+
25+
<ItemGroup>
26+
<None Update="report.json" CopyToOutputDirectory="PreserveNewest" />
27+
</ItemGroup>
28+
29+
</Project>
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
using System.Text.Json;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using Microsoft.Extensions.Logging;
4+
using NeoReports.Abstractions;
5+
using NeoReports.Core.Configuration;
6+
using NeoReports.Core.Pipeline;
7+
using NeoReports.Destinations.Local;
8+
using NeoReports.Formats.Csv;
9+
using NeoReports.Sources.Sql;
10+
11+
// Sample 05 — config-driven report reading from SQL Server (dynamic path + A3).
12+
//
13+
// The whole report is defined in report.json, with a "sql" source and no typed POCO. Point it at a
14+
// SQL Server that has a Sales(Id BIGINT, Customer NVARCHAR, Amount DECIMAL, Date DATETIME2) table:
15+
//
16+
// dotnet run --project samples/05-dynamic-config-sql -- "<connection-string>"
17+
//
18+
// Compared to sample 01 (the typed SQL report) this reads the exact same data, but the report shape
19+
// lives entirely in JSON. Swapping back to sample 04's in-memory source is just a config change.
20+
21+
var connectionString = args.Length > 0
22+
? args[0]
23+
: "Server=localhost;Database=Sales;Trusted_Connection=True;TrustServerCertificate=True";
24+
25+
// Load the config and inject the connection string (JSON-escaped) into the placeholder, so no
26+
// secret is committed to report.json. Path.Join never drops earlier segments.
27+
var configPath = Path.Join(AppContext.BaseDirectory, "report.json");
28+
var json = (await File.ReadAllTextAsync(configPath))
29+
.Replace("\"__CONNECTION_STRING__\"", JsonSerializer.Serialize(connectionString), StringComparison.Ordinal);
30+
31+
var config = new JsonReportConfigParser().Parse(json);
32+
33+
var services = new ServiceCollection();
34+
services.AddLogging(b => b.AddConsole().SetMinimumLevel(LogLevel.Information));
35+
services.AddSqlConfigSource(); // source "sql"
36+
services.AddSingleton<IWriterFactory>(new CsvWriterFactory(new CsvOptions())); // format "csv"
37+
services.AddSingleton<IDestinationFactory>(
38+
new LocalDestinationFactory("./out/{name}-{date:yyyy-MM-dd}.{ext}")); // dest "local"
39+
40+
await using var provider = services.BuildServiceProvider();
41+
42+
var report = ReportConfigCompiler.Compile(config, provider);
43+
44+
var logger = provider.GetRequiredService<ILoggerFactory>().CreateLogger("dynamic-sql");
45+
var exec = new ReportExecutionContext(
46+
Guid.NewGuid().ToString("N"), config.Name, parameters: null, logger, CancellationToken.None);
47+
48+
var result = await ReportRunner.ExecuteAsync(report, exec, provider, CancellationToken.None);
49+
50+
Console.WriteLine($"Report: {config.Name}");
51+
Console.WriteLine($"Status: {result.Status}");
52+
Console.WriteLine($"Records read/written: {result.Stats.RecordsRead}/{result.Stats.RecordsWritten}");
53+
foreach (var upload in result.Uploads)
54+
Console.WriteLine($"Uploaded: {upload.RemotePath} (success={upload.Success})");
55+
56+
return result.Status == ReportRunStatus.Failed ? 1 : 0;
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# 05 — dynamic config → SQL Server → CSV
2+
3+
The config-driven path (A2) reading from a real **SQL Server** source (A3). The report is defined
4+
entirely in [`report.json`](report.json) with a `"sql"` source — no typed POCO.
5+
6+
```bash
7+
dotnet run --project samples/05-dynamic-config-sql -- "Server=localhost;Database=Sales;Trusted_Connection=True;TrustServerCertificate=True"
8+
# writes ./out/monthly-sales-<date>.csv
9+
```
10+
11+
Expects a `Sales(Id BIGINT, Customer NVARCHAR, Amount DECIMAL, Date DATETIME2)` table (same schema
12+
as sample 01). The connection string passed on the command line is injected into the config's
13+
`__CONNECTION_STRING__` placeholder, so no secret lives in `report.json`.
14+
15+
How it works:
16+
17+
- `services.AddSqlConfigSource()` registers the `"sql"` source provider.
18+
- `JsonReportConfigParser` parses the document; `ReportConfigCompiler` compiles it into the same
19+
runnable report the fluent builder produces.
20+
- The SQL source materializes positional `ReportRecord`s by matching each schema column to the
21+
result-set column by name, reusing the v1 keyset engine (connection-per-page, opaque cursor).
22+
23+
This is sample 04 with one change: the `source` section is `"sql"` instead of `"inmemory"`.
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"name": "monthly-sales",
3+
"pageSize": 1000,
4+
"source": {
5+
"type": "sql",
6+
"properties": {
7+
"connectionString": "__CONNECTION_STRING__",
8+
"sql": "SELECT Id, Customer, Amount, Date FROM Sales WHERE (@cursor IS NULL OR Id > @cursor) ORDER BY Id",
9+
"key": "Id",
10+
"pageSize": 1000
11+
}
12+
},
13+
"columns": [
14+
{ "name": "Id", "type": "Integer", "displayName": "Sale ID", "nullable": false },
15+
{ "name": "Customer", "type": "String" },
16+
{ "name": "Amount", "type": "Decimal", "displayName": "Amount", "format": "C2", "culture": "pt-BR" },
17+
{ "name": "Date", "type": "DateTime", "displayName": "Sale Date", "format": "yyyy-MM-dd" }
18+
],
19+
"outputs": [ { "format": "csv" } ],
20+
"destinations": [
21+
{ "type": "local", "properties": { "path": "./out/{name}-{date:yyyy-MM-dd}.{ext}" } }
22+
]
23+
}
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
using Microsoft.Extensions.DependencyInjection;
2+
using Microsoft.Extensions.DependencyInjection.Extensions;
3+
using NeoReports.Abstractions;
4+
5+
namespace NeoReports.Sources.Sql;
6+
7+
/// <summary>DI helpers for the SQL source.</summary>
8+
public static class ServiceCollectionExtensions
9+
{
10+
/// <summary>
11+
/// Registers the config-driven SQL source provider (<c>type: "sql"</c>) so reports defined in
12+
/// configuration can read from SQL Server. Safe to call multiple times.
13+
/// </summary>
14+
/// <param name="services">The service collection.</param>
15+
public static IServiceCollection AddSqlConfigSource(this IServiceCollection services)
16+
{
17+
ArgumentNullException.ThrowIfNull(services);
18+
services.TryAddEnumerable(ServiceDescriptor.Singleton<IConfigSourceProvider, SqlConfigSourceProvider>());
19+
return services;
20+
}
21+
}
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
using System.Data.Common;
2+
using System.Globalization;
3+
using System.Text.Json;
4+
using NeoReports.Abstractions;
5+
6+
namespace NeoReports.Sources.Sql;
7+
8+
/// <summary>
9+
/// Config-driven SQL source for the dynamic path (<c>type: "sql"</c>). Reads its settings from the
10+
/// source <c>properties</c> (<c>connectionString</c>, <c>sql</c>, <c>key</c>, optional
11+
/// <c>pageSize</c>) and produces an <see cref="IBatchSource{T}"/> of positional
12+
/// <see cref="ReportRecord"/>s. Each row is materialized by reading the result-set column whose name
13+
/// matches each schema column (case-insensitive), reusing the v1 keyset paging engine.
14+
/// </summary>
15+
public sealed class SqlConfigSourceProvider : IConfigSourceProvider
16+
{
17+
/// <inheritdoc />
18+
public string Type => "sql";
19+
20+
/// <inheritdoc />
21+
public IBatchSource<ReportRecord> Create(SourceConfig source, ReportSchema schema, IServiceProvider services)
22+
{
23+
ArgumentNullException.ThrowIfNull(source);
24+
ArgumentNullException.ThrowIfNull(schema);
25+
26+
IReadOnlyDictionary<string, object?>? properties = source.Properties;
27+
string connectionString = RequireString(properties, "connectionString");
28+
string sql = RequireString(properties, "sql");
29+
string key = RequireString(properties, "key");
30+
int pageSize = OptionalInt(properties, "pageSize") ?? 1000;
31+
32+
return new SqlKeysetSource<ReportRecord>(
33+
connectionString, sql, key, pageSize, schema,
34+
parameters: null,
35+
materialize: (reader, ordinals) => Materialize(reader, ordinals, schema));
36+
}
37+
38+
private static ReportRecord Materialize(
39+
DbDataReader reader, IReadOnlyDictionary<string, int> ordinalByName, ReportSchema schema)
40+
{
41+
var values = new object?[schema.Count];
42+
for (var i = 0; i < schema.Count; i++)
43+
{
44+
values[i] = ordinalByName.TryGetValue(schema.Columns[i].Name, out int ordinal) && !reader.IsDBNull(ordinal)
45+
? reader.GetValue(ordinal)
46+
: null;
47+
}
48+
49+
return new ReportRecord(schema, values);
50+
}
51+
52+
private static string RequireString(IReadOnlyDictionary<string, object?>? properties, string key)
53+
{
54+
if (properties is not null
55+
&& properties.TryGetValue(key, out var value)
56+
&& value is string text
57+
&& !string.IsNullOrWhiteSpace(text))
58+
{
59+
return text;
60+
}
61+
62+
throw new ConfigurationException($"The SQL source requires a non-empty '{key}' property.");
63+
}
64+
65+
private static int? OptionalInt(IReadOnlyDictionary<string, object?>? properties, string key)
66+
{
67+
if (properties is null || !properties.TryGetValue(key, out var value) || value is null)
68+
return null;
69+
70+
return value switch
71+
{
72+
int i => i,
73+
long l => checked((int)l),
74+
double d => (int)d,
75+
string s when int.TryParse(s, NumberStyles.Integer, CultureInfo.InvariantCulture, out int parsed) => parsed,
76+
JsonElement { ValueKind: JsonValueKind.Number } e => e.GetInt32(),
77+
_ => throw new ConfigurationException(
78+
$"The SQL source property '{key}' must be an integer (was {value.GetType().Name})."),
79+
};
80+
}
81+
}

src/Sources/NeoReports.Sources.Sql/SqlKeysetSource.cs

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ public sealed class SqlKeysetSource<T> : IBatchSource<T>
2020
private readonly string _keyColumn;
2121
private readonly int _pageSize;
2222
private readonly IReadOnlyDictionary<string, object?> _parameters;
23-
private readonly RecordMaterializer<T> _materializer = new();
23+
private readonly Func<DbDataReader, IReadOnlyDictionary<string, int>, T> _materialize;
2424

2525
/// <summary>Creates the source.</summary>
2626
/// <param name="connectionString">SQL Server connection string.</param>
@@ -36,6 +36,23 @@ public SqlKeysetSource(
3636
int pageSize,
3737
ReportSchema schema,
3838
IReadOnlyDictionary<string, object?>? parameters = null)
39+
: this(connectionString, sql, keyColumn, pageSize, schema, parameters, materialize: null)
40+
{
41+
}
42+
43+
/// <summary>
44+
/// Creates the source with a custom row materializer. Used by the dynamic path to materialize a
45+
/// positional <c>ReportRecord</c> by schema name; when <paramref name="materialize"/> is null the
46+
/// reflection-based <see cref="RecordMaterializer{T}"/> (typed POCO) is used.
47+
/// </summary>
48+
internal SqlKeysetSource(
49+
string connectionString,
50+
string sql,
51+
string keyColumn,
52+
int pageSize,
53+
ReportSchema schema,
54+
IReadOnlyDictionary<string, object?>? parameters,
55+
Func<DbDataReader, IReadOnlyDictionary<string, int>, T>? materialize)
3956
{
4057
_connectionString = connectionString ?? throw new ArgumentNullException(nameof(connectionString));
4158
_sql = sql ?? throw new ArgumentNullException(nameof(sql));
@@ -44,6 +61,7 @@ public SqlKeysetSource(
4461
_pageSize = pageSize;
4562
Schema = schema ?? throw new ArgumentNullException(nameof(schema));
4663
_parameters = parameters ?? new Dictionary<string, object?>();
64+
_materialize = materialize ?? new RecordMaterializer<T>().Materialize;
4765
}
4866

4967
/// <inheritdoc />
@@ -86,7 +104,7 @@ public async Task<BatchResult<T>> ReadBatchAsync(BatchContext context, Cancellat
86104

87105
while (read < _pageSize && await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
88106
{
89-
records.Add(_materializer.Materialize(reader, ordinals));
107+
records.Add(_materialize(reader, ordinals));
90108
if (keyOrdinal >= 0 && !reader.IsDBNull(keyOrdinal))
91109
lastKey = Convert.ToString(reader.GetValue(keyOrdinal), CultureInfo.InvariantCulture);
92110
read++;

0 commit comments

Comments
 (0)