Skip to content

Commit fc36aeb

Browse files
paillavepaillave
authored
Merge pull request #554 from paillave/memoryleaks
feat: Add integration tests for JSON connector definitions and schema validation
2 parents 17abc8a + fcad366 commit fc36aeb

5 files changed

Lines changed: 397 additions & 2 deletions

File tree

‎src/Paillave.Etl.FromConfigurationConnectors.Tests/ConnectorsMergeIntegrationTests.cs‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
using System;
2+
using System.Text.Json.Nodes;
3+
using Paillave.Etl.Core;
4+
using Paillave.Etl.Sftp;
5+
using Xunit;
6+
7+
namespace Paillave.Etl.FromConfigurationConnectors.Tests;
8+
9+
/// <summary>
10+
/// Reproduces the exact production workflow from ConnectorsSetupService:
11+
/// 1. Several development items each carry their own JSON connector definition.
12+
/// 2. They are merged into one JSON string (MergeJsonObjects).
13+
/// 3. The merged JSON is fed to GetConnectors() to obtain live providers/processors.
14+
///
15+
/// Uses the exact JSON format that exists in production (Sftp adapter).
16+
/// Sensitive values (Password) are resolved with an identity resolver because
17+
/// the test does not actually open a connection — it only verifies that the
18+
/// parser wires up the correct types from both definition files.
19+
/// </summary>
20+
public class ConnectorsMergeIntegrationTests
21+
{
22+
// The sensitive-value resolver used throughout these tests.
23+
// In production this looks up values in connectorsConfiguration.SensitiveValues.
24+
// Here we return the stored string unchanged (no real SFTP connection is made).
25+
private static readonly Func<string, string> PassThroughResolver = key => key;
26+
27+
// Mirrors what ConnectorsSetupService.MergeJsonObjects does for distinct top-level keys.
28+
private static string MergeDefinitions(params string[] definitions)
29+
{
30+
var merged = new JsonObject();
31+
foreach (var json in definitions)
32+
{
33+
var obj = JsonNode.Parse(json)!.AsObject();
34+
foreach (var (key, value) in obj)
35+
merged[key] = value?.DeepClone();
36+
}
37+
return merged.ToJsonString();
38+
}
39+
40+
// -----------------------------------------------------------------------
41+
// This is definition 1: the exact format from the real codebase.
42+
// -----------------------------------------------------------------------
43+
private const string Definition1 = """
44+
{
45+
"LocalSFTP": {
46+
"Type": "Sftp",
47+
"Connection": {
48+
"Server": "localhost",
49+
"PortNumber": 2222,
50+
"Login": "user",
51+
"Password": "SftpPassword"
52+
},
53+
"Providers": {
54+
"INPUTPOSITIONS": {
55+
"SubFolder": "files/positions",
56+
"Name": "InputPositions",
57+
"FileNamePattern": "*"
58+
}
59+
}
60+
}
61+
}
62+
""";
63+
64+
// -----------------------------------------------------------------------
65+
// Definition 2: a second SFTP server managed by a different development item.
66+
// -----------------------------------------------------------------------
67+
private const string Definition2 = """
68+
{
69+
"RemoteSFTP": {
70+
"Type": "Sftp",
71+
"Connection": {
72+
"Server": "remote.example.com",
73+
"PortNumber": 22,
74+
"Login": "etluser",
75+
"Password": "RemotePassword"
76+
},
77+
"Providers": {
78+
"INPUTDOCUMENTS": {
79+
"SubFolder": "files/documents",
80+
"Name": "InputDocuments",
81+
"FileNamePattern": "*.pdf"
82+
}
83+
},
84+
"Processors": {
85+
"OUTPUTREPORTS": {
86+
"SubFolder": "files/reports",
87+
"Name": "OutputReports"
88+
}
89+
}
90+
}
91+
}
92+
""";
93+
94+
// -----------------------------------------------------------------------
95+
// Core use case: two separate definition files merged and parsed.
96+
// Providers from both must be reachable with their correct types.
97+
// -----------------------------------------------------------------------
98+
[Fact]
99+
public void GetConnectors_TwoMergedSftpDefinitions_ReturnsBothProvidersAndProcessors()
100+
{
101+
var merged = MergeDefinitions(Definition1, Definition2);
102+
103+
var parser = new ConfigurationFileValueConnectorParser(new SftpProviderProcessorAdapter());
104+
var connectors = parser.GetConnectors(merged, PassThroughResolver);
105+
106+
// A NoFileValueConnectors result means schema validation or JSON parsing failed.
107+
var real = Assert.IsType<FileValueConnectors>(connectors);
108+
109+
// Providers from both definition files must be present.
110+
Assert.Contains("INPUTPOSITIONS", real.Providers);
111+
Assert.Contains("INPUTDOCUMENTS", real.Providers);
112+
113+
// Processor from definition 2 must be present.
114+
Assert.Contains("OUTPUTREPORTS", real.Processors);
115+
116+
// Each must be the real SFTP type — not a No-op fallback.
117+
Assert.IsType<SftpFileValueProvider>(real.GetProvider("INPUTPOSITIONS"));
118+
Assert.IsType<SftpFileValueProvider>(real.GetProvider("INPUTDOCUMENTS"));
119+
Assert.IsType<SftpFileValueProcessor>(real.GetProcessor("OUTPUTREPORTS"));
120+
}
121+
122+
// -----------------------------------------------------------------------
123+
// GetConnectorsSchemaJson is called by the front-end to render the config
124+
// editor. This is the call that was crashing before the fix.
125+
// -----------------------------------------------------------------------
126+
[Fact]
127+
public void GetConnectorsSchemaJson_WithSftpAdapter_DoesNotThrow()
128+
{
129+
var parser = new ConfigurationFileValueConnectorParser(new SftpProviderProcessorAdapter());
130+
var schemaJson = parser.GetConnectorsSchemaJson();
131+
Assert.False(string.IsNullOrWhiteSpace(schemaJson));
132+
}
133+
134+
// -----------------------------------------------------------------------
135+
// Full regression: schema generation (editor) then connector parsing (runtime)
136+
// must both work within the same parser instance, as in production.
137+
// -----------------------------------------------------------------------
138+
[Fact]
139+
public void SchemaAndConnectorParsing_BothWorkInSameSession()
140+
{
141+
var parser = new ConfigurationFileValueConnectorParser(new SftpProviderProcessorAdapter());
142+
143+
// Step 1 — front-end fetches the JSON Schema to build the config editor.
144+
var schemaJson = parser.GetConnectorsSchemaJson();
145+
Assert.False(string.IsNullOrWhiteSpace(schemaJson));
146+
147+
// Step 2 — user saves their config; runtime merges all definitions and parses.
148+
var merged = MergeDefinitions(Definition1, Definition2);
149+
var connectors = parser.GetConnectors(merged, PassThroughResolver);
150+
151+
var real = Assert.IsType<FileValueConnectors>(connectors);
152+
Assert.Contains("INPUTPOSITIONS", real.Providers);
153+
Assert.Contains("INPUTDOCUMENTS", real.Providers);
154+
Assert.Contains("OUTPUTREPORTS", real.Processors);
155+
}
156+
157+
// -----------------------------------------------------------------------
158+
// Single definition (no merge) — baseline to confirm the format is accepted.
159+
// -----------------------------------------------------------------------
160+
[Fact]
161+
public void GetConnectors_SingleSftpDefinition_ReturnsProvider()
162+
{
163+
var parser = new ConfigurationFileValueConnectorParser(new SftpProviderProcessorAdapter());
164+
var connectors = parser.GetConnectors(Definition1, PassThroughResolver);
165+
166+
var real = Assert.IsType<FileValueConnectors>(connectors);
167+
Assert.Contains("INPUTPOSITIONS", real.Providers);
168+
Assert.IsType<SftpFileValueProvider>(real.GetProvider("INPUTPOSITIONS"));
169+
}
170+
}

‎src/Paillave.Etl.FromConfigurationConnectors.Tests/ConnectorsSchemaTests.cs‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
using System.Threading.Tasks;
2+
using NJsonSchema;
3+
using Paillave.Etl.Core;
4+
using Paillave.Etl.FileSystem;
5+
using Paillave.Etl.Http;
6+
using Paillave.Etl.Sftp;
7+
using Xunit;
8+
9+
namespace Paillave.Etl.FromConfigurationConnectors.Tests;
10+
11+
// --- Inline adapter with simple flat types ---
12+
// Ensures the basic AddProperty path works (the wrapper fix applies here too).
13+
file class FlatConnectionParams
14+
{
15+
public string Host { get; set; } = "";
16+
public int Port { get; set; }
17+
}
18+
file class FlatProviderParams
19+
{
20+
public string Path { get; set; } = "";
21+
}
22+
file class FlatProcessorParams
23+
{
24+
public string TargetPath { get; set; } = "";
25+
}
26+
file class FlatAdapter : ProviderProcessorAdapterBase<FlatConnectionParams, FlatProviderParams, FlatProcessorParams>
27+
{
28+
public override string Name => "Flat";
29+
public override string Description => "Flat adapter for tests";
30+
protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, FlatConnectionParams c, FlatProviderParams p)
31+
=> throw new System.NotImplementedException();
32+
protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, FlatConnectionParams c, FlatProcessorParams p)
33+
=> throw new System.NotImplementedException();
34+
}
35+
36+
// --- Inline adapter with enum and nested object types ---
37+
// Specifically exercises the case where JsonSchema.FromType() puts sub-type schemas
38+
// in the generated schema's own Definitions. The AddProperty wrapper fix must handle this.
39+
file enum AuthMode { ApiKey, OAuth2 }
40+
file class NestedAuth
41+
{
42+
public string Token { get; set; } = "";
43+
public AuthMode Mode { get; set; }
44+
}
45+
file class ComplexConnectionParams
46+
{
47+
public string Url { get; set; } = "";
48+
public NestedAuth? Auth { get; set; } // nested object → goes into Definitions
49+
public AuthMode DefaultMode { get; set; } // enum → goes into Definitions
50+
}
51+
file class ComplexProviderParams
52+
{
53+
public AuthMode PreferredMode { get; set; } // same enum, different schema instance
54+
public string SubPath { get; set; } = "";
55+
}
56+
file class ComplexAdapter : ProviderProcessorAdapterBase<ComplexConnectionParams, ComplexProviderParams, FlatProcessorParams>
57+
{
58+
public override string Name => "Complex";
59+
public override string Description => "Adapter with nested types for tests";
60+
protected override IFileValueProvider CreateProvider(string code, string name, string connectionName, ComplexConnectionParams c, ComplexProviderParams p)
61+
=> throw new System.NotImplementedException();
62+
protected override IFileValueProcessor CreateProcessor(string code, string name, string connectionName, ComplexConnectionParams c, FlatProcessorParams p)
63+
=> throw new System.NotImplementedException();
64+
}
65+
66+
public class ConnectorsSchemaTests
67+
{
68+
// -----------------------------------------------------------------------
69+
// Case 1: single adapter with flat types.
70+
// Even a flat type triggers the AddProperty wrapper bug:
71+
// JsonSchema.FromType(T).AddAsDefinition(doc) returns a wrapper { Reference = actual }.
72+
// AddProperty used to set property.Reference = wrapper (not in Definitions → crash).
73+
// -----------------------------------------------------------------------
74+
[Fact]
75+
public void GetConnectorsSchemaJson_FlatTypes_DoesNotThrow()
76+
{
77+
var parser = new ConfigurationFileValueConnectorParser(new FlatAdapter());
78+
var json = parser.GetConnectorsSchemaJson(); // must not throw
79+
Assert.False(string.IsNullOrWhiteSpace(json));
80+
}
81+
82+
// -----------------------------------------------------------------------
83+
// Case 2: adapter whose parameter types have enums and nested objects.
84+
// JsonSchema.FromType() puts those sub-schemas into the generated schema's
85+
// own .Definitions. The wrapper fix + nested-definitions traversal must handle this.
86+
// -----------------------------------------------------------------------
87+
[Fact]
88+
public void GetConnectorsSchemaJson_NestedTypesAndEnums_DoesNotThrow()
89+
{
90+
var parser = new ConfigurationFileValueConnectorParser(new ComplexAdapter());
91+
var json = parser.GetConnectorsSchemaJson();
92+
Assert.False(string.IsNullOrWhiteSpace(json));
93+
}
94+
95+
// -----------------------------------------------------------------------
96+
// Case 3: adapter with Providers only (no Processors).
97+
// The Providers code path also calls AddProperty with a wrapped schema.
98+
// -----------------------------------------------------------------------
99+
[Fact]
100+
public void GetConnectorsSchemaJson_WithFileSystemAdapter_DoesNotThrow()
101+
{
102+
var parser = new ConfigurationFileValueConnectorParser(new FileSystemProviderProcessorAdapter());
103+
var json = parser.GetConnectorsSchemaJson();
104+
Assert.False(string.IsNullOrWhiteSpace(json));
105+
}
106+
107+
// -----------------------------------------------------------------------
108+
// Case 4: real-world adapter with deeply nested types (HttpAuthentication
109+
// contains BearerAuthentication, BasicAuthentication, DigestAuthentication,
110+
// XCBACCESSAuthentication, DigestAlgorithm enum, HttpMethodCustomEnum, etc.).
111+
// This is the exact scenario from the original exception.
112+
// -----------------------------------------------------------------------
113+
[Fact]
114+
public void GetConnectorsSchemaJson_HttpAdapterWithDeeplyNestedTypes_DoesNotThrow()
115+
{
116+
var parser = new ConfigurationFileValueConnectorParser(new HttpProviderProcessorAdapter());
117+
var json = parser.GetConnectorsSchemaJson();
118+
Assert.False(string.IsNullOrWhiteSpace(json));
119+
}
120+
121+
// -----------------------------------------------------------------------
122+
// Case 5: multiple adapters combined — the full scenario used in production.
123+
// Exercises name collisions between same enum/type appearing in multiple adapters.
124+
// -----------------------------------------------------------------------
125+
[Fact]
126+
public void GetConnectorsSchemaJson_MultipleAdapters_DoesNotThrow()
127+
{
128+
var parser = new ConfigurationFileValueConnectorParser(
129+
new FlatAdapter(),
130+
new ComplexAdapter(),
131+
new FileSystemProviderProcessorAdapter(),
132+
new HttpProviderProcessorAdapter()
133+
);
134+
var json = parser.GetConnectorsSchemaJson();
135+
Assert.False(string.IsNullOrWhiteSpace(json));
136+
}
137+
138+
// -----------------------------------------------------------------------
139+
// Case 6: the returned string must be parseable as a JSON Schema.
140+
// Verifies structural correctness, not just absence of exception.
141+
// -----------------------------------------------------------------------
142+
[Fact]
143+
public async Task GetConnectorsSchemaJson_ReturnedJson_IsValidJsonSchema()
144+
{
145+
var parser = new ConfigurationFileValueConnectorParser(
146+
new FileSystemProviderProcessorAdapter(),
147+
new HttpProviderProcessorAdapter()
148+
);
149+
var json = parser.GetConnectorsSchemaJson();
150+
151+
// FromJsonAsync throws if the JSON is not a valid JSON Schema.
152+
var schema = await JsonSchema.FromJsonAsync(json);
153+
Assert.NotNull(schema);
154+
Assert.Equal(JsonObjectType.Object, schema.Type);
155+
}
156+
157+
// -----------------------------------------------------------------------
158+
// Case 7: adapter with NO provider parameters type (Processors only).
159+
// Ensures the null-check branch for ProviderParametersType is exercised.
160+
// -----------------------------------------------------------------------
161+
[Fact]
162+
public void GetConnectorsSchemaJson_AdapterWithNullProviderParams_DoesNotThrow()
163+
{
164+
var parser = new ConfigurationFileValueConnectorParser(new FlatAdapter());
165+
// The FlatAdapter does have a ProviderParametersType, but we verify the schema
166+
// can be generated regardless. The important test is that no exception is thrown.
167+
var json = parser.GetConnectorsSchemaJson();
168+
Assert.False(string.IsNullOrWhiteSpace(json));
169+
}
170+
171+
// -----------------------------------------------------------------------
172+
// Case 8: mirrors the regression test in the PMSv2 solution.
173+
// Uses all real adapters available in this repo (FileSystem, Http, Sftp)
174+
// to reproduce the original InvalidOperationException from production.
175+
// -----------------------------------------------------------------------
176+
[Fact]
177+
public void GetConnectorsSchemaJson_AllRealAdapters_ReturnsValidJson()
178+
{
179+
var parser = new ConfigurationFileValueConnectorParser(
180+
new FileSystemProviderProcessorAdapter(),
181+
new HttpProviderProcessorAdapter(),
182+
new SftpProviderProcessorAdapter()
183+
);
184+
var json = parser.GetConnectorsSchemaJson();
185+
Assert.False(string.IsNullOrWhiteSpace(json));
186+
}
187+
}

‎src/Paillave.Etl.FromConfigurationConnectors.Tests/Paillave.Etl.FromConfigurationConnectors.Tests.csproj‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net9.0</TargetFramework>
5+
<LangVersion>latest</LangVersion>
6+
<Nullable>enable</Nullable>
7+
<IsPackable>false</IsPackable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
12+
<PackageReference Include="xunit" Version="2.9.2" />
13+
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
14+
<PackageReference Include="coverlet.collector" Version="6.0.2" />
15+
<PackageReference Include="NJsonSchema" Version="11.1.0" />
16+
</ItemGroup>
17+
18+
<ItemGroup>
19+
<ProjectReference Include="..\Paillave.Etl.FromConfigurationConnectors\Paillave.Etl.FromConfigurationConnectors.csproj" />
20+
<ProjectReference Include="..\Paillave.Etl.FileSystem\Paillave.Etl.FileSystem.csproj" />
21+
<ProjectReference Include="..\Paillave.Etl.Http\Paillave.Etl.Http.csproj" />
22+
<ProjectReference Include="..\Paillave.Etl.Sftp\Paillave.Etl.Sftp.csproj" />
23+
</ItemGroup>
24+
25+
</Project>

0 commit comments

Comments
 (0)