Skip to content

Commit 76932f3

Browse files
fix(csharp): expand SEA catalog wildcards in GetSchemas client-side (PECO-3035) [SEA]
JDBC metadata APIs (getSchemas(catalog, schemaPattern)) accept SQL LIKE-style patterns in the catalog argument (%, _, \_), and Thrift honors those patterns server-side. The SEA path wrapped the catalog in backticks as a literal identifier, so SHOW SCHEMAS IN `%` returned empty rather than expanding to all catalogs. This change adds client-side wildcard expansion in StatementExecutionConnection: - null or pure '%' / '*' -> SHOW SCHEMAS IN ALL CATALOGS - pattern with unescaped %/_ -> SHOW CATALOGS LIKE '<pat>' then per-catalog SHOW SCHEMAS IN `<cat>` aggregated together - literal name -> unchanged (single SHOW SCHEMAS IN `<catalog>`) Both the IGetObjectsDataProvider.GetSchemasAsync path (used by GetObjects) and StatementExecutionStatement.GetSchemasAsync (direct GetSchemas metadata command) now share a single ListSchemasAsync helper. Wildcard detection honors backslash escapes (\_ stays literal).
1 parent 2a552f9 commit 76932f3

4 files changed

Lines changed: 250 additions & 95 deletions

File tree

csharp/src/StatementExecution/MetadataCommands/MetadataCommandBase.cs

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,5 +99,52 @@ protected static void AppendCatalogScope(StringBuilder sql, string? catalog)
9999
else
100100
sql.Append(string.Format(InCatalogFormat, QuoteIdentifier(catalog)));
101101
}
102+
103+
/// <summary>
104+
/// Returns true when <paramref name="pattern"/> contains a SQL LIKE wildcard
105+
/// (% or _) that is NOT escaped by a preceding backslash. JDBC metadata APIs
106+
/// treat catalog/schema/table arguments as LIKE patterns, but SEA SHOW commands
107+
/// take literal identifiers, so callers must expand wildcards client-side.
108+
/// </summary>
109+
internal static bool ContainsUnescapedWildcard(string? pattern)
110+
{
111+
if (string.IsNullOrEmpty(pattern))
112+
return false;
113+
114+
bool escapeNext = false;
115+
for (int i = 0; i < pattern!.Length; i++)
116+
{
117+
char c = pattern[i];
118+
if (c == '\\')
119+
{
120+
// Two backslashes in a row are an escaped backslash literal, not an escape.
121+
if (i + 1 < pattern.Length && pattern[i + 1] == '\\')
122+
{
123+
i++;
124+
continue;
125+
}
126+
escapeNext = !escapeNext;
127+
}
128+
else if (escapeNext)
129+
{
130+
escapeNext = false;
131+
}
132+
else if (c == '%' || c == '_')
133+
{
134+
return true;
135+
}
136+
}
137+
return false;
138+
}
139+
140+
/// <summary>
141+
/// Returns true when <paramref name="pattern"/> is a pure "match anything"
142+
/// pattern: a single unescaped % (or *). These can be optimised to
143+
/// SHOW SCHEMAS IN ALL CATALOGS without enumerating catalogs.
144+
/// </summary>
145+
internal static bool IsMatchAnything(string? pattern)
146+
{
147+
return pattern == "%" || pattern == "*";
148+
}
102149
}
103150
}

csharp/src/StatementExecution/StatementExecutionConnection.cs

Lines changed: 122 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -594,53 +594,10 @@ async Task<IReadOnlyList<string>> IGetObjectsDataProvider.GetCatalogsAsync(strin
594594

595595
async Task<IReadOnlyList<(string catalog, string schema)>> IGetObjectsDataProvider.GetSchemasAsync(string? catalogPattern, string? schemaPattern, CancellationToken cancellationToken)
596596
{
597-
// Note: catalogPattern comes from GetObjectsResultBuilder which resolves individual
598-
// catalog names before calling this method. Despite the "pattern" name (from the
599-
// IGetObjectsDataProvider interface), the value passed to ShowSchemasCommand is used
600-
// as a literal catalog identifier (backtick-quoted), not a wildcard pattern.
601-
string sql = new ShowSchemasCommand(catalogPattern, schemaPattern).Build();
602-
603-
List<RecordBatch> batches;
604-
try
605-
{
606-
batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false);
607-
}
608-
catch (DatabricksException ex) when (ex.IsObjectNotFoundException())
609-
{
610-
return System.Array.Empty<(string, string)>();
611-
}
612-
613-
// SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog
614-
// SHOW SCHEMAS IN `catalog` returns 1 column: databaseName
615-
bool showSchemasInAllCatalogs = catalogPattern == null;
616-
617-
var result = new List<(string, string)>();
618-
foreach (var batch in batches)
619-
{
620-
StringArray? catalogArray = null;
621-
StringArray? schemaArray = null;
622-
623-
if (showSchemasInAllCatalogs)
624-
{
625-
schemaArray = batch.Column(0) as StringArray;
626-
catalogArray = batch.Column(1) as StringArray;
627-
}
628-
else
629-
{
630-
schemaArray = batch.Column(0) as StringArray;
631-
}
632-
633-
if (schemaArray == null) continue;
634-
for (int i = 0; i < batch.Length; i++)
635-
{
636-
if (schemaArray.IsNull(i)) continue;
637-
string catalog = catalogArray != null && !catalogArray.IsNull(i)
638-
? catalogArray.GetString(i)
639-
: catalogPattern ?? "";
640-
result.Add((catalog, schemaArray.GetString(i)));
641-
}
642-
}
643-
return result;
597+
// PECO-3035: catalogPattern follows JDBC LIKE semantics (% / _ / \_ ). The SEA
598+
// backend treats SHOW SCHEMAS IN `<catalog>` as a literal lookup, so we resolve
599+
// wildcards client-side here (see ListSchemasAsync for details).
600+
return await ListSchemasAsync(catalogPattern, schemaPattern, cancellationToken).ConfigureAwait(false);
644601
}
645602

646603
async Task<IReadOnlyList<(string catalog, string schema, string table, string tableType)>> IGetObjectsDataProvider.GetTablesAsync(
@@ -785,6 +742,124 @@ internal List<RecordBatch> ExecuteMetadataSql(string sql, CancellationToken canc
785742
return ExecuteMetadataSqlAsync(sql, cancellationToken).GetAwaiter().GetResult();
786743
}
787744

745+
/// <summary>
746+
/// Executes SHOW SCHEMAS with JDBC-style catalog pattern semantics. The SEA
747+
/// backend treats <c>SHOW SCHEMAS IN `<catalog>`</c> as a literal identifier
748+
/// lookup — it does not expand <c>%</c> / <c>_</c> wildcards. To match Thrift
749+
/// behaviour (PECO-3035), this helper resolves wildcards client-side:
750+
/// <list type="bullet">
751+
/// <item><description><c>null</c> or "%"/"*" → <c>SHOW SCHEMAS IN ALL CATALOGS</c>.</description></item>
752+
/// <item><description>A pattern containing unescaped <c>%</c> or <c>_</c> → <c>SHOW CATALOGS LIKE '&lt;pat&gt;'</c> to enumerate matching catalogs, then per-catalog <c>SHOW SCHEMAS IN `&lt;cat&gt;`</c> calls aggregated together.</description></item>
753+
/// <item><description>A literal name → single <c>SHOW SCHEMAS IN `&lt;catalog&gt;`</c> call.</description></item>
754+
/// </list>
755+
/// Returns a list of <c>(catalog, schema)</c> pairs in deterministic order.
756+
/// </summary>
757+
internal async Task<List<(string catalog, string schema)>> ListSchemasAsync(
758+
string? catalogPattern, string? schemaPattern, CancellationToken cancellationToken)
759+
{
760+
// Fast path: null or pure "match anything" → SHOW SCHEMAS IN ALL CATALOGS.
761+
if (catalogPattern == null || MetadataCommands.MetadataCommandBase.IsMatchAnything(catalogPattern))
762+
{
763+
return await ExecuteShowSchemasAsync(null, schemaPattern, cancellationToken).ConfigureAwait(false);
764+
}
765+
766+
// Non-trivial wildcard: enumerate matching catalogs and iterate per-catalog.
767+
if (MetadataCommands.MetadataCommandBase.ContainsUnescapedWildcard(catalogPattern))
768+
{
769+
string catalogsSql = new ShowCatalogsCommand(catalogPattern).Build();
770+
List<RecordBatch> catalogBatches;
771+
try
772+
{
773+
catalogBatches = await ExecuteMetadataSqlAsync(catalogsSql, cancellationToken).ConfigureAwait(false);
774+
}
775+
catch (DatabricksException ex) when (ex.IsObjectNotFoundException())
776+
{
777+
return new List<(string, string)>();
778+
}
779+
780+
var matchedCatalogs = new List<string>();
781+
foreach (var batch in catalogBatches)
782+
{
783+
var catalogArray = batch.Column(0) as StringArray;
784+
if (catalogArray == null) continue;
785+
for (int i = 0; i < catalogArray.Length; i++)
786+
{
787+
if (!catalogArray.IsNull(i))
788+
matchedCatalogs.Add(catalogArray.GetString(i));
789+
}
790+
}
791+
792+
var aggregated = new List<(string, string)>();
793+
foreach (string cat in matchedCatalogs)
794+
{
795+
try
796+
{
797+
var perCatalog = await ExecuteShowSchemasAsync(cat, schemaPattern, cancellationToken).ConfigureAwait(false);
798+
aggregated.AddRange(perCatalog);
799+
}
800+
catch (DatabricksException ex) when (ex.IsObjectNotFoundException())
801+
{
802+
// Catalog exists per SHOW CATALOGS but has no schemas / disappeared mid-query — skip.
803+
}
804+
}
805+
return aggregated;
806+
}
807+
808+
// Literal catalog name.
809+
return await ExecuteShowSchemasAsync(catalogPattern, schemaPattern, cancellationToken).ConfigureAwait(false);
810+
}
811+
812+
/// <summary>
813+
/// Issues a single SHOW SCHEMAS command (with the given literal catalog or
814+
/// <c>IN ALL CATALOGS</c>) and decodes the result. Caller is responsible for
815+
/// resolving wildcard patterns before calling this method.
816+
/// </summary>
817+
private async Task<List<(string catalog, string schema)>> ExecuteShowSchemasAsync(
818+
string? catalog, string? schemaPattern, CancellationToken cancellationToken)
819+
{
820+
string sql = new ShowSchemasCommand(catalog, schemaPattern).Build();
821+
List<RecordBatch> batches;
822+
try
823+
{
824+
batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false);
825+
}
826+
catch (DatabricksException ex) when (ex.IsObjectNotFoundException())
827+
{
828+
return new List<(string, string)>();
829+
}
830+
831+
// SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog
832+
// SHOW SCHEMAS IN `catalog` returns 1 column: databaseName
833+
bool showSchemasInAllCatalogs = catalog == null;
834+
var result = new List<(string, string)>();
835+
foreach (var batch in batches)
836+
{
837+
StringArray? catalogArray = null;
838+
StringArray? schemaArray;
839+
840+
if (showSchemasInAllCatalogs)
841+
{
842+
schemaArray = batch.Column(0) as StringArray;
843+
catalogArray = batch.Column(1) as StringArray;
844+
}
845+
else
846+
{
847+
schemaArray = batch.Column(0) as StringArray;
848+
}
849+
850+
if (schemaArray == null) continue;
851+
for (int i = 0; i < batch.Length; i++)
852+
{
853+
if (schemaArray.IsNull(i)) continue;
854+
string cat = catalogArray != null && !catalogArray.IsNull(i)
855+
? catalogArray.GetString(i)
856+
: catalog ?? "";
857+
result.Add((cat, schemaArray.GetString(i)));
858+
}
859+
}
860+
return result;
861+
}
862+
788863
/// <summary>
789864
/// Executes a SHOW COLUMNS command. When catalog is null, iterates over all catalogs
790865
/// since SHOW COLUMNS IN ALL CATALOGS is not yet supported by the backend.

csharp/src/StatementExecution/StatementExecutionStatement.cs

Lines changed: 11 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,62 +1076,25 @@ private async Task<QueryResult> GetSchemasAsync(CancellationToken cancellationTo
10761076
&& MetadataUtilities.NormalizeSparkCatalog(_metadataCatalogName) != null)
10771077
return MetadataSchemaFactory.CreateEmptySchemasResult();
10781078

1079-
string sql = new ShowSchemasCommand(
1079+
// PECO-3035: catalog follows JDBC LIKE semantics. ListSchemasAsync expands
1080+
// wildcards client-side (SHOW SCHEMAS IN ALL CATALOGS or per-catalog dispatch)
1081+
// and returns a flat list of (catalog, schema) pairs.
1082+
var rows = await _connection.ListSchemasAsync(
10801083
catalog,
1081-
EscapePatternWildcardsInName(_metadataSchemaName)).Build();
1082-
activity?.SetTag("sql_query", sql);
1083-
1084-
List<RecordBatch> batches;
1085-
try
1086-
{
1087-
batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false);
1088-
}
1089-
catch (DatabricksException ex) when (ex.IsObjectNotFoundException())
1090-
{
1091-
activity?.AddEvent("statement.get_schemas.object_not_found", [
1092-
new("error", ex.Message)
1093-
]);
1094-
return MetadataSchemaFactory.CreateEmptySchemasResult();
1095-
}
1096-
1097-
// SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog
1098-
// SHOW SCHEMAS IN `catalog` returns 1 column: databaseName
1099-
bool showAllCatalogs = catalog == null;
1084+
EscapePatternWildcardsInName(_metadataSchemaName),
1085+
cancellationToken).ConfigureAwait(false);
11001086

11011087
var tableSchemaBuilder = new StringArray.Builder();
11021088
var tableCatalogBuilder = new StringArray.Builder();
1103-
int count = 0;
1104-
foreach (var batch in batches)
1089+
foreach (var (cat, schemaName) in rows)
11051090
{
1106-
StringArray? catalogArray = null;
1107-
StringArray? schemaArray = null;
1108-
1109-
if (showAllCatalogs)
1110-
{
1111-
schemaArray = batch.Column(0) as StringArray;
1112-
catalogArray = batch.Column(1) as StringArray;
1113-
}
1114-
else
1115-
{
1116-
schemaArray = batch.Column(0) as StringArray;
1117-
}
1118-
1119-
if (schemaArray == null) continue;
1120-
for (int i = 0; i < batch.Length; i++)
1121-
{
1122-
if (schemaArray.IsNull(i)) continue;
1123-
tableSchemaBuilder.Append(schemaArray.GetString(i));
1124-
string catalogValue = catalogArray != null && !catalogArray.IsNull(i)
1125-
? catalogArray.GetString(i)
1126-
: catalog ?? "";
1127-
tableCatalogBuilder.Append(catalogValue);
1128-
count++;
1129-
}
1091+
tableSchemaBuilder.Append(schemaName);
1092+
tableCatalogBuilder.Append(cat);
11301093
}
11311094

1132-
activity?.SetTag("result_count", count);
1095+
activity?.SetTag("result_count", rows.Count);
11331096
var schema = MetadataSchemaFactory.CreateSchemasSchema();
1134-
return new QueryResult(count, new HiveInfoArrowStream(schema, new IArrowArray[]
1097+
return new QueryResult(rows.Count, new HiveInfoArrowStream(schema, new IArrowArray[]
11351098
{
11361099
tableSchemaBuilder.Build(), tableCatalogBuilder.Build()
11371100
}));

csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
using Apache.Arrow;
2121
using Apache.Arrow.Adbc;
2222
using Apache.Arrow.Adbc.Tests;
23+
using Apache.Arrow.Ipc;
2324
using AdbcDrivers.HiveServer2;
2425
using Xunit;
2526
using Xunit.Abstractions;
@@ -369,6 +370,75 @@ public void GetTableSchema_ThriftAndSEA_SameFieldNames()
369370
}
370371
}
371372

373+
// --- GetObjects: catalog wildcard pattern (PECO-3035) ---
374+
375+
/// <summary>
376+
/// Returns the total number of (catalog, schema) pairs in a
377+
/// GetObjects(depth=DbSchemas) result stream. The result schema is
378+
/// [catalog_name (string), catalog_db_schemas (list&lt;struct{db_schema_name,...}&gt;)].
379+
/// </summary>
380+
private static async Task<int> CountSchemasInGetObjects(IArrowArrayStream stream)
381+
{
382+
int total = 0;
383+
while (true)
384+
{
385+
using var batch = await stream.ReadNextRecordBatchAsync();
386+
if (batch == null) break;
387+
// Column 1 is the list<struct> of per-catalog schemas. Each list entry
388+
// is a struct array; its Length tells us how many schemas the catalog has.
389+
if (batch.Column(1) is not ListArray schemasList) continue;
390+
var schemasStruct = schemasList.Values as StructArray;
391+
if (schemasStruct == null) continue;
392+
for (int i = 0; i < batch.Length; i++)
393+
{
394+
if (schemasList.IsNull(i)) continue;
395+
int start = schemasList.ValueOffsets[i];
396+
int end = schemasList.ValueOffsets[i + 1];
397+
total += end - start;
398+
}
399+
}
400+
return total;
401+
}
402+
403+
[SkippableFact]
404+
public async Task SEA_GetObjects_CatalogPercentWildcard_ReturnsSchemasFromAllCatalogs()
405+
{
406+
// PECO-3035: SEA must treat "%" in the catalog argument as a wildcard,
407+
// matching Thrift / JDBC behavior. Before the fix, SEA wraps "%" in
408+
// backticks and looks for a catalog literally named "%", finding no
409+
// schemas → schema count = 0. With the fix, "%" expands to all catalogs
410+
// and we get the full set of schemas (matching catalogPattern=null).
411+
//
412+
// Note: counting schemas (column 1, the list<struct>) rather than just
413+
// catalogs (column 0) is what surfaces the bug — GetObjects always
414+
// populates catalogs via GetCatalogsAsync (which handles "%" via
415+
// SHOW CATALOGS LIKE), but GetSchemasAsync was passing "%" literally.
416+
SkipIfNotConfigured();
417+
using var conn = CreateSeaConnection();
418+
419+
using var baselineStream = conn.GetObjects(
420+
depth: AdbcConnection.GetObjectsDepth.DbSchemas,
421+
catalogPattern: null,
422+
dbSchemaPattern: null,
423+
tableNamePattern: null,
424+
tableTypes: null,
425+
columnNamePattern: null);
426+
int baselineSchemaCount = await CountSchemasInGetObjects(baselineStream);
427+
428+
using var wildcardStream = conn.GetObjects(
429+
depth: AdbcConnection.GetObjectsDepth.DbSchemas,
430+
catalogPattern: "%",
431+
dbSchemaPattern: null,
432+
tableNamePattern: null,
433+
tableTypes: null,
434+
columnNamePattern: null);
435+
int wildcardSchemaCount = await CountSchemasInGetObjects(wildcardStream);
436+
437+
Assert.True(baselineSchemaCount > 0,
438+
"Baseline (catalogPattern=null) must return at least one schema");
439+
Assert.Equal(baselineSchemaCount, wildcardSchemaCount);
440+
}
441+
372442
// --- GetTableTypes ---
373443

374444
[SkippableFact]

0 commit comments

Comments
 (0)