Skip to content

Commit 5afae0a

Browse files
committed
[BugFix] Fix Delta Lake un-partitioned materialized view query rewrite
DeltaLakePartitionTraits.getUpdatedPartitionNames returned null because partition-change tracking is not implemented for Delta Lake. The MV timeliness arbiter treats a null base-table update info as "unknown" and forces a full refresh, which marks the materialized view stale and disables query rewrite in the default CHECKED consistency mode. Return an empty set instead of null (i.e. "no updated partitions detected"), matching the behavior of other connectors that do not yet implement partition-change tracking (e.g. Hudi), so a fresh MV over a Delta Lake table stays eligible for query rewrite. Add a unit test asserting the MV timeliness arbiter reports NO_REFRESH (not FULL) for a Delta Lake based MV in the default CHECKED consistency mode. Signed-off-by: GavinMar <yangguansuo@starrocks.com>
1 parent 4297c7a commit 5afae0a

2 files changed

Lines changed: 100 additions & 2 deletions

File tree

fe/fe-core/src/main/java/com/starrocks/connector/partitiontraits/DeltaLakePartitionTraits.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
package com.starrocks.connector.partitiontraits;
1616

17+
import com.google.common.collect.Sets;
1718
import com.starrocks.catalog.BaseTableInfo;
1819
import com.starrocks.catalog.DeltaLakePartitionKey;
1920
import com.starrocks.catalog.MaterializedView;
@@ -37,7 +38,11 @@ public boolean isSupportPCTRefresh() {
3738
@Override
3839
public Set<String> getUpdatedPartitionNames(List<BaseTableInfo> baseTables,
3940
MaterializedView.AsyncRefreshContext context) {
40-
// TODO: implement
41-
return null;
41+
// TODO: Implement Delta Lake partition update tracking. Until then, return an empty set (meaning
42+
// "no updated partitions detected") rather than null. Returning null is interpreted as "update info
43+
// unknown", which forces the timeliness arbiter to treat the MV as stale (full refresh) and disables
44+
// query rewrite entirely in the default CHECKED consistency mode. Empty set keeps the MV eligible for
45+
// rewrite, matching the behavior of the other not-yet-implemented connectors (e.g. Hudi).
46+
return Sets.newHashSet();
4247
}
4348
}
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright 2021-present StarRocks, Inc. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.starrocks.sql.optimizer.rule.transformation.materialization;
16+
17+
import com.google.common.base.Joiner;
18+
import com.starrocks.catalog.BaseTableInfo;
19+
import com.starrocks.catalog.Database;
20+
import com.starrocks.catalog.DeltaLakeTable;
21+
import com.starrocks.catalog.MaterializedView;
22+
import com.starrocks.catalog.MvRefreshArbiter;
23+
import com.starrocks.catalog.MvUpdateInfo;
24+
import com.starrocks.catalog.Table;
25+
import com.starrocks.catalog.mv.MVTimelinessArbiter;
26+
import com.starrocks.qe.ConnectContext;
27+
import com.starrocks.server.GlobalStateMgr;
28+
import com.starrocks.server.MetadataMgr;
29+
import com.starrocks.sql.plan.ConnectorPlanTestBase;
30+
import mockit.Mock;
31+
import mockit.MockUp;
32+
import org.junit.jupiter.api.Assertions;
33+
import org.junit.jupiter.api.BeforeAll;
34+
import org.junit.jupiter.api.Test;
35+
36+
import java.util.Optional;
37+
38+
public class MvRewriteDeltaLakeTest extends MVTestBase {
39+
40+
@BeforeAll
41+
public static void beforeClass() throws Exception {
42+
MVTestBase.beforeClass();
43+
ConnectorPlanTestBase.mockCatalog(connectContext, "deltalake_catalog");
44+
}
45+
46+
/**
47+
* Regression guard for the Delta Lake un-partitioned MV query rewrite bug (issue #11409).
48+
*
49+
* <p>{@code DeltaLakePartitionTraits#getUpdatedPartitionNames} used to return {@code null}. The MV timeliness
50+
* arbiter interprets a null base-table update info as "unknown", returns {@link MvUpdateInfo.MvToRefreshType#FULL}
51+
* and therefore treats the MV as stale, disabling query rewrite in the default CHECKED consistency mode.
52+
*
53+
* <p>After the fix it returns an empty set ("no updated partitions detected"), so the arbiter returns
54+
* {@link MvUpdateInfo.MvToRefreshType#NO_REFRESH} and the MV stays eligible for rewrite. This test asserts that
55+
* timeliness decision directly, which is exactly the signal that gates the rewrite (a full end-to-end EXPLAIN
56+
* assertion is covered by the SQL test {@code test_mv_deltalake_rewrite}).
57+
*/
58+
@Test
59+
public void testDeltaLakeUnPartitionedMvIsNotStaleInCheckedMode() throws Exception {
60+
// The mocked Delta Lake table has no snapshot, so stub a stable table identifier (used for base-table
61+
// matching) to avoid an NPE and keep the query/MV base-table identifiers consistent.
62+
new MockUp<DeltaLakeTable>() {
63+
@Mock
64+
public String getTableIdentifier() {
65+
return Joiner.on(":").join("tbl", "mocked-delta-uuid");
66+
}
67+
};
68+
new MockUp<MetadataMgr>() {
69+
@Mock
70+
public Optional<Table> getTableWithIdentifier(ConnectContext context, BaseTableInfo baseTableInfo) {
71+
return Optional.of(new DeltaLakeTable());
72+
}
73+
};
74+
75+
String mvName = "mv_delta_checked";
76+
// No query_rewrite_consistency property => defaults to CHECKED, the mode in which the bug manifested.
77+
starRocksAssert.withMaterializedView("create materialized view " + mvName +
78+
" refresh deferred manual " +
79+
" as select col1, col2 from deltalake_catalog.deltalake_db.tbl");
80+
81+
Database testDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb("test");
82+
MaterializedView mv = (MaterializedView) GlobalStateMgr.getCurrentState().getLocalMetastore()
83+
.getTable(testDb.getFullName(), mvName);
84+
85+
MvUpdateInfo info = MvRefreshArbiter.getMVTimelinessUpdateInfo(
86+
mv, MVTimelinessArbiter.QueryRewriteParams.ofRefresh());
87+
// Before the fix this was FULL (stale => no rewrite); after the fix it must be NO_REFRESH (rewriteable).
88+
Assertions.assertEquals(MvUpdateInfo.MvToRefreshType.NO_REFRESH, info.getMVToRefreshType());
89+
Assertions.assertTrue(info.isValidRewrite());
90+
91+
starRocksAssert.dropMaterializedView(mvName);
92+
}
93+
}

0 commit comments

Comments
 (0)