Skip to content

Commit 601ce03

Browse files
beinanBeinan Wang
beinan
and
Beinan Wang
committed
[Feature] Add Lance connector optimizer operators and rules
Add the FE optimizer plumbing for the Lance connector: - LogicalLanceScanOperator / PhysicalLanceScanOperator with visitor hooks - LOGICAL_LANCE_SCAN / PHYSICAL_LANCE_SCAN enum entries in OperatorType - LanceScanImplementationRule + IMP_LANCE_LSCAN_TO_PSCAN rule type - Wire into RuleSet, PushDownPredicateScanRule, StatisticsCalculator, LogicalPlanPrinter, and RelationTransformer This follows the same pattern used by the Kudu connector. Signed-off-by: Beinan Wang <soros.wang@yahoo.com> Co-Authored-By: Beinan Wang <beinanwang@microsoft.com> Signed-off-by: Beinan Wang <soros.wang@yahoo.com>
1 parent 1b77dce commit 601ce03

12 files changed

Lines changed: 237 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/LogicalPlanPrinter.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,11 @@ public OperatorStr visitPhysicalPaimonScan(OptExpression optExpression, Integer
488488
return visitScanCommon(optExpression, step, "PAIMON SCAN");
489489
}
490490

491+
@Override
492+
public OperatorStr visitPhysicalLanceScan(OptExpression optExpression, Integer step) {
493+
return visitScanCommon(optExpression, step, "LANCE SCAN");
494+
}
495+
491496
public OperatorStr visitPhysicalProject(OptExpression optExpression, Integer step) {
492497
return visit(optExpression.getInputs().get(0), step);
493498
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptExpressionVisitor.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,10 @@ public R visitPhysicalKuduScan(OptExpression optExpression, C context) {
174174
return visitPhysicalScan(optExpression, context);
175175
}
176176

177+
public R visitPhysicalLanceScan(OptExpression optExpression, C context) {
178+
return visitPhysicalScan(optExpression, context);
179+
}
180+
177181
public R visitPhysicalSchemaScan(OptExpression optExpression, C context) {
178182
return visitPhysicalScan(optExpression, context);
179183
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/OperatorType.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public enum OperatorType {
3636
LOGICAL_ODPS_SCAN,
3737
LOGICAL_ICEBERG_METADATA_SCAN,
3838
LOGICAL_KUDU_SCAN,
39+
LOGICAL_LANCE_SCAN,
3940
LOGICAL_SCHEMA_SCAN,
4041
LOGICAL_MYSQL_SCAN,
4142
LOGICAL_ES_SCAN,
@@ -89,6 +90,7 @@ public enum OperatorType {
8990
PHYSICAL_ODPS_SCAN,
9091
PHYSICAL_ICEBERG_METADATA_SCAN,
9192
PHYSICAL_KUDU_SCAN,
93+
PHYSICAL_LANCE_SCAN,
9294
PHYSICAL_SCHEMA_SCAN,
9395
PHYSICAL_MYSQL_SCAN,
9496
PHYSICAL_META_SCAN,

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/OperatorVisitor.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.starrocks.sql.optimizer.operator.logical.LogicalJDBCScanOperator;
3737
import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
3838
import com.starrocks.sql.optimizer.operator.logical.LogicalKuduScanOperator;
39+
import com.starrocks.sql.optimizer.operator.logical.LogicalLanceScanOperator;
3940
import com.starrocks.sql.optimizer.operator.logical.LogicalLimitOperator;
4041
import com.starrocks.sql.optimizer.operator.logical.LogicalMetaScanOperator;
4142
import com.starrocks.sql.optimizer.operator.logical.LogicalMysqlScanOperator;
@@ -80,6 +81,7 @@
8081
import com.starrocks.sql.optimizer.operator.physical.PhysicalIntersectOperator;
8182
import com.starrocks.sql.optimizer.operator.physical.PhysicalJDBCScanOperator;
8283
import com.starrocks.sql.optimizer.operator.physical.PhysicalKuduScanOperator;
84+
import com.starrocks.sql.optimizer.operator.physical.PhysicalLanceScanOperator;
8385
import com.starrocks.sql.optimizer.operator.physical.PhysicalLimitOperator;
8486
import com.starrocks.sql.optimizer.operator.physical.PhysicalLookUpOperator;
8587
import com.starrocks.sql.optimizer.operator.physical.PhysicalMergeJoinOperator;
@@ -162,6 +164,10 @@ public R visitLogicalKuduScan(LogicalKuduScanOperator node, C context) {
162164
return visitLogicalTableScan(node, context);
163165
}
164166

167+
public R visitLogicalLanceScan(LogicalLanceScanOperator node, C context) {
168+
return visitLogicalTableScan(node, context);
169+
}
170+
165171
public R visitLogicalHudiScan(LogicalHudiScanOperator node, C context) {
166172
return visitLogicalTableScan(node, context);
167173
}
@@ -377,6 +383,10 @@ public R visitPhysicalKuduScan(PhysicalKuduScanOperator node, C context) {
377383
return visitOperator(node, context);
378384
}
379385

386+
public R visitPhysicalLanceScan(PhysicalLanceScanOperator node, C context) {
387+
return visitOperator(node, context);
388+
}
389+
380390
public R visitPhysicalTopN(PhysicalTopNOperator node, C context) {
381391
return visitOperator(node, context);
382392
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/logical/LogicalLanceScanOperator.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright 2021-present StarRocks, Inc. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.starrocks.sql.optimizer.operator.logical;
16+
17+
import com.google.common.base.Preconditions;
18+
import com.starrocks.catalog.Column;
19+
import com.starrocks.catalog.LanceTable;
20+
import com.starrocks.catalog.Table;
21+
import com.starrocks.sql.optimizer.operator.OperatorType;
22+
import com.starrocks.sql.optimizer.operator.OperatorVisitor;
23+
import com.starrocks.sql.optimizer.operator.ScanOperatorPredicates;
24+
import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
25+
import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
26+
27+
import java.util.Map;
28+
29+
public class LogicalLanceScanOperator extends LogicalScanOperator {
30+
private ScanOperatorPredicates predicates = new ScanOperatorPredicates();
31+
32+
public LogicalLanceScanOperator(Table table,
33+
Map<ColumnRefOperator, Column> colRefToColumnMetaMap,
34+
Map<Column, ColumnRefOperator> columnMetaToColRefMap,
35+
long limit, ScalarOperator predicate) {
36+
super(OperatorType.LOGICAL_LANCE_SCAN, table, colRefToColumnMetaMap, columnMetaToColRefMap, limit,
37+
predicate, null);
38+
Preconditions.checkState(table instanceof LanceTable);
39+
}
40+
41+
private LogicalLanceScanOperator() {
42+
super(OperatorType.LOGICAL_LANCE_SCAN);
43+
}
44+
45+
@Override
46+
public <R, C> R accept(OperatorVisitor<R, C> visitor, C context) {
47+
return visitor.visitLogicalLanceScan(this, context);
48+
}
49+
50+
@Override
51+
public ScanOperatorPredicates getScanOperatorPredicates() {
52+
return this.predicates;
53+
}
54+
55+
@Override
56+
public boolean isEmptyOutputRows() {
57+
return !table.isUnPartitioned() && predicates.getSelectedPartitionIds().isEmpty();
58+
}
59+
60+
@Override
61+
public void setScanOperatorPredicates(ScanOperatorPredicates predicates) {
62+
this.predicates = predicates;
63+
}
64+
65+
public static class Builder
66+
extends LogicalScanOperator.Builder<LogicalLanceScanOperator, LogicalLanceScanOperator.Builder> {
67+
68+
@Override
69+
protected LogicalLanceScanOperator newInstance() {
70+
return new LogicalLanceScanOperator();
71+
}
72+
73+
@Override
74+
public LogicalLanceScanOperator.Builder withOperator(LogicalLanceScanOperator scanOperator) {
75+
super.withOperator(scanOperator);
76+
builder.predicates = scanOperator.predicates.clone();
77+
return this;
78+
}
79+
}
80+
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/physical/PhysicalLanceScanOperator.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Copyright 2021-present StarRocks, Inc. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.starrocks.sql.optimizer.operator.physical;
16+
17+
import com.starrocks.sql.optimizer.OptExpression;
18+
import com.starrocks.sql.optimizer.OptExpressionVisitor;
19+
import com.starrocks.sql.optimizer.base.ColumnRefSet;
20+
import com.starrocks.sql.optimizer.operator.OperatorType;
21+
import com.starrocks.sql.optimizer.operator.OperatorVisitor;
22+
import com.starrocks.sql.optimizer.operator.ScanOperatorPredicates;
23+
import com.starrocks.sql.optimizer.operator.logical.LogicalLanceScanOperator;
24+
25+
public class PhysicalLanceScanOperator extends PhysicalScanOperator {
26+
private ScanOperatorPredicates predicates;
27+
28+
public PhysicalLanceScanOperator(LogicalLanceScanOperator scan) {
29+
super(OperatorType.PHYSICAL_LANCE_SCAN, scan);
30+
this.predicates = scan.getScanOperatorPredicates();
31+
}
32+
33+
@Override
34+
public ScanOperatorPredicates getScanOperatorPredicates() {
35+
return this.predicates;
36+
}
37+
38+
@Override
39+
public void setScanOperatorPredicates(ScanOperatorPredicates predicates) {
40+
this.predicates = predicates;
41+
}
42+
43+
@Override
44+
public <R, C> R accept(OperatorVisitor<R, C> visitor, C context) {
45+
return visitor.visitPhysicalLanceScan(this, context);
46+
}
47+
48+
@Override
49+
public <R, C> R accept(OptExpressionVisitor<R, C> visitor, OptExpression optExpression, C context) {
50+
return visitor.visitPhysicalLanceScan(optExpression, context);
51+
}
52+
53+
@Override
54+
public ColumnRefSet getUsedColumns() {
55+
ColumnRefSet refs = super.getUsedColumns();
56+
predicates.getNoEvalPartitionConjuncts().forEach(d -> refs.union(d.getUsedColumns()));
57+
predicates.getPartitionConjuncts().forEach(d -> refs.union(d.getUsedColumns()));
58+
predicates.getMinMaxConjuncts().forEach(d -> refs.union(d.getUsedColumns()));
59+
predicates.getMinMaxColumnRefMap().keySet().forEach(refs::union);
60+
return refs;
61+
}
62+
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import com.starrocks.sql.optimizer.rule.implementation.IntersectImplementationRule;
4040
import com.starrocks.sql.optimizer.rule.implementation.JDBCScanImplementationRule;
4141
import com.starrocks.sql.optimizer.rule.implementation.KuduScanImplementationRule;
42+
import com.starrocks.sql.optimizer.rule.implementation.LanceScanImplementationRule;
4243
import com.starrocks.sql.optimizer.rule.implementation.LimitImplementationRule;
4344
import com.starrocks.sql.optimizer.rule.implementation.MergeJoinImplementationRule;
4445
import com.starrocks.sql.optimizer.rule.implementation.MetaScanImplementationRule;
@@ -205,6 +206,7 @@ public class RuleSet {
205206
new OdpsScanImplementationRule(),
206207
new IcebergMetadataScanImplementationRule(),
207208
new KuduScanImplementationRule(),
209+
new LanceScanImplementationRule(),
208210
new SchemaScanImplementationRule(),
209211
new MysqlScanImplementationRule(),
210212
new EsScanImplementationRule(),

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleType.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ public enum RuleType {
264264
IMP_ODPS_LSCAN_TO_PSCAN,
265265
IMP_ICEBERG_METADATA_LSCAN_TO_PSCAN,
266266
IMP_KUDU_LSCAN_TO_PSCAN,
267+
IMP_LANCE_LSCAN_TO_PSCAN,
267268
IMP_SCHEMA_LSCAN_TO_PSCAN,
268269
IMP_MYSQL_LSCAN_TO_PSCAN,
269270
IMP_ES_LSCAN_TO_PSCAN,

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/implementation/LanceScanImplementationRule.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// Copyright 2021-present StarRocks, Inc. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.starrocks.sql.optimizer.rule.implementation;
16+
17+
import com.google.common.collect.Lists;
18+
import com.starrocks.sql.optimizer.OptExpression;
19+
import com.starrocks.sql.optimizer.OptimizerContext;
20+
import com.starrocks.sql.optimizer.operator.OperatorType;
21+
import com.starrocks.sql.optimizer.operator.logical.LogicalLanceScanOperator;
22+
import com.starrocks.sql.optimizer.operator.pattern.Pattern;
23+
import com.starrocks.sql.optimizer.operator.physical.PhysicalLanceScanOperator;
24+
import com.starrocks.sql.optimizer.rule.RuleType;
25+
26+
import java.util.List;
27+
28+
public class LanceScanImplementationRule extends ImplementationRule {
29+
public LanceScanImplementationRule() {
30+
super(RuleType.IMP_LANCE_LSCAN_TO_PSCAN,
31+
Pattern.create(OperatorType.LOGICAL_LANCE_SCAN));
32+
}
33+
34+
@Override
35+
public List<OptExpression> transform(OptExpression input, OptimizerContext context) {
36+
LogicalLanceScanOperator scan = (LogicalLanceScanOperator) input.getOp();
37+
PhysicalLanceScanOperator physicalLanceScan = new PhysicalLanceScanOperator(scan);
38+
OptExpression result = new OptExpression(physicalLanceScan);
39+
return Lists.newArrayList(result);
40+
}
41+
}

‎fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateScanRule.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ public class PushDownPredicateScanRule extends TransformationRule {
5353
OperatorType.LOGICAL_ICEBERG_METADATA_SCAN,
5454
OperatorType.LOGICAL_ICEBERG_EQUALITY_DELETE_SCAN,
5555
OperatorType.LOGICAL_KUDU_SCAN,
56+
OperatorType.LOGICAL_LANCE_SCAN,
5657
OperatorType.LOGICAL_SCHEMA_SCAN,
5758
OperatorType.LOGICAL_ES_SCAN,
5859
OperatorType.LOGICAL_META_SCAN,

0 commit comments

Comments
 (0)