Skip to content

Commit e679720

Browse files
Cool6689Cool6689
committed
Implement GROUP BY ALL functionality
1 parent abb9ef9 commit e679720

6 files changed

Lines changed: 294 additions & 13 deletions

File tree

‎integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGroupByAllTableIT.java‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.relational.it.query.recent;
21+
22+
import org.apache.iotdb.it.env.EnvFactory;
23+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
24+
import org.apache.iotdb.itbase.category.TableClusterIT;
25+
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
26+
27+
import org.junit.AfterClass;
28+
import org.junit.BeforeClass;
29+
import org.junit.Test;
30+
import org.junit.experimental.categories.Category;
31+
import org.junit.runner.RunWith;
32+
33+
import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData;
34+
import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail;
35+
import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
36+
37+
@RunWith(IoTDBTestRunner.class)
38+
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
39+
public class IoTDBGroupByAllTableIT {
40+
protected static final String DATABASE_NAME = "test";
41+
42+
protected static final String[] createSqls =
43+
new String[] {
44+
"CREATE DATABASE " + DATABASE_NAME,
45+
"USE " + DATABASE_NAME,
46+
"CREATE TABLE table1(device_id STRING TAG, s1 DOUBLE FIELD, x INT32 FIELD, y INT32 FIELD)",
47+
"INSERT INTO table1(time, device_id, s1, x, y) VALUES (1, 'd1', 20.0, 1, 10)",
48+
"INSERT INTO table1(time, device_id, s1, x, y) VALUES (2, 'd1', 30.0, 2, 20)",
49+
"INSERT INTO table1(time, device_id, s1, x, y) VALUES (3, 'd2', 20.0, 1, 30)",
50+
"INSERT INTO table1(time, device_id, s1, x, y) VALUES (4, 'd2', 40.0, 3, 40)",
51+
"FLUSH"
52+
};
53+
54+
@BeforeClass
55+
public static void setUp() throws Exception {
56+
EnvFactory.getEnv().initClusterEnvironment();
57+
prepareTableData(createSqls);
58+
}
59+
60+
@AfterClass
61+
public static void tearDown() throws Exception {
62+
EnvFactory.getEnv().cleanClusterEnvironment();
63+
}
64+
65+
@Test
66+
public void testBasicInference() {
67+
String[] expectedHeader = new String[] {"device_id", "_col1"};
68+
String[] retArray = new String[] {"d1,25.0,", "d2,30.0,"};
69+
70+
tableResultSetEqualTest(
71+
"SELECT device_id, avg(s1) FROM table1 GROUP BY ALL ORDER BY device_id",
72+
expectedHeader,
73+
retArray,
74+
DATABASE_NAME);
75+
76+
expectedHeader = new String[] {"d", "avg_s1"};
77+
retArray = new String[] {"d1,25.0,", "d2,30.0,"};
78+
tableResultSetEqualTest(
79+
"SELECT device_id AS d, avg(s1) AS avg_s1 FROM table1 GROUP BY ALL ORDER BY d",
80+
expectedHeader,
81+
retArray,
82+
DATABASE_NAME);
83+
}
84+
85+
@Test
86+
public void testExpressionAndConstantInference() {
87+
String[] expectedHeader = new String[] {"_col0", "_col1"};
88+
String[] retArray = new String[] {"false,2,", "true,2,"};
89+
90+
tableResultSetEqualTest(
91+
"SELECT s1 > 25, count(*) FROM table1 GROUP BY ALL ORDER BY 1",
92+
expectedHeader,
93+
retArray,
94+
DATABASE_NAME);
95+
96+
retArray = new String[] {"factory_01,4,"};
97+
tableResultSetEqualTest(
98+
"SELECT 'factory_01', count(*) FROM table1 GROUP BY ALL",
99+
expectedHeader,
100+
retArray,
101+
DATABASE_NAME);
102+
103+
retArray = new String[] {"100,4,"};
104+
tableResultSetEqualTest(
105+
"SELECT 100, count(*) FROM table1 GROUP BY ALL", expectedHeader, retArray, DATABASE_NAME);
106+
107+
expectedHeader = new String[] {"device_id", "_col1", "_col2"};
108+
retArray = new String[] {"d1,1,25.0,", "d2,1,30.0,"};
109+
tableResultSetEqualTest(
110+
"SELECT device_id, 1, avg(s1) FROM table1 GROUP BY ALL ORDER BY device_id",
111+
expectedHeader,
112+
retArray,
113+
DATABASE_NAME);
114+
}
115+
116+
@Test
117+
public void testAggregateExpressionsAreSkipped() {
118+
String[] expectedHeader = new String[] {"_col0", "_col1"};
119+
String[] retArray = new String[] {"7.0,7.0,"};
120+
121+
tableResultSetEqualTest(
122+
"SELECT sum(x), abs(sum(x)) FROM table1 GROUP BY ALL",
123+
expectedHeader,
124+
retArray,
125+
DATABASE_NAME);
126+
127+
expectedHeader = new String[] {"_col0"};
128+
retArray = new String[] {"27.5,"};
129+
tableResultSetEqualTest(
130+
"SELECT avg(s1) FROM table1 GROUP BY ALL", expectedHeader, retArray, DATABASE_NAME);
131+
}
132+
133+
@Test
134+
public void testMixedAggregateExpressionValidation() {
135+
tableAssertTestFail(
136+
"SELECT s1 + avg(y) FROM table1 GROUP BY ALL",
137+
"must be an aggregate expression or appear in GROUP BY clause",
138+
DATABASE_NAME);
139+
140+
String[] expectedHeader = new String[] {"s1", "_col1"};
141+
String[] retArray = new String[] {"20.0,40.0,", "30.0,50.0,", "40.0,80.0,"};
142+
143+
tableResultSetEqualTest(
144+
"SELECT s1, s1 + avg(y) FROM table1 GROUP BY ALL ORDER BY s1",
145+
expectedHeader,
146+
retArray,
147+
DATABASE_NAME);
148+
}
149+
150+
@Test
151+
public void testSelectAllAfterExpansion() {
152+
String[] expectedHeader = new String[] {"time", "device_id", "s1", "x", "y", "_col5"};
153+
String[] retArray =
154+
new String[] {
155+
"1970-01-01T00:00:00.001Z,d1,20.0,1,10,1,",
156+
"1970-01-01T00:00:00.002Z,d1,30.0,2,20,1,",
157+
"1970-01-01T00:00:00.003Z,d2,20.0,1,30,1,",
158+
"1970-01-01T00:00:00.004Z,d2,40.0,3,40,1,"
159+
};
160+
161+
tableResultSetEqualTest(
162+
"SELECT *, count(*) FROM table1 GROUP BY ALL ORDER BY time",
163+
expectedHeader,
164+
retArray,
165+
DATABASE_NAME);
166+
}
167+
168+
@Test
169+
public void testWindowFunctionValidation() {
170+
String[] expectedHeader = new String[] {"device_id", "_col1", "_col2"};
171+
String[] retArray = new String[] {"d1,1,25.0,", "d2,1,30.0,"};
172+
173+
tableResultSetEqualTest(
174+
"SELECT device_id, count(*) OVER (PARTITION BY device_id), avg(s1) FROM table1 GROUP BY ALL ORDER BY device_id",
175+
expectedHeader,
176+
retArray,
177+
DATABASE_NAME);
178+
179+
tableAssertTestFail(
180+
"SELECT device_id, rank() OVER (ORDER BY s1), avg(s1) FROM table1 GROUP BY ALL",
181+
"must be an aggregate expression or appear in GROUP BY clause",
182+
DATABASE_NAME);
183+
}
184+
185+
@Test
186+
public void testIllegalAutoGroupByCombination() {
187+
tableAssertTestFail(
188+
"SELECT device_id, count(*) FROM table1 GROUP BY ALL, device_id",
189+
"GROUP BY ALL cannot be combined with explicit grouping elements",
190+
DATABASE_NAME);
191+
192+
tableAssertTestFail(
193+
"SELECT device_id, count(*) FROM table1 GROUP BY ALL, ROLLUP(device_id)",
194+
"GROUP BY ALL cannot be combined with explicit grouping elements",
195+
DATABASE_NAME);
196+
197+
tableAssertTestFail(
198+
"SELECT device_id, count(*) FROM table1 GROUP BY CUBE(device_id), ALL",
199+
"GROUP BY ALL cannot be combined with explicit grouping elements",
200+
DATABASE_NAME);
201+
}
202+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2773,12 +2773,23 @@ private Analysis.GroupingSetAnalysis analyzeGroupBy(
27732773
FunctionCall gapFillColumn = null;
27742774
ImmutableList.Builder<Expression> gapFillGroupingExpressions = ImmutableList.builder();
27752775

2776-
checkGroupingSetsCount(node.getGroupBy().get());
2777-
for (GroupingElement groupingElement : node.getGroupBy().get().getGroupingElements()) {
2776+
GroupBy groupBy = node.getGroupBy().get();
2777+
List<Expression> allGroupByExpressions = ImmutableList.of();
2778+
List<GroupingElement> groupingElements;
2779+
2780+
if (groupBy.isAll()) {
2781+
allGroupByExpressions = inferAllGroupByExpressions(outputExpressions);
2782+
groupingElements = ImmutableList.of(new SimpleGroupBy(allGroupByExpressions));
2783+
} else {
2784+
checkGroupingSetsCount(groupBy);
2785+
groupingElements = groupBy.getGroupingElements();
2786+
}
2787+
2788+
for (GroupingElement groupingElement : groupingElements) {
27782789
if (groupingElement instanceof SimpleGroupBy) {
27792790
for (Expression column : groupingElement.getExpressions()) {
27802791
// simple GROUP BY expressions allow ordinals or arbitrary expressions
2781-
if (column instanceof LongLiteral) {
2792+
if (!groupBy.isAll() && column instanceof LongLiteral) {
27822793
long ordinal = ((LongLiteral) column).getParsedValue();
27832794
if (ordinal < 1 || ordinal > outputExpressions.size()) {
27842795
throw new SemanticException(
@@ -2869,6 +2880,9 @@ private Analysis.GroupingSetAnalysis analyzeGroupBy(
28692880
"%s is not comparable, and therefore cannot be used in GROUP BY", type));
28702881
}
28712882
}
2883+
if (groupBy.isAll()) {
2884+
validateAllGroupByWindowFunctions(allGroupByExpressions, scope, outputExpressions);
2885+
}
28722886

28732887
Analysis.GroupingSetAnalysis groupingSets =
28742888
new Analysis.GroupingSetAnalysis(
@@ -2919,6 +2933,37 @@ private Expression resolveGroupBySelectAlias(
29192933
.orElse(expression);
29202934
}
29212935

2936+
private List<Expression> inferAllGroupByExpressions(List<Expression> outputExpressions) {
2937+
ImmutableList.Builder<Expression> groupingExpressions = ImmutableList.builder();
2938+
for (Expression expression : outputExpressions) {
2939+
if (extractAggregateFunctions(ImmutableList.of(expression)).isEmpty()
2940+
&& extractWindowFunctions(ImmutableList.of(expression)).isEmpty()) {
2941+
groupingExpressions.add(expression);
2942+
}
2943+
}
2944+
return groupingExpressions.build();
2945+
}
2946+
2947+
private void validateAllGroupByWindowFunctions(
2948+
List<Expression> groupingExpressions, Scope scope, List<Expression> outputExpressions) {
2949+
List<FunctionCall> windowFunctions = extractWindowFunctions(outputExpressions);
2950+
for (FunctionCall windowFunction : windowFunctions) {
2951+
Analysis.ResolvedWindow window = analysis.getWindow(windowFunction);
2952+
ImmutableList.Builder<Expression> expressions = ImmutableList.builder();
2953+
expressions.addAll(windowFunction.getArguments());
2954+
expressions.addAll(window.getPartitionBy());
2955+
getSortItemsFromOrderBy(window.getOrderBy()).stream()
2956+
.map(SortItem::getSortKey)
2957+
.forEach(expressions::add);
2958+
if (window.getFrame().isPresent()) {
2959+
WindowFrame frame = window.getFrame().get();
2960+
frame.getStart().getValue().ifPresent(expressions::add);
2961+
frame.getEnd().flatMap(FrameBound::getValue).ifPresent(expressions::add);
2962+
}
2963+
verifySourceAggregations(groupingExpressions, scope, expressions.build(), analysis);
2964+
}
2965+
}
2966+
29222967
private boolean isDateBinGapFill(Expression column) {
29232968
return column instanceof FunctionCall
29242969
&& DATE_BIN

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2635,10 +2635,24 @@ public Node visitSelectAll(RelationalSqlParser.SelectAllContext ctx) {
26352635
}
26362636

26372637
@Override
2638-
public Node visitGroupBy(RelationalSqlParser.GroupByContext ctx) {
2638+
public Node visitAllGroupBy(RelationalSqlParser.AllGroupByContext ctx) {
2639+
return new GroupBy(getLocation(ctx), false, true, ImmutableList.of());
2640+
}
2641+
2642+
@Override
2643+
public Node visitExplicitGroupBy(RelationalSqlParser.ExplicitGroupByContext ctx) {
2644+
if (ctx.setQuantifier() == null && ctx.groupingElement().size() > 1) {
2645+
for (RelationalSqlParser.GroupingElementContext element : ctx.groupingElement()) {
2646+
if (element.getText().equalsIgnoreCase("ALL")) {
2647+
throw new SemanticException(
2648+
"GROUP BY ALL cannot be combined with explicit grouping elements");
2649+
}
2650+
}
2651+
}
26392652
return new GroupBy(
26402653
getLocation(ctx),
26412654
isDistinct(ctx.setQuantifier()),
2655+
false,
26422656
visit(ctx.groupingElement(), GroupingElement.class));
26432657
}
26442658

‎iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/sql/ast/GroupBy.java‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,25 +33,37 @@ public class GroupBy extends Node {
3333
private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(GroupBy.class);
3434

3535
private final boolean isDistinct;
36+
private final boolean isAll;
3637
private final List<GroupingElement> groupingElements;
3738

3839
public GroupBy(boolean isDistinct, List<GroupingElement> groupingElements) {
39-
super(null);
40-
this.isDistinct = isDistinct;
41-
this.groupingElements = ImmutableList.copyOf(requireNonNull(groupingElements));
40+
this(null, isDistinct, false, groupingElements);
4241
}
4342

4443
public GroupBy(
4544
NodeLocation location, boolean isDistinct, List<GroupingElement> groupingElements) {
46-
super(requireNonNull(location, "location is null"));
45+
this(location, isDistinct, false, groupingElements);
46+
}
47+
48+
public GroupBy(
49+
NodeLocation location,
50+
boolean isDistinct,
51+
boolean isAll,
52+
List<GroupingElement> groupingElements) {
53+
super(location);
4754
this.isDistinct = isDistinct;
55+
this.isAll = isAll;
4856
this.groupingElements = ImmutableList.copyOf(requireNonNull(groupingElements));
4957
}
5058

5159
public boolean isDistinct() {
5260
return isDistinct;
5361
}
5462

63+
public boolean isAll() {
64+
return isAll;
65+
}
66+
5567
public List<GroupingElement> getGroupingElements() {
5668
return groupingElements;
5769
}
@@ -76,18 +88,20 @@ public boolean equals(Object o) {
7688
}
7789
GroupBy groupBy = (GroupBy) o;
7890
return isDistinct == groupBy.isDistinct
91+
&& isAll == groupBy.isAll
7992
&& Objects.equals(groupingElements, groupBy.groupingElements);
8093
}
8194

8295
@Override
8396
public int hashCode() {
84-
return Objects.hash(isDistinct, groupingElements);
97+
return Objects.hash(isDistinct, isAll, groupingElements);
8598
}
8699

87100
@Override
88101
public String toString() {
89102
return toStringHelper(this)
90103
.add("isDistinct", isDistinct)
104+
.add("isAll", isAll)
91105
.add("groupingElements", groupingElements)
92106
.toString();
93107
}
@@ -98,7 +112,7 @@ public boolean shallowEquals(Node other) {
98112
return false;
99113
}
100114

101-
return isDistinct == ((GroupBy) other).isDistinct;
115+
return isDistinct == ((GroupBy) other).isDistinct && isAll == ((GroupBy) other).isAll;
102116
}
103117

104118
@Override

‎iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/queryengine/plan/relational/sql/util/CommonQuerySqlFormatter.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -404,14 +404,19 @@ public Void visitQuerySpecification(QuerySpecification node, Integer indent) {
404404

405405
node.getGroupBy()
406406
.ifPresent(
407-
groupBy ->
407+
groupBy -> {
408+
if (groupBy.isAll()) {
409+
append(indent, "GROUP BY ALL").append('\n');
410+
} else {
408411
append(
409412
indent,
410413
"GROUP BY "
411414
+ (groupBy.isDistinct() ? " DISTINCT " : "")
412415
+ org.apache.iotdb.commons.queryengine.plan.relational.sql.util
413416
.ExpressionFormatter.formatGroupBy(groupBy.getGroupingElements()))
414-
.append('\n'));
417+
.append('\n');
418+
}
419+
});
415420

416421
node.getHaving()
417422
.ifPresent(having -> append(indent, "HAVING " + formatExpression(having)).append('\n'));

0 commit comments

Comments
 (0)