Skip to content

Commit e8f4b57

Browse files
committed
Fix pipe text date conversion
1 parent d6d5755 commit e8f4b57

4 files changed

Lines changed: 97 additions & 5 deletions

File tree

integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,7 @@ private void createTestDataForString(Tablet tablet, int j) {
346346
"enable = true",
347347
"true",
348348
"false",
349+
"2024-06-28 08:00:00",
349350
};
350351
for (int i = 0; i < generateDataSize; i++) {
351352
tablet.addValue(

integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -547,6 +547,7 @@ private Binary[] createTestDataForString() {
547547
"enable = true",
548548
"true",
549549
"false",
550+
"2024-06-28 08:00:00",
550551
"12345678910",
551552
"123231232132131233213123123123123123131312",
552553
"123231232132131233213123123123123123131312.212312321312312",

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -808,16 +808,36 @@ private static long parseTimestamp(final String value) {
808808
}
809809

810810
private static int parseDate(final String value) {
811-
if (value == null || value.isEmpty()) {
811+
if (value == null) {
812812
return DEFAULT_DATE;
813813
}
814-
try {
815-
if (TypeInferenceUtils.isNumber(value)) {
816-
int date = Integer.parseInt(value);
814+
final String trimmedValue = StringUtils.trim(value);
815+
if (trimmedValue.isEmpty()) {
816+
return DEFAULT_DATE;
817+
}
818+
if (TypeInferenceUtils.isNumber(trimmedValue)) {
819+
try {
820+
int date = Integer.parseInt(trimmedValue);
817821
DateUtils.parseIntToLocalDate(date);
818822
return date;
823+
} catch (final Exception e) {
824+
return DEFAULT_DATE;
819825
}
820-
return DateTimeUtils.parseDateExpressionToInt(StringUtils.trim(value));
826+
}
827+
try {
828+
return DateTimeUtils.parseDateExpressionToInt(trimmedValue);
829+
} catch (final Exception e) {
830+
return parseDateTimeToDate(trimmedValue);
831+
}
832+
}
833+
834+
private static int parseDateTimeToDate(final String value) {
835+
try {
836+
return DateUtils.parseDateExpressionToInt(
837+
Instant.ofEpochMilli(
838+
DateTimeUtils.convertDatetimeStrToLong(value, ZoneOffset.UTC, 0, "ms"))
839+
.atZone(ZoneOffset.UTC)
840+
.toLocalDate());
821841
} catch (final Exception e) {
822842
return DEFAULT_DATE;
823843
}
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.db.pipe.receiver.transform.converter;
21+
22+
import org.apache.tsfile.common.conf.TSFileConfig;
23+
import org.apache.tsfile.enums.TSDataType;
24+
import org.apache.tsfile.utils.Binary;
25+
import org.apache.tsfile.utils.DateUtils;
26+
import org.junit.Assert;
27+
import org.junit.Test;
28+
29+
public class ValueConverterTest {
30+
31+
@Test
32+
public void testTextLikeDateTimeToDate() {
33+
final int expectedDate = DateUtils.parseDateExpressionToInt("2024-06-28");
34+
final Binary dateTime = binary("2024-06-28 08:00:00");
35+
36+
Assert.assertEquals(expectedDate, ValueConverter.convertTextToDate(dateTime));
37+
Assert.assertEquals(expectedDate, ValueConverter.convertStringToDate(dateTime));
38+
Assert.assertEquals(expectedDate, ValueConverter.convertBlobToDate(dateTime));
39+
}
40+
41+
@Test
42+
public void testTextLikeDateToDateKeepsExistingFallbacks() {
43+
final int expectedDate = DateUtils.parseDateExpressionToInt("2024-06-28");
44+
final int defaultDate = DateUtils.parseDateExpressionToInt("1970-01-01");
45+
46+
Assert.assertEquals(expectedDate, ValueConverter.convertTextToDate(binary("20240628")));
47+
Assert.assertEquals(expectedDate, ValueConverter.convertTextToDate(binary("2024-06-28")));
48+
Assert.assertEquals(defaultDate, ValueConverter.convertTextToDate(binary("12345678910")));
49+
Assert.assertEquals(defaultDate, ValueConverter.convertTextToDate(binary("invalid-date")));
50+
}
51+
52+
@Test
53+
public void testTextArrayDateTimeToDate() {
54+
final int expectedDate = DateUtils.parseDateExpressionToInt("2024-06-28");
55+
final int defaultDate = DateUtils.parseDateExpressionToInt("1970-01-01");
56+
57+
final int[] dates =
58+
(int[])
59+
ArrayConverter.convert(
60+
TSDataType.TEXT,
61+
TSDataType.DATE,
62+
new Binary[] {binary("2024-06-28 08:00:00"), binary("invalid-date")});
63+
64+
Assert.assertArrayEquals(new int[] {expectedDate, defaultDate}, dates);
65+
}
66+
67+
private static Binary binary(final String value) {
68+
return new Binary(value, TSFileConfig.STRING_CHARSET);
69+
}
70+
}

0 commit comments

Comments
 (0)