Skip to content

Commit 76d8e4c

Browse files
igorpeshanskyigorpeshansky
committed
Address review feedback
Signed-off-by: Igor Peshansky <ipeshansky@nvidia.com>
1 parent df1b165 commit 76d8e4c

2 files changed

Lines changed: 69 additions & 64 deletions

File tree

‎integration_tests/src/main/python/map_test.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -526,6 +526,20 @@ def test_str_to_map_expr_with_all_regex_delimiters():
526526
'str_to_map(a, "[,]{1,10}", "[:]{1,10}") as m5'
527527
), conf={'spark.sql.mapKeyDedupPolicy': 'LAST_WIN'})
528528

529+
# test case-insensitive delimiters
530+
flag_rows = [
531+
('k1xv1PAIRk2xv2',),
532+
('k1Xv1pairk2xv2',),
533+
('k1Xv1PAIRk2xv2',),
534+
(None,),
535+
]
536+
assert_gpu_and_cpu_are_equal_collect(
537+
lambda spark: spark.createDataFrame(flag_rows, ['a']).selectExpr(
538+
'str_to_map(a, "(?i)pair", "x") as inline_pair',
539+
'str_to_map(a, "pair", "(?i)x") as inline_key_value',
540+
'str_to_map(a, "(?i:pair)", "(?i:x)") as scoped_both'),
541+
conf={'spark.sql.mapKeyDedupPolicy': 'LAST_WIN'})
542+
529543

530544
@pytest.mark.parametrize('empty_type', all_empty_string_types)
531545
def test_str_to_map_input_all_empty(empty_type):

‎integration_tests/src/main/python/regexp_test.py‎

Lines changed: 55 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ def test_split_re_negative_limit():
5757

5858
def test_split_re_zero_limit():
5959
data_gen = mk_str_gen('([bf]o{0,2}:){1,7}') \
60-
.with_special_case('boo:and:foo')
60+
.with_special_case('boo:and:foo') \
61+
.with_special_case('foo:FOO:FoO')
6162
assert_gpu_and_cpu_are_equal_collect(
6263
lambda spark : unary_op_df(spark, data_gen).selectExpr(
6364
'split(a, "[:]", 0)',
@@ -69,7 +70,9 @@ def test_split_re_zero_limit():
6970
'split(a, "f[o]+", 0)',
7071
'split(a, "f[o]*", 0)',
7172
'split(a, "f[o]?", 0)',
72-
'split(a, "[o]", 0)'),
73+
'split(a, "[o]", 0)',
74+
'split(a, "(?i)foo", 0)',
75+
'split(a, "(?i:foo)", 0)'),
7376
conf=_regexp_conf)
7477

7578
def test_split_re_one_limit():
@@ -91,7 +94,8 @@ def test_split_re_one_limit():
9194

9295
def test_split_re_positive_limit():
9396
data_gen = mk_str_gen('([bf]o{0,2}:){1,7}') \
94-
.with_special_case('boo:and:foo')
97+
.with_special_case('boo:and:foo') \
98+
.with_special_case('foo:FOO:FoO')
9599
assert_gpu_and_cpu_are_equal_collect(
96100
lambda spark : unary_op_df(spark, data_gen).selectExpr(
97101
'split(a, "[:]", 2)',
@@ -103,7 +107,9 @@ def test_split_re_positive_limit():
103107
'split(a, "f[o]+", 2)',
104108
'split(a, "f[o]*", 9)',
105109
'split(a, "f[o]?", 5)',
106-
'split(a, "[o]", 5)'),
110+
'split(a, "[o]", 5)',
111+
'split(a, "(?i)foo", 2)',
112+
'split(a, "(?i:foo)", 5)'),
107113
conf=_regexp_conf)
108114

109115
def test_split_re_no_limit():
@@ -470,14 +476,22 @@ def test_re_replace_null():
470476
conf=_regexp_conf)
471477

472478
def test_regexp_replace():
473-
gen = mk_str_gen('[abcd]{0,3}')
479+
gen = mk_str_gen('[a-dA-D]{0,4}')
474480
assert_gpu_and_cpu_are_equal_collect(
475-
lambda spark: unary_op_df(spark, gen).selectExpr(
476-
'regexp_replace(a, "a", "A")',
477-
'regexp_replace(a, "[^xyz]", "A")',
478-
'regexp_replace(a, "([^x])|([^y])", "A")',
479-
'regexp_replace(a, "(?:aa)+", "A")',
480-
'regexp_replace(a, "a|b|c", "A")'),
481+
lambda spark: unary_op_df(spark, gen).selectExpr(
482+
'regexp_replace(a, "a", "A")',
483+
'regexp_replace(a, "[^xyz]", "A")',
484+
'regexp_replace(a, "([^x])|([^y])", "A")',
485+
'regexp_replace(a, "(?:aa)+", "A")',
486+
'regexp_replace(a, "a|b|c", "A")',
487+
# case folding
488+
'regexp_replace(a, "(?i)abc", "X")',
489+
'regexp_replace(a, "(?i:abc)", "Y")',
490+
'regexp_replace(a, "(?i)a(?-i)b", "Z")',
491+
# $1 backref into a case-insensitive capture group
492+
'regexp_replace(a, "(?i)(abc)", "[$1]")',
493+
'regexp_replace(a, "((?i)abc)", "[$1]")',
494+
'regexp_replace(a, "((?i:abc))", "[$1]")'),
481495
conf=_regexp_conf)
482496

483497
# https://github.com/NVIDIA/spark-rapids/issues/14742
@@ -645,7 +659,7 @@ def test_regexp_replace_character_set_negated():
645659
conf=_regexp_conf)
646660

647661
def test_regexp_extract():
648-
gen = mk_str_gen('[abcd]{1,3}[0-9]{1,3}/?[abcd]{1,3}')
662+
gen = mk_str_gen('[a-dA-D]{1,3}[0-9]{1,3}/?[a-dA-D]{1,3}')
649663
assert_gpu_and_cpu_are_equal_collect(
650664
lambda spark: unary_op_df(spark, gen).selectExpr(
651665
'regexp_extract(a, "([0-9]+)", 1)',
@@ -657,7 +671,13 @@ def test_regexp_extract():
657671
'regexp_extract(a, "^([a-d]*)([0-9]*)\\\\/([a-d]*)", 3)',
658672
'regexp_extract(a, "^([a-d]*)([0-9]*)\\\\/([a-d]*)$", 3)',
659673
'regexp_extract(a, "^([a-d]*)([0-9]*)(\\\\/[a-d]*)", 3)',
660-
'regexp_extract(a, "^([a-d]*)([0-9]*)(\\\\/[a-d]*)$", 3)'),
674+
'regexp_extract(a, "^([a-d]*)([0-9]*)(\\\\/[a-d]*)$", 3)',
675+
# (?i) folds the capture-group contents but does not change group numbering
676+
'regexp_extract(a, "(?i)([a-c]+)([0-9]+)", 1)',
677+
'regexp_extract(a, "(?i)([a-c]+)([0-9]+)", 2)',
678+
# the flag turns on partway through, so only group 2 is case-insensitive
679+
'regexp_extract(a, "([a-c]+)(?i)([a-c]+)", 2)',
680+
'regexp_extract(a, "(?i)(abc)", 1)'),
661681
conf=_regexp_conf)
662682

663683
capture_group_gen = mk_str_gen('[abcd]{1,2}')
@@ -669,18 +689,6 @@ def test_regexp_extract():
669689
'regexp_extract(a, "(a)(b)|(c)(d)", 4)'),
670690
conf=_regexp_conf)
671691

672-
def test_regexp_extract_case_insensitive_inline_flag():
673-
gen = mk_str_gen('[abcABC]{1,3}[0-9]{1,2}[abcABC]{1,3}')
674-
assert_gpu_and_cpu_are_equal_collect(
675-
lambda spark: unary_op_df(spark, gen).selectExpr(
676-
# (?i) folds the capture-group contents but does not change group numbering
677-
'regexp_extract(a, "(?i)([a-c]+)([0-9]+)", 1)',
678-
'regexp_extract(a, "(?i)([a-c]+)([0-9]+)", 2)',
679-
# the flag turns on partway through, so only group 2 is case-insensitive
680-
'regexp_extract(a, "([a-c]+)(?i)([a-c]+)", 2)',
681-
'regexp_extract(a, "(?i)(abc)", 1)'),
682-
conf=_regexp_conf)
683-
684692
def test_regexp_extract_no_match():
685693
gen = mk_str_gen('[abcd]{1,3}[0-9]{1,3}[abcd]{1,3}')
686694
assert_gpu_and_cpu_are_equal_collect(
@@ -1036,15 +1044,35 @@ def test_predefined_character_classes():
10361044
conf=_regexp_conf)
10371045

10381046
def test_rlike():
1039-
gen = mk_str_gen('[abcd]{1,3}')
1047+
gen = mk_str_gen('[a-dA-D]{1,4}')
10401048
assert_gpu_and_cpu_are_equal_collect(
10411049
lambda spark: unary_op_df(spark, gen).selectExpr(
10421050
'a rlike "a{2}"',
10431051
'a rlike "a{1,3}"',
10441052
'a rlike "a{1,}"',
10451053
'a rlike "a[bc]d"',
10461054
'a rlike "a[bc]d"',
1047-
'a rlike "^[a-d]*$"'),
1055+
'a rlike "^[a-d]*$"',
1056+
# case-insensitive inline flag
1057+
'a rlike "(?i)abc"',
1058+
'a rlike "(?i)[a-c]"',
1059+
'a rlike "a(?i)b"',
1060+
'a rlike "(?i)a(?-i)b"',
1061+
'a rlike "a|(?i)b"',
1062+
'a rlike "(?i)(a|b)c"',
1063+
'a rlike "(?i:abc)"',
1064+
'a rlike "(?i:a|b)"',
1065+
'a rlike "(?-i:a)b"',
1066+
# nested scoped-flags groups
1067+
'a rlike "(?i:(?-i:a)b)"',
1068+
'a rlike "(?-i:a(?i:b)c)"',
1069+
'a rlike "(?i:a(?-i:b(?i:c))d)"',
1070+
# a bare inline flag inside a scoped-flags group
1071+
'a rlike "(?i:a(?-i)b)"',
1072+
'a rlike "(?-i:a(?i)b)"',
1073+
# a negated non-case-insensitive flag is a no-op (mode off by default)
1074+
'a rlike "(?-s:abc)"',
1075+
'a rlike "(?i-s:abc)"'),
10481076
conf=_regexp_conf)
10491077

10501078
def test_rlike_embedded_null():
@@ -1132,43 +1160,6 @@ def test_rlike_fallback_lookarounds_independent_named():
11321160
'RLike',
11331161
conf=_regexp_conf)
11341162

1135-
def test_rlike_case_insensitive_inline_flag():
1136-
gen = mk_str_gen('[a-dA-D]{1,4}')
1137-
assert_gpu_and_cpu_are_equal_collect(
1138-
lambda spark: unary_op_df(spark, gen).selectExpr(
1139-
'a rlike "(?i)abc"',
1140-
'a rlike "(?i)[a-c]"',
1141-
'a rlike "a(?i)b"',
1142-
'a rlike "(?i)a(?-i)b"',
1143-
'a rlike "a|(?i)b"',
1144-
'a rlike "(?i)(a|b)c"',
1145-
'a rlike "(?i:abc)"',
1146-
'a rlike "(?i:a|b)"',
1147-
'a rlike "(?-i:a)b"',
1148-
# nested scoped-flags groups
1149-
'a rlike "(?i:(?-i:a)b)"',
1150-
'a rlike "(?-i:a(?i:b)c)"',
1151-
'a rlike "(?i:a(?-i:b(?i:c))d)"',
1152-
# a bare inline flag inside a scoped-flags group
1153-
'a rlike "(?i:a(?-i)b)"',
1154-
'a rlike "(?-i:a(?i)b)"',
1155-
# a negated non-case-insensitive flag is a no-op (mode off by default)
1156-
'a rlike "(?-s:abc)"',
1157-
'a rlike "(?i-s:abc)"'),
1158-
conf=_regexp_conf)
1159-
1160-
def test_regexp_replace_case_insensitive_inline_flag():
1161-
gen = mk_str_gen('[a-cA-C]{0,4}')
1162-
# case folding must also work in replace mode, including a `$1` backref into a
1163-
# case-insensitive capture group
1164-
assert_gpu_and_cpu_are_equal_collect(
1165-
lambda spark: unary_op_df(spark, gen).selectExpr(
1166-
'regexp_replace(a, "(?i)abc", "X")',
1167-
'regexp_replace(a, "(?i:abc)", "Y")',
1168-
'regexp_replace(a, "(?i)a(?-i)b", "Z")',
1169-
'regexp_replace(a, "(?i)(abc)", "[$1]")'),
1170-
conf=_regexp_conf)
1171-
11721163
@allow_non_gpu('ProjectExec', 'RLike')
11731164
def test_rlike_fallback_unsupported_inline_flags():
11741165
gen = mk_str_gen('[abcd]{1,3}')

0 commit comments

Comments
 (0)