Skip to content

Commit d20967f

Browse files
igorpeshanskyigorpeshansky
committed
Add support for parsing inline flags
Parse Java inline flag groups into the regex AST, both the bare directive form and the scoped (non-capturing) form, which were previously misparsed or rejected. Restructure parseGroup so the two are distinguished cleanly. Only parsing is added here; the transpiler still falls back on any inline flag. Signed-off-by: Igor Peshansky <ipeshansky@nvidia.com>
1 parent a123c6d commit d20967f

2 files changed

Lines changed: 127 additions & 35 deletions

File tree

‎sql-plugin/src/main/scala/com/nvidia/spark/rapids/RegexParser.scala‎

Lines changed: 99 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -177,43 +177,79 @@ class RegexParser(pattern: String) {
177177
base
178178
}
179179

180+
private def parseFlags(): RegexFlagSet = {
181+
def parseFlagSet(): Set[RegexFlag] = {
182+
var result = Set.empty[RegexFlag]
183+
while (!eof() && peek().exists(ch => RegexFlag.allFlagsString.contains(ch))) {
184+
result += RegexFlag.fromChar(consume())
185+
}
186+
result
187+
}
188+
val flags = parseFlagSet()
189+
val negated = if (peek().contains('-')) {
190+
consumeExpected('-'); parseFlagSet()
191+
} else Set.empty[RegexFlag]
192+
RegexFlagSet(flags, negated)
193+
}
194+
180195
private def parseGroup(): RegexAST = {
181-
val groupType = if (pos + 1 < pattern.length
182-
&& pattern.charAt(pos) == '?'
183-
&& ":!=<>".contains(pattern.charAt(pos+1))) {
196+
val groupType = if (peek().contains('?')) {
184197
consumeExpected('?')
185-
consume() match { // guaranteed exhaustive by the contains call above
186-
case ':' => RegexGroup.NonCapturing
187-
case '!' => RegexGroup.NegativeLookahead
188-
case '=' => RegexGroup.PositiveLookahead
189-
case '>' => RegexGroup.Independent
190-
case '<' if pos < pattern.length => consume() match {
191-
case '!' => RegexGroup.NegativeLookbehind
192-
case '=' => RegexGroup.PositiveLookbehind
193-
case ch if isLetter(ch) =>
194-
val nameStart = pos-1
195-
while (!eof() && peek().exists(c => isLetter(c) || isAsciiDigit(c))) {
196-
skip()
197-
}
198-
val name = pattern.substring(nameStart, pos)
199-
if (!peek().contains('>')) {
200-
throw new RegexUnsupportedException(
201-
"Illegal named capture group: malformed <name>", Some(nameStart-1))
202-
}
203-
consumeExpected('>')
204-
RegexGroup.Named(name)
205-
case _ => throw new RegexUnsupportedException(
206-
"Unexpected character after '<' in group", Some(pos-1))
198+
peek() match {
199+
case None => throw new RegexUnsupportedException("Unterminated inline flags", Some(pos))
200+
case Some(ch) if ":!=<>".contains(ch) => consumeExpected(ch) match {
201+
// guaranteed exhaustive by the contains call above
202+
case ':' => RegexGroup.NonCapturing
203+
case '!' => RegexGroup.NegativeLookahead
204+
case '=' => RegexGroup.PositiveLookahead
205+
case '>' => RegexGroup.Independent
206+
case '<' if pos < pattern.length => consume() match {
207+
case '!' => RegexGroup.NegativeLookbehind
208+
case '=' => RegexGroup.PositiveLookbehind
209+
case ch if isLetter(ch) =>
210+
val nameStart = pos-1
211+
while (!eof() && peek().exists(c => isLetter(c) || isAsciiDigit(c))) {
212+
skip()
213+
}
214+
val name = pattern.substring(nameStart, pos)
215+
if (!peek().contains('>')) {
216+
throw new RegexUnsupportedException(
217+
"Illegal named capture group: malformed <name>", Some(nameStart-1))
218+
}
219+
consumeExpected('>')
220+
RegexGroup.Named(name)
221+
case _ => throw new RegexUnsupportedException(
222+
"Unexpected character after '<' in group", Some(pos-1))
223+
}
224+
case '<' => throw new RegexUnsupportedException(
225+
"Pattern may not end with trailing '<' in group", Some(pos-1))
207226
}
208-
case '<' => throw new RegexUnsupportedException(
209-
"Pattern may not end with trailing '<' in group", Some(pos-1))
227+
case Some(_) =>
228+
val flags = parseFlags()
229+
// Leave the ':' to distinguish scoped flags from inline flags.
230+
peek() match {
231+
case Some(':') | Some(')') | None =>
232+
case Some(ch) =>
233+
throw new RegexUnsupportedException(s"Unexpected inline flag '$ch'", Some(pos))
234+
}
235+
RegexGroup.ScopedFlags(flags)
210236
}
211237
} else {
212238
RegexGroup.Capturing
213239
}
214-
val term = parseUntil(() => peek().contains(')'))
215-
consumeExpected(')')
216-
RegexGroup(groupType, term)
240+
(peek(), groupType) match {
241+
case (None, RegexGroup.ScopedFlags(_)) =>
242+
throw new RegexUnsupportedException("Unterminated inline flags", Some(pos))
243+
case (None, _) => throw new RegexUnsupportedException("Unclosed group", Some(pos))
244+
case (Some(')'), RegexGroup.ScopedFlags(flags)) =>
245+
consumeExpected(')')
246+
RegexInlineFlags(flags)
247+
case (Some(ch), _) =>
248+
if (ch == ':') consumeExpected(':')
249+
val term = parseUntil(() => peek().contains(')'))
250+
consumeExpected(')')
251+
RegexGroup(groupType, term)
252+
}
217253
}
218254

219255
private def parseCharacterClass(): RegexCharacterClass = {
@@ -1633,6 +1669,8 @@ class CudfRegexTranspiler(mode: RegexMode) {
16331669
"Independent groups are not supported"
16341670
case RegexGroup.Named(_) =>
16351671
"Named capture groups are not supported"
1672+
case RegexGroup.ScopedFlags(_) =>
1673+
"Scoped inline flags are not supported"
16361674
case _ =>
16371675
s"Unknown group type: ${g.groupType}"
16381676
}
@@ -1779,6 +1817,35 @@ sealed case class RegexSequence(parts: ListBuffer[RegexAST]) extends RegexAST {
17791817
override def toRegexString: String = parts.map(_.toRegexString).mkString
17801818
}
17811819

1820+
sealed abstract class RegexFlag(val char: Char)
1821+
object RegexFlag {
1822+
case object CaseInsensitive extends RegexFlag('i')
1823+
case object UnixLines extends RegexFlag('d')
1824+
case object Multiline extends RegexFlag('m')
1825+
case object DotAll extends RegexFlag('s')
1826+
case object UnicodeCase extends RegexFlag('u')
1827+
case object Comments extends RegexFlag('x')
1828+
case object UnicodeClasses extends RegexFlag('U')
1829+
final val all: Seq[RegexFlag] =
1830+
Seq(CaseInsensitive, UnixLines, Multiline, DotAll, UnicodeCase, Comments, UnicodeClasses)
1831+
final val fromChar: Map[Char, RegexFlag] = all.map(f => f.char -> f).toMap
1832+
final val allFlagsString: String = all.map(_.char).mkString
1833+
}
1834+
1835+
sealed case class RegexFlagSet(flags: Set[RegexFlag], negated: Set[RegexFlag]) {
1836+
def isEmpty: Boolean = flags.isEmpty && negated.isEmpty
1837+
def toRegexString: String = {
1838+
val flagsString = flags.map(_.char).mkString
1839+
val negatedString = if (negated.isEmpty) "" else s"-${negated.map(_.char).mkString}"
1840+
s"${flagsString}${negatedString}"
1841+
}
1842+
}
1843+
1844+
sealed case class RegexInlineFlags(flags: RegexFlagSet) extends RegexAST {
1845+
override def children(): Seq[RegexAST] = Seq.empty
1846+
override def toRegexString: String = s"(?${flags.toRegexString})"
1847+
}
1848+
17821849
object RegexGroup {
17831850
sealed trait Type
17841851
case object Capturing extends Type
@@ -1789,6 +1856,7 @@ object RegexGroup {
17891856
case object NegativeLookbehind extends Type
17901857
case class Named(name: String) extends Type
17911858
case object Independent extends Type
1859+
case class ScopedFlags(flags: RegexFlagSet) extends Type
17921860
}
17931861

17941862
sealed case class RegexGroup(groupType: RegexGroup.Type, term: RegexAST) extends RegexAST {
@@ -1807,6 +1875,7 @@ sealed case class RegexGroup(groupType: RegexGroup.Type, term: RegexAST) extends
18071875
case NegativeLookbehind => s"(?<!${term.toRegexString})"
18081876
case Named(name) => s"(?<$name>${term.toRegexString})"
18091877
case Independent => s"(?>${term.toRegexString})"
1878+
case ScopedFlags(flags) => s"(?${flags.toRegexString}:${term.toRegexString})"
18101879
}
18111880
}
18121881

‎tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionParserSuite.scala‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,31 @@ class RegularExpressionParserSuite extends AnyFunSuite {
9292
RegexGroup(RegexGroup.Named("n"), RegexSequence(ListBuffer(RegexChar('f')))))))
9393
}
9494

95+
test("flags") {
96+
assert(parse("(?i)(?m-s)(?-duxU)(?)(?i-)(?-)") ===
97+
RegexSequence(ListBuffer(
98+
RegexInlineFlags(RegexFlagSet(Set(RegexFlag.CaseInsensitive), Set())),
99+
RegexInlineFlags(RegexFlagSet(Set(RegexFlag.Multiline), Set(RegexFlag.DotAll))),
100+
RegexInlineFlags(RegexFlagSet(Set(),
101+
Set(RegexFlag.UnixLines, RegexFlag.UnicodeCase, RegexFlag.Comments,
102+
RegexFlag.UnicodeClasses))),
103+
RegexInlineFlags(RegexFlagSet(Set(), Set())),
104+
RegexInlineFlags(RegexFlagSet(Set(RegexFlag.CaseInsensitive), Set())),
105+
RegexInlineFlags(RegexFlagSet(Set(), Set())))))
106+
}
107+
108+
test("scoped inline flags") {
109+
assert(parse("(?i:ab)") ===
110+
RegexSequence(ListBuffer(
111+
RegexGroup(RegexGroup.ScopedFlags(RegexFlagSet(Set(RegexFlag.CaseInsensitive), Set())),
112+
RegexSequence(ListBuffer(RegexChar('a'), RegexChar('b')))))))
113+
assert(parse("(?i-s:a)") ===
114+
RegexSequence(ListBuffer(
115+
RegexGroup(RegexGroup.ScopedFlags(
116+
RegexFlagSet(Set(RegexFlag.CaseInsensitive), Set(RegexFlag.DotAll))),
117+
RegexSequence(ListBuffer(RegexChar('a')))))))
118+
}
119+
95120
test("character class") {
96121
assert(parse("[a-z+A-Z]") ===
97122
RegexSequence(ListBuffer(
@@ -256,14 +281,12 @@ class RegularExpressionParserSuite extends AnyFunSuite {
256281
}
257282

258283
test("group containing quantifier") {
259-
val e = intercept[RegexUnsupportedException] {
260-
parse("(?)")
261-
}
262-
assert(e.getMessage.startsWith("Base expression cannot start with quantifier"))
263-
264284
assert(parse("(?:a?)") === RegexSequence(ListBuffer(
265285
RegexGroup(RegexGroup.NonCapturing, RegexSequence(ListBuffer(
266286
RegexRepetition(RegexChar('a'), SimpleQuantifier('?'))))))))
287+
assert(parse("(?i:a)") === RegexSequence(ListBuffer(
288+
RegexGroup(RegexGroup.ScopedFlags(RegexFlagSet(Set(RegexFlag.CaseInsensitive), Set())),
289+
RegexSequence(ListBuffer(RegexChar('a')))))))
267290
}
268291

269292
test("group not starting with ? is a capturing group") {

0 commit comments

Comments
 (0)