DOPE-176: add missing Token functions and expressions for token handling

core/src/main/kotlin/ch/ergon/dope/resolvable/expression/type/function/token/ContainsTokenExpression.kt

@@ -0,0 +1,23 @@
+package ch.ergon.dope.resolvable.expression.type.function.token
+
+import ch.ergon.dope.resolvable.expression.operator.FunctionOperator
+import ch.ergon.dope.resolvable.expression.type.TypeExpression
+import ch.ergon.dope.resolvable.expression.type.toDopeType
+import ch.ergon.dope.validtype.BooleanType
+import ch.ergon.dope.validtype.StringType
+
+data class ContainsTokenExpression(
+    val inputExpression: TypeExpression<StringType>,
+    val tokenExpression: TypeExpression<StringType>,
+    val tokenOptions: ContainsTokenOptions? = null,
+) : FunctionOperator<BooleanType>
+
+fun TypeExpression<StringType>.containsToken(
+    tokenExpression: TypeExpression<StringType>,
+    tokenOptions: ContainsTokenOptions? = null,
+) = ContainsTokenExpression(this, tokenExpression, tokenOptions)
+
+fun TypeExpression<StringType>.containsToken(
+    token: String,
+    tokenOptions: ContainsTokenOptions? = null,
+) = containsToken(token.toDopeType(), tokenOptions)

core/src/main/kotlin/ch/ergon/dope/resolvable/expression/type/function/token/ContainsTokenLikeExpression.kt

@@ -0,0 +1,24 @@
+package ch.ergon.dope.resolvable.expression.type.function.token
+
+import ch.ergon.dope.resolvable.expression.operator.FunctionOperator
+import ch.ergon.dope.resolvable.expression.type.TypeExpression
+import ch.ergon.dope.resolvable.expression.type.toDopeType
+import ch.ergon.dope.validtype.BooleanType
+import ch.ergon.dope.validtype.StringType
+import ch.ergon.dope.validtype.ValidType
+
+data class ContainsTokenLikeExpression(
+    val inputObject: TypeExpression<out ValidType>,
+    val likeExpression: TypeExpression<StringType>,
+    val tokenOptions: ContainsTokenOptions? = null,
+) : FunctionOperator<BooleanType>
+
+fun TypeExpression<out ValidType>.containsTokenLike(
+    likeExpression: TypeExpression<StringType>,
+    tokenOptions: ContainsTokenOptions? = null,
+) = ContainsTokenLikeExpression(this, likeExpression, tokenOptions)
+
+fun TypeExpression<out ValidType>.containsTokenLike(
+    likeExpression: String,
+    tokenOptions: ContainsTokenOptions? = null,
+) = containsTokenLike(likeExpression.toDopeType(), tokenOptions)

core/src/main/kotlin/ch/ergon/dope/resolvable/expression/type/function/token/ContainsTokenOptions.kt

@@ -0,0 +1,19 @@
+package ch.ergon.dope.resolvable.expression.type.function.token
+
+import ch.ergon.dope.resolvable.Resolvable
+
+data class ContainsTokenOptions(
+    val hasNames: Boolean? = null,
+    val case: TokenCase? = null,
+    val includeSpecialCharacters: Boolean? = null,
+    val split: Boolean? = null,
+    val trim: Boolean? = null,
+) : Resolvable
+
+fun containsTokenOptions(
+    hasNames: Boolean? = null,
+    case: TokenCase? = null,
+    includeSpecialCharacters: Boolean? = null,
+    split: Boolean? = null,
+    trim: Boolean? = null,
+) = ContainsTokenOptions(hasNames, case, includeSpecialCharacters, split, trim)

core/src/main/kotlin/ch/ergon/dope/resolvable/expression/type/function/token/ContainsTokenRegexpExpression.kt

@@ -0,0 +1,24 @@
+package ch.ergon.dope.resolvable.expression.type.function.token
+
+import ch.ergon.dope.resolvable.expression.operator.FunctionOperator
+import ch.ergon.dope.resolvable.expression.type.TypeExpression
+import ch.ergon.dope.resolvable.expression.type.toDopeType
+import ch.ergon.dope.validtype.BooleanType
+import ch.ergon.dope.validtype.StringType
+import ch.ergon.dope.validtype.ValidType
+
+data class ContainsTokenRegexpExpression(
+    val inputObject: TypeExpression<out ValidType>,
+    val regexExpression: TypeExpression<StringType>,
+    val tokenOptions: ContainsTokenOptions? = null,
+) : FunctionOperator<BooleanType>
+
+fun TypeExpression<out ValidType>.containsTokenRegexp(
+    regexExpression: TypeExpression<StringType>,
+    tokenOptions: ContainsTokenOptions? = null,
+) = ContainsTokenRegexpExpression(this, regexExpression, tokenOptions)
+
+fun TypeExpression<out ValidType>.containsTokenRegexp(
+    regexExpression: String,
+    tokenOptions: ContainsTokenOptions? = null,
+) = containsTokenRegexp(regexExpression.toDopeType(), tokenOptions)

core/src/main/kotlin/ch/ergon/dope/resolvable/expression/type/function/token/TokenOptions.kt

@@ -0,0 +1,17 @@
+package ch.ergon.dope.resolvable.expression.type.function.token
+
+import ch.ergon.dope.resolvable.Resolvable
+
+enum class TokenCase { LOWER, UPPER }
+
+data class TokensOptions(
+    val hasName: Boolean? = null,
+    val case: TokenCase? = null,
+    val includeSpecialCharacters: Boolean? = null,
+) : Resolvable
+
+fun tokenOptions(
+    hasName: Boolean? = null,
+    case: TokenCase? = null,
+    includeSpecialCharacters: Boolean? = null,
+) = TokensOptions(hasName, case, includeSpecialCharacters)

core/src/main/kotlin/ch/ergon/dope/resolvable/expression/type/function/token/TokensExpression.kt

@@ -0,0 +1,28 @@
+package ch.ergon.dope.resolvable.expression.type.function.token
+
+import ch.ergon.dope.resolvable.expression.operator.FunctionOperator
+import ch.ergon.dope.resolvable.expression.type.TypeExpression
+import ch.ergon.dope.resolvable.expression.type.toDopeType
+import ch.ergon.dope.validtype.ArrayType
+import ch.ergon.dope.validtype.StringType
+
+data class TokensExpression(
+    val inString: TypeExpression<ArrayType<StringType>>,
+    val tokensOptions: TokensOptions? = null,
+) : FunctionOperator<ArrayType<StringType>>
+
+fun TypeExpression<ArrayType<StringType>>.tokenize(tokensOptions: TokensOptions) = TokensExpression(this, tokensOptions)
+
+fun TypeExpression<ArrayType<StringType>>.tokenize(
+    hasName: Boolean? = null,
+    case: TokenCase? = null,
+    includeSpecialCharacters: Boolean? = null,
+) = TokensExpression(this, tokenOptions(hasName, case, includeSpecialCharacters))
+
+fun List<String>.tokenize(tokensOptions: TokensOptions) = toDopeType().tokenize(tokensOptions)
+
+fun List<String>.tokenize(
+    hasName: Boolean? = null,
+    case: TokenCase? = null,
+    includeSpecialCharacters: Boolean? = null,
+) = toDopeType().tokenize(tokenOptions(hasName, case, includeSpecialCharacters))

couchbase/src/integrationTest/kotlin/ch/ergon/dope/integrationTest/TestCouchbaseDatabase.kt

@@ -66,6 +66,7 @@ object TestCouchbaseDatabase {
     }
 
     private fun ensureAuditCollections() = runBlocking {
+        cluster.waitUntilReady(MAX_TIMEOUT_IN_SECONDS.seconds)
         cluster.waitUntilReady(MAX_TIMEOUT_IN_SECONDS.seconds)
         cluster.query("CREATE SCOPE `$BUCKET`.`app` IF NOT EXISTS").execute()
         cluster.query("CREATE COLLECTION `$BUCKET`.`app`.`order_audit` IF NOT EXISTS").execute()
@@ -74,11 +75,22 @@ object TestCouchbaseDatabase {
     private fun ensureIndexes() = runBlocking {
         cluster.waitUntilReady(MAX_TIMEOUT_IN_SECONDS.seconds)
         cluster.query("CREATE PRIMARY INDEX IF NOT EXISTS ON `$BUCKET`").execute()
-        cluster.query("CREATE PRIMARY INDEX IF NOT EXISTS ON `$BUCKET`.`app`.`order_audit`").execute()
+        retryOnFailure { cluster.query("CREATE PRIMARY INDEX IF NOT EXISTS ON `$BUCKET`.`app`.`order_audit`").execute() }
         cluster.query("CREATE INDEX IF NOT EXISTS `ix_order_employee` ON `$BUCKET`(`employee`) WHERE `type` = \"order\"").execute()
         cluster.query("CREATE INDEX IF NOT EXISTS `ix_order_client` ON `$BUCKET`(`client`) WHERE `type` = \"order\"").execute()
     }
 
+    private suspend fun <T> retryOnFailure(maxRetries: Int = MAX_RETRIES, delayMs: Long = 2000, action: suspend () -> T): T {
+        repeat(maxRetries - 1) {
+            try {
+                return action()
+            } catch (_: Exception) {
+                Thread.sleep(delayMs)
+            }
+        }
+        return action()
+    }
+
     private fun initDatabase() {
         val bucket = cluster.bucket(BUCKET)
         val collection = bucket.defaultCollection()

couchbase/src/main/kotlin/ch/ergon/dope/couchbase/resolver/CouchbaseResolver.kt

@@ -30,7 +30,8 @@ import ch.ergon.dope.resolvable.expression.rowscope.windowdefinition.WindowDefin
 import ch.ergon.dope.resolvable.expression.rowscope.windowdefinition.WindowFrameClause
 import ch.ergon.dope.resolvable.expression.type.CaseClass
 import ch.ergon.dope.resolvable.expression.type.ObjectEntryPrimitive
-import ch.ergon.dope.resolvable.expression.type.function.string.factory.CustomTokenOptions
+import ch.ergon.dope.resolvable.expression.type.function.token.ContainsTokenOptions
+import ch.ergon.dope.resolvable.expression.type.function.token.TokensOptions
 import ch.ergon.dope.resolver.QueryResolver
 
 interface AbstractCouchbaseResolver : QueryResolver<CouchbaseDopeQuery> {
@@ -88,7 +89,9 @@ class CouchbaseResolver(
 
             is Asterisk -> resolve(resolvable)
 
-            is CustomTokenOptions -> resolve(resolvable)
+            is ContainsTokenOptions -> resolve(resolvable)
+
+            is TokensOptions -> resolve(resolvable)
 
             is HashOrNestedLoopHint -> resolve(resolvable)
 

couchbase/src/main/kotlin/ch/ergon/dope/couchbase/resolver/expression/FunctionOperatorResolver.kt

@@ -7,6 +7,7 @@ import ch.ergon.dope.couchbase.util.formatFunctionQueryString
 import ch.ergon.dope.merge
 import ch.ergon.dope.orEmpty
 import ch.ergon.dope.resolvable.expression.operator.FunctionOperator
+import ch.ergon.dope.resolvable.expression.type.TypeExpression
 import ch.ergon.dope.resolvable.expression.type.function.FunctionExpression
 import ch.ergon.dope.resolvable.expression.type.function.conditional.DecodeExpression
 import ch.ergon.dope.resolvable.expression.type.function.conditional.Nvl2Expression
@@ -15,9 +16,15 @@ import ch.ergon.dope.resolvable.expression.type.function.search.ISearchFunctionE
 import ch.ergon.dope.resolvable.expression.type.function.search.SearchDependencyFunctionExpression
 import ch.ergon.dope.resolvable.expression.type.function.search.SearchFunctionType
 import ch.ergon.dope.resolvable.expression.type.function.string.MaskExpression
-import ch.ergon.dope.resolvable.expression.type.function.string.TokensExpression
+import ch.ergon.dope.resolvable.expression.type.function.token.ContainsTokenExpression
+import ch.ergon.dope.resolvable.expression.type.function.token.ContainsTokenLikeExpression
+import ch.ergon.dope.resolvable.expression.type.function.token.ContainsTokenOptions
+import ch.ergon.dope.resolvable.expression.type.function.token.ContainsTokenRegexpExpression
+import ch.ergon.dope.resolvable.expression.type.function.token.TokensExpression
 import ch.ergon.dope.resolvable.expression.type.function.type.ToNumberExpression
 import ch.ergon.dope.resolvable.expression.type.toDopeType
+import ch.ergon.dope.validtype.StringType
+import ch.ergon.dope.validtype.ValidType
 
 interface FunctionOperatorResolver : AbstractCouchbaseResolver {
     fun resolve(typeExpression: FunctionOperator<*>): CouchbaseDopeQuery = when (typeExpression) {
@@ -51,16 +58,37 @@ interface FunctionOperatorResolver : AbstractCouchbaseResolver {
         }
 
         is TokensExpression -> {
-            val inString = typeExpression.inStr.toDopeType().toDopeQuery(this)
-            val optionsDopeQuery = typeExpression.options?.toDopeQuery(this).takeIf { !it?.queryString.isNullOrEmpty() }
+            val inStringDopeQuery = typeExpression.inString.toDopeQuery(this)
+            val optionsDopeQuery = typeExpression.tokensOptions?.toDopeQuery(this).takeIf { !it?.queryString.isNullOrEmpty() }
             val functionQueryString = formatFunctionQueryString(
                 "TOKENS",
-                inString.queryString,
+                inStringDopeQuery.queryString,
                 optionsDopeQuery?.queryString,
             )
-            CouchbaseDopeQuery(functionQueryString, optionsDopeQuery?.parameters.orEmpty())
+            CouchbaseDopeQuery(functionQueryString, inStringDopeQuery.parameters.merge(optionsDopeQuery?.parameters))
         }
 
+        is ContainsTokenExpression -> resolve(
+            "CONTAINS_TOKEN",
+            typeExpression.inputExpression,
+            typeExpression.tokenExpression,
+            typeExpression.tokenOptions,
+        )
+
+        is ContainsTokenLikeExpression -> resolve(
+            "CONTAINS_TOKEN_LIKE",
+            typeExpression.inputObject,
+            typeExpression.likeExpression,
+            typeExpression.tokenOptions,
+        )
+
+        is ContainsTokenRegexpExpression -> resolve(
+            "CONTAINS_TOKEN_REGEXP",
+            typeExpression.inputObject,
+            typeExpression.regexExpression,
+            typeExpression.tokenOptions,
+        )
+
         is MaskExpression -> {
             val inputStringDopeQuery = typeExpression.inStr.toDopeQuery(this)
             val optionsString =
@@ -160,4 +188,25 @@ interface FunctionOperatorResolver : AbstractCouchbaseResolver {
 
         else -> throw UnsupportedOperationException("Not supported: $typeExpression")
     }
+
+    private fun resolve(
+        symbol: String,
+        inputObject: TypeExpression<out ValidType>,
+        tokenExpression: TypeExpression<StringType>,
+        options: ContainsTokenOptions?,
+    ): CouchbaseDopeQuery {
+        val inputObjectDopeQuery = inputObject.toDopeQuery(this)
+        val tokenExpressionDopeQuery = tokenExpression.toDopeQuery(this)
+        val optionsDopeQuery = options?.toDopeQuery(this).takeIf { !it?.queryString.isNullOrEmpty() }
+        val functionQueryString = formatFunctionQueryString(
+            symbol,
+            inputObjectDopeQuery.queryString,
+            tokenExpressionDopeQuery.queryString,
+            optionsDopeQuery?.queryString,
+        )
+        return CouchbaseDopeQuery(
+            functionQueryString,
+            inputObjectDopeQuery.parameters.merge(tokenExpressionDopeQuery.parameters, optionsDopeQuery?.parameters),
+        )
+    }
 }

couchbase/src/main/kotlin/ch/ergon/dope/couchbase/resolver/expression/TypeExpressionResolver.kt

@@ -40,7 +40,8 @@ import ch.ergon.dope.resolvable.expression.type.collection.SatisfiesExpression
 import ch.ergon.dope.resolvable.expression.type.function.array.UnpackExpression
 import ch.ergon.dope.resolvable.expression.type.function.date.DateComponentType
 import ch.ergon.dope.resolvable.expression.type.function.date.DateUnitType
-import ch.ergon.dope.resolvable.expression.type.function.string.factory.CustomTokenOptions
+import ch.ergon.dope.resolvable.expression.type.function.token.ContainsTokenOptions
+import ch.ergon.dope.resolvable.expression.type.function.token.TokensOptions
 import ch.ergon.dope.resolvable.expression.type.range.RangeIndexedLike
 import ch.ergon.dope.resolvable.expression.type.range.RangeLike
 import ch.ergon.dope.resolvable.expression.type.relational.BetweenExpression
@@ -367,11 +368,27 @@ interface TypeExpressionResolver : InfixOperatorResolver, FunctionOperatorResolv
         )
     }
 
-    fun resolve(customTokenOptions: CustomTokenOptions): CouchbaseDopeQuery {
+    fun resolve(tokensOptions: TokensOptions): CouchbaseDopeQuery {
         val options = listOfNotNull(
-            customTokenOptions.name?.let { "name" to it },
-            customTokenOptions.case?.let { "case" to "\"${it.queryString}\"" },
-            customTokenOptions.specials?.let { "specials" to it },
+            tokensOptions.hasName?.let { "name" to it },
+            tokensOptions.case?.let { "case" to "\"${it.queryString}\"" },
+            tokensOptions.includeSpecialCharacters?.let { "specials" to it },
+        )
+        val queryString = options
+            .joinToString(", ", "{", "}") { (key, value) -> "\"$key\": $value" }
+            .takeIf { options.isNotEmpty() }
+            .orEmpty()
+
+        return CouchbaseDopeQuery(queryString = queryString)
+    }
+
+    fun resolve(containsTokenOptions: ContainsTokenOptions): CouchbaseDopeQuery {
+        val options = listOfNotNull(
+            containsTokenOptions.hasNames?.let { "names" to it },
+            containsTokenOptions.case?.let { "case" to "\"${it.queryString}\"" },
+            containsTokenOptions.includeSpecialCharacters?.let { "specials" to it },
+            containsTokenOptions.split?.let { "split" to it },
+            containsTokenOptions.trim?.let { "trim" to it },
         )
         val queryString = options
             .joinToString(", ", "{", "}") { (key, value) -> "\"$key\": $value" }