feat: do not assume a "_tokens" column suffix in searchNgrams

In the previously introduced "searchNgrams" filter function, it was only
possible to specify Spanner column names which had a suffix of
"_tokens". While this may be a best practice it is not mandatory and
technically the column can be given any name so we make no assumptions
on the column name and allow the user to pass the actual tokenized
column name.

Author: alethenorio

spanfiltering/transpile_test.go

@@ -163,10 +163,10 @@ func TestTranspileFilter(t *testing.T) {
 
 		{
 			name:   "searchNgrams: 2-arg basic",
-			filter: `searchNgrams(display_name, "abc")`,
+			filter: `searchNgrams(display_name_tokens, "abc")`,
 			declarations: []filtering.DeclarationOption{
 				filtering.DeclareStandardFunctions(),
-				filtering.DeclareIdent("display_name", filtering.TypeString),
+				filtering.DeclareIdent("display_name_tokens", filtering.TypeString),
 				DeclareSearchNgramsFunction(),
 			},
 			expectedSQL: `(SEARCH_NGRAMS(display_name_tokens, @param_0))`,
@@ -177,10 +177,10 @@ func TestTranspileFilter(t *testing.T) {
 
 		{
 			name:   "searchNgrams: 5-arg all set",
-			filter: `searchNgrams(display_name, "abc", "en", 3, 0.8)`,
+			filter: `searchNgrams(display_name_tokens, "abc", "en", 3, 0.8)`,
 			declarations: []filtering.DeclarationOption{
 				filtering.DeclareStandardFunctions(),
-				filtering.DeclareIdent("display_name", filtering.TypeString),
+				filtering.DeclareIdent("display_name_tokens", filtering.TypeString),
 				DeclareSearchNgramsFunction(),
 			},
 			expectedSQL: `(SEARCH_NGRAMS(display_name_tokens, @param_0, ` +
@@ -196,10 +196,10 @@ func TestTranspileFilter(t *testing.T) {
 
 		{
 			name:   "searchNgrams: 5-arg skip language_tag",
-			filter: `searchNgrams(display_name, "abc", "", 3, 0.0)`,
+			filter: `searchNgrams(display_name_tokens, "abc", "", 3, 0.0)`,
 			declarations: []filtering.DeclarationOption{
 				filtering.DeclareStandardFunctions(),
-				filtering.DeclareIdent("display_name", filtering.TypeString),
+				filtering.DeclareIdent("display_name_tokens", filtering.TypeString),
 				DeclareSearchNgramsFunction(),
 			},
 			expectedSQL: `(SEARCH_NGRAMS(display_name_tokens, @param_0, min_ngrams => @param_1))`,
@@ -211,10 +211,10 @@ func TestTranspileFilter(t *testing.T) {
 
 		{
 			name:   "searchNgrams: 5-arg skip all optional",
-			filter: `searchNgrams(display_name, "abc", "", 0, 0.0)`,
+			filter: `searchNgrams(display_name_tokens, "abc", "", 0, 0.0)`,
 			declarations: []filtering.DeclarationOption{
 				filtering.DeclareStandardFunctions(),
-				filtering.DeclareIdent("display_name", filtering.TypeString),
+				filtering.DeclareIdent("display_name_tokens", filtering.TypeString),
 				DeclareSearchNgramsFunction(),
 			},
 			expectedSQL: `(SEARCH_NGRAMS(display_name_tokens, @param_0))`,
@@ -225,10 +225,10 @@ func TestTranspileFilter(t *testing.T) {
 
 		{
 			name:   "searchNgrams: combined with AND",
-			filter: `searchNgrams(display_name, "abc") AND author = "Karin Boye"`,
+			filter: `searchNgrams(display_name_tokens, "abc") AND author = "Karin Boye"`,
 			declarations: []filtering.DeclarationOption{
 				filtering.DeclareStandardFunctions(),
-				filtering.DeclareIdent("display_name", filtering.TypeString),
+				filtering.DeclareIdent("display_name_tokens", filtering.TypeString),
 				filtering.DeclareIdent("author", filtering.TypeString),
 				DeclareSearchNgramsFunction(),
 			},

spanfiltering/transpiler.go

@@ -16,10 +16,6 @@ import (
 // FunctionSearchNgrams is the function name for SEARCH_NGRAMS in filter expressions.
 const FunctionSearchNgrams = "searchNgrams"
 
-const (
-	tokenColumnSuffix = "_tokens"
-)
-
 // DeclareSearchNgramsFunction declares the searchNgrams function for use in filter expressions.
 // It declares two overloads:
 //   - 2-arg: searchNgrams(column, query) — required params only
@@ -400,12 +396,12 @@ func (t *Transpiler) transpileSearchNgramsCallExpr(e *expr.Expr) (spansql.BoolEx
 			callExpr.GetFunction(), len(args),
 		)
 	}
-	// Arg 0: column identifier → append _tokens suffix.
+	// Arg 0: column identifier
 	identExpr := args[0].GetIdentExpr()
 	if identExpr == nil {
 		return nil, fmt.Errorf("first argument to %s must be an identifier", callExpr.GetFunction())
 	}
-	tokenColumn := spansql.ID(identExpr.GetName() + tokenColumnSuffix)
+	tokenColumn := spansql.ID(identExpr.GetName())
 	// Arg 1: ngrams_query string, must be at least 2 characters.
 	queryConst := args[1].GetConstExpr()
 	if queryConst == nil {