Introduce embedding provider switcher and full switch to Voyage AI for CI/CD

.github/workflows/_test-hcd.yml

@@ -13,7 +13,7 @@ env:
   AWS_ECR_ROLE_NAME: ${{ secrets.AWS_ECR_ROLE_NAME }}
   AWS_ECR_REPOSITORY: ${{ secrets.AWS_ECR_REPOSITORY }}
   AWS_ECR_REGISTRY: ${{ secrets.AWS_ECR_REGISTRY }}
-  AWS_ECR_HCD_IMAGE_TAG: "1.2.3"
+  AWS_ECR_HCD_IMAGE_TAG: "2.0.6"
 
 jobs:
   test:

docker-compose/docker-compose.yml

@@ -1,12 +1,12 @@
 services:
   hcd:
-    image: 559669398656.dkr.ecr.us-west-2.amazonaws.com/engops-shared/hcd/prod/hcd:1.2.3
+    image: 559669398656.dkr.ecr.us-west-2.amazonaws.com/engops-shared/hcd/prod/hcd:2.0.6
     networks:
       - stargate
     mem_limit: 2G
     environment:
       - MAX_HEAP_SIZE=1536M
-      - CLUSTER_NAME=hcd-1.2.3-cluster
+      - CLUSTER_NAME=hcd-2.0.6-cluster
       - DS_LICENSE=accept
       - HCD_AUTO_CONF_OFF=xcassandra.yaml
       - JVM_EXTRA_OPTS=-Dcassandra.sai.latest.version=ec
@@ -21,7 +21,7 @@ services:
       retries: 20
 
   data-api:
-    image: stargateio/data-api:v1.0.32
+    image: stargateio/data-api:v1.0.46
     depends_on:
       hcd:
         condition: service_healthy

test/DataStax.AstraDB.DataApi.IntegrationTests/EmbeddingProviderSwitcher.cs

@@ -0,0 +1,46 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace DataStax.AstraDB.DataApi.IntegrationTests;
+
+/// <summary>
+/// Static configuration on the embedding provider for all vectorize-related integration testing.
+/// </summary>
+public static class EmbeddingProviderSwitcher {
+
+    /// <summary>
+    /// Embedding provider to use.
+    /// </summary>
+    public const string Provider = "voyageAI";
+    /// <summary>
+    /// Embedding model to use.
+    /// </summary>
+    public const string ModelName = "voyage-2";
+
+    /// <summary>
+    /// A dimension value compatible with the provider/model choice.
+    /// </summary>
+    public const int Dimension = 1024;
+
+    /// <summary>
+    /// Name of the secret in Astra scoped to the DB being used to test.
+    /// </summary>
+    public const string KMSSecretName = "SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI";
+    /// <summary>
+    /// Name of the environment variable containing the embedding API key.
+    /// </summary>
+    public const string SecretEnvironmentVariableName = "HEADER_EMBEDDING_API_KEY_VOYAGEAI";
+}

test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs

@@ -91,8 +91,8 @@ public class SimpleObjectWithVectorizeAttribute
 
 [CollectionName("coll_SimpleObjectWithVectorizeShSecret")]
 [CollectionVectorize(
-    Provider = "openai", ModelName = "text-embedding-3-small",
-    AuthenticationPairs = new string[] {"providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_OPENAI"}
+    Provider = EmbeddingProviderSwitcher.Provider, ModelName = EmbeddingProviderSwitcher.ModelName,
+    AuthenticationPairs = new string[] {"providerKey", EmbeddingProviderSwitcher.KMSSecretName}
 )]
 public class SimpleObjectWithVectorizeAttributeShSecret
 {
@@ -106,12 +106,12 @@ public class SimpleObjectWithVectorizeAttributeShSecret
 [CollectionName("coll_SimpleObjectWithVectorizeShSecret2A")]
 [CollectionVector(
     SimilarityMetric.Euclidean,
-    123,
+    EmbeddingProviderSwitcher.Dimension,
     SourceModel="bert"
 )]
 [CollectionVectorize(
-    Provider = "openai", ModelName = "text-embedding-3-small",
-    AuthenticationPairs = new string[] {"providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_OPENAI"}
+    Provider = EmbeddingProviderSwitcher.Provider, ModelName = EmbeddingProviderSwitcher.ModelName,
+    AuthenticationPairs = new string[] {"providerKey", EmbeddingProviderSwitcher.KMSSecretName}
 )]
 public class SimpleObjectWithVectorizeAttributeShSecret2A
 {
@@ -123,7 +123,7 @@ public class SimpleObjectWithVectorizeAttributeShSecret2A
 }
 
 [CollectionName("coll_SimpleObjectWithVectorizeHeader")]
-[CollectionVectorize(Provider = "openai", ModelName = "text-embedding-3-small")]
+[CollectionVectorize(Provider = EmbeddingProviderSwitcher.Provider, ModelName = EmbeddingProviderSwitcher.ModelName)]
 public class SimpleObjectWithVectorizeAttributeHeader
 {
     [DocumentId]
@@ -279,7 +279,7 @@ public class RowBookVectorizeHeaderBased
 {
     [ColumnPrimaryKey(1)]
     public string Title { get; set; }
-    [ColumnVectorize("openai", "text-embedding-3-small", dimension: 1536)]
+    [ColumnVectorize(EmbeddingProviderSwitcher.Provider, EmbeddingProviderSwitcher.ModelName, dimension: EmbeddingProviderSwitcher.Dimension)]
     public object Author { get; set; }
     [ColumnPrimaryKey(2)]
     public int NumberOfPages { get; set; }
@@ -294,8 +294,8 @@ public class RowBookVectorizeSharedSecret
     [ColumnPrimaryKey(1)]
     public string Title { get; set; }
     [ColumnVectorize(
-        "openai", "text-embedding-3-small", dimension: 1536,
-        authenticationPairs: new string[] {"providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_OPENAI"}
+        EmbeddingProviderSwitcher.Provider, EmbeddingProviderSwitcher.ModelName, dimension: EmbeddingProviderSwitcher.Dimension,
+        authenticationPairs: new string[] {"providerKey", EmbeddingProviderSwitcher.KMSSecretName}
     )]
     public object Author { get; set; }
     [ColumnPrimaryKey(2)]
@@ -311,8 +311,8 @@ public class RowBookVectorizeSharedSecretWithParameters
     [ColumnPrimaryKey(1)]
     public string Title { get; set; }
     [ColumnVectorize(
-        "voyageAI", "voyage-2",
-        authenticationPairs: new string[] { "providerKey", "SHARED_SECRET_EMBEDDING_API_KEY_VOYAGEAI" },
+        EmbeddingProviderSwitcher.Provider, EmbeddingProviderSwitcher.ModelName,
+        authenticationPairs: new string[] { "providerKey", EmbeddingProviderSwitcher.KMSSecretName },
         parameterPairs: new object[] { "autoTruncate", false }
     )]
     public object Author { get; set; }
@@ -842,8 +842,8 @@ public class FakeDocument
 
 [CollectionName("coll_one_embedding_header_test")]
 [CollectionVectorize(
-    "voyageAI",
-    "voyage-2",
+    EmbeddingProviderSwitcher.Provider,
+    EmbeddingProviderSwitcher.ModelName,
     parameterPairs: new object[] { "autoTruncate", false }
 )]
 public class DocumentForEmbeddingHeaderTest
@@ -876,7 +876,7 @@ public class RowForEmbeddingHeaderTest
     [ColumnPrimaryKey(1)]
     public string Id { get; set; }
     [ColumnVectorize(
-        "voyageAI", "voyage-2", dimension: 1024,
+        EmbeddingProviderSwitcher.Provider, EmbeddingProviderSwitcher.ModelName, dimension: EmbeddingProviderSwitcher.Dimension,
         parameterPairs: new object[] { "autoTruncate", false }
     )]
     public object Name { get; set; }

test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs

@@ -552,11 +552,11 @@ await collection.InsertManyAsync(new List<SimpleObjectWithVector> {
         }
     }
 
-    // Requires VOYAGE embedding provider
-    [Fact(Skip="Should be run after exporting the environment variable quoted below")]
+    // Requires a certain embedding provider (see EmbeddingProviderSwitcher)
+    [Fact(Skip="Should be run after exporting the environment variable quoted in EmbeddingProviderSwitcher")]
     public async Task Test_CollectionEmbeddingHeaders()
     {
-        var embeddingAPIKey = Environment.GetEnvironmentVariable("HEADER_EMBEDDING_API_KEY_VOYAGEAI") ?? "kaboom";
+        var embeddingAPIKey = Environment.GetEnvironmentVariable(EmbeddingProviderSwitcher.SecretEnvironmentVariableName) ?? "kaboom";
         try
         {
             var collectionCr = await fixture.Database.CreateCollectionAsync<DocumentForEmbeddingHeaderTest>(

test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalTableTests.cs

@@ -1252,11 +1252,11 @@ await table.InsertManyAsync(new List<SimpleTwoColumnRow> {
         }
     }
 
-    // Requires VOYAGE embedding provider
-    [Fact(Skip="Should be run after exporting the environment variable quoted below")]
+    // Requires a certain embedding provider (see EmbeddingProviderSwitcher)
+    [Fact(Skip="Should be run after exporting the environment variable quoted in EmbeddingProviderSwitcher")]
     public async Task Test_TableEmbeddingHeaders()
     {
-        var embeddingAPIKey = Environment.GetEnvironmentVariable("HEADER_EMBEDDING_API_KEY_VOYAGEAI") ?? "kaboom";
+        var embeddingAPIKey = Environment.GetEnvironmentVariable(EmbeddingProviderSwitcher.SecretEnvironmentVariableName) ?? "kaboom";
         try
         {
             var tableCr = await fixture.Database.CreateTableAsync<RowForEmbeddingHeaderTest>(

test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionFARRCursorTests.cs

@@ -308,7 +308,8 @@ public async Task Test_CollectionVectorFARRCursor_RerankingHeader()
         await Assert.ThrowsAsync<CommandException>(async () => await cur.ToListAsync());
     }
 
-    [Fact]
+    // TODO: re-enable this test when rerank-override is available in the target Data API version
+    [Fact(Skip="This test is disabled until Rerak-Override is released.")]
     public async Task Test_CollectionVectorFARRRerankOverride()
     {
         var filledCollection = _fixture.FilledVectorCollection;
@@ -358,7 +359,8 @@ await Assert.ThrowsAsync<CommandException>(async () =>
         );
     }
 
-    [Fact]
+    // TODO: re-enable this test when rerank-override is available in the target Data API version
+    [Fact(Skip="This test is disabled until Rerak-Override is released.")]
     public async Task Test_CollectionVectorizeFARRRerankOverride()
     {
         var filledCollection = _fixture.FilledVectorizeCollection;