Gemini Testing via VCR (#1770)

* Delete old replay files

* Replace Gemini test setup with VCR

* Improve behavior of Gemini tests

* Add additional logic to VCR setup

* Record new Gemini responses with VCR

* Improved megalinter filtering

* User helper to drop recorded headers

* Clean up comment

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: TimPansino

.mega-linter.yml

@@ -25,7 +25,15 @@ CLEAR_REPORT_FOLDER: true
 VALIDATE_ALL_CODEBASE: true
 IGNORE_GITIGNORED_FILES: true
 FAIL_IF_MISSING_LINTER_IN_FLAVOR: true
-FILTER_REGEX_EXCLUDE: "(.*/?packages/.*)" # Ignore packages directories
+FILTER_REGEX_EXCLUDE: "(.*cassette.*\\.ya?ml)" # Ignore recorded cassette.yaml files
+ADDITIONAL_EXCLUDED_DIRECTORIES:
+  - .pytest_cache
+  - .ruff_cache
+  - .tox
+  - .venv
+  - megalinter-reports
+  - newrelic.egg-info
+  - newrelic/packages
 
 ENABLE_LINTERS: # If you use ENABLE_LINTERS variable, all other linters will be disabled by default
   - ACTION_ACTIONLINT

tests/mlmodel_gemini/cassette.yaml

@@ -18,6 +18,7 @@
 import google.genai
 import pytest
 from testing_support.fixture.event_loop import event_loop as loop
+from testing_support.fixture.vcr import *  # noqa: F403
 from testing_support.fixtures import (
     collector_agent_registration_fixture,
     collector_available_fixture,
@@ -46,61 +47,27 @@
 
 
 @pytest.fixture
-def gemini_client(replay_id, is_vertex):
+def gemini_client(vcr_recording, is_vertex):
     """
-    This configures the Gemini client to use a ReplayApiClient which will either record or replay responses depending
-    on the mode. The mode can be controlled by setting NEW_RELIC_TESTING_RECORD_GEMINI_RESPONSES=1 as an environment
-    variable to run using the real Gemini backend. (Default: mocking)
-    """
-    from newrelic.core.config import _environ_as_bool
+    This configures the Gemini client to use a fake API key when replaying responses through VCR.
 
-    record_mode = _environ_as_bool("NEW_RELIC_TESTING_RECORD_GEMINI_RESPONSES", False)
-    # Auto mode will record any missing files, but still replay existing files.
-    # This allows us to add new tests without having to re-record everything.
-    # If you need to re-record everything, you can delete the existing replays completely.
-    replay_client_mode = "auto" if record_mode else "replay"
+    To record new responses, set a valid API key and run pytest with the flag --record-mode=new_episodes.
+    """
 
-    if record_mode:
+    if vcr_recording:
         google_api_key = os.environ.get("GOOGLE_API_KEY")
         if not google_api_key:
             raise RuntimeError("GOOGLE_API_KEY environment variable required.")
     else:
-        google_api_key = os.environ["GOOGLE_API_KEY"] = "GEMINI_API_KEY"
-
-    # Set the replay directory to a location in this test suite.
-    replay_dir = Path(__file__).parent / "replays"
-    replay_dir.mkdir(exist_ok=True)  # Recreate this directory if it's missing for recording purposes
-    os.environ["GOOGLE_GENAI_REPLAYS_DIRECTORY"] = str(replay_dir)
+        google_api_key = os.environ["GOOGLE_API_KEY"] = "FAKE_GEMINI_API_KEY"
 
-    # Monkeypatch the Gemini client to use the replay client which will either record or replay responses depending on the mode.
-    replay_client = google.genai._replay_api_client.ReplayApiClient(mode=replay_client_mode, replay_id=replay_id)
-    google.genai.client.Client._get_api_client = lambda self, *args, **kwargs: replay_client
     gemini_client = google.genai.Client(api_key=google_api_key, vertexai=is_vertex)
 
     yield gemini_client
 
-    gemini_client._api_client.close()
     gemini_client.close()
 
 
-@pytest.fixture
-def replay_id():
-    pytest_name = os.environ.get("PYTEST_CURRENT_TEST").split("::")
-    test_module = Path(pytest_name[0]).with_suffix("").name
-    test_name, test_params = pytest_name[-1].split(" ")[0].split("[")
-
-    # Don't use the actual parameter string in the replay ID to avoid having different replays for certain parameters
-    # like is_async and to avoid issues with sorting. We also can't directly use the fixtures or we will cause the
-    # embedding tests to run despite not having streaming or chat parameters. Instead we will just infer the info
-    # from the test names and reproduce it in a standardized format.
-    is_streaming = "stream" in test_params
-    is_chat = "chat" in test_params
-    is_vertex = "vertex" in test_params
-    test_params_suffix = f"{'streaming' if is_streaming else 'invoke'}-{'chat' if is_chat else 'model'}-{'vertex' if is_vertex else 'standard'}"
-
-    return f"{test_module}/{test_name}/{test_params_suffix}"
-
-
 @pytest.fixture(scope="session", params=["standard", "vertex"])
 def is_vertex(request):
     return request.param == "vertex"