ci: Fix L0_triton_cli_test_vllm--base (#122)

Author: pskiran1

README.md

@@ -356,7 +356,7 @@ curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/
 }'
 
 # Profile model with GenAI-Perf
-triton profile -m llama-3.1-8b-instruct --service-kind openai --endpoint-type chat --url localhost:9000 --streaming
+triton profile -m llama-3.1-8b-instruct --endpoint-type chat --url localhost:9000 --streaming
 ```
 
 ## Additional Dependencies for Custom Environments

pyproject.toml

@@ -51,7 +51,7 @@ dependencies = [
     "grpcio>=1.67.0",
     # Use explicit client version matching genai-perf version for tagged release
     "tritonclient[all] == 2.55.0",
-    "genai-perf @ git+https://github.com/triton-inference-server/perf_analyzer.git@r25.02#subdirectory=genai-perf",
+    "genai-perf @ git+https://github.com/triton-inference-server/perf_analyzer.git@main#subdirectory=genai-perf",
     # Misc deps
     "directory-tree == 0.0.4", # may remove in future
     # https://github.com/docker/docker-py/issues/3256#issuecomment-2376439000

src/triton_cli/repository.py

@@ -315,7 +315,6 @@ def __generate_vllm_model(self, huggingface_id: str):
         model_contents = json.dumps(
             {
                 "model": huggingface_id,
-                "disable_log_requests": True,
                 "gpu_memory_utilization": 0.85,
             }
         )

tests/test_e2e.py

@@ -89,9 +89,7 @@ def test_tensorrtllm_openai_e2e(self, trtllm_openai_server):
         TritonCommands._clear()
         TritonCommands._import(model, source=source, backend="tensorrtllm")
         trtllm_openai_server.start()
-        TritonCommands._profile(
-            model, service_kind="openai", endpoint_type="chat", url="localhost:9000"
-        )
+        TritonCommands._profile(model, endpoint_type="chat", url="localhost:9000")
 
     @pytest.mark.skipif(
         os.environ.get("IMAGE_KIND") != "VLLM", reason="Only run for VLLM image"
@@ -141,9 +139,7 @@ def test_vllm_openai_e2e(self, vllm_openai_server):
         TritonCommands._clear()
         TritonCommands._import(model, source=source)
         vllm_openai_server.start()
-        TritonCommands._profile(
-            model, service_kind="openai", endpoint_type="chat", url="localhost:9000"
-        )
+        TritonCommands._profile(model, endpoint_type="chat", url="localhost:9000")
 
     @pytest.mark.skipif(
         os.environ.get("CI_PIPELINE") == "GITHUB_ACTIONS",

tests/test_models/mock_llm/config.pbtxt

@@ -37,6 +37,18 @@ input [
     name: "max_tokens"
     data_type: TYPE_INT32
     dims: [ 1 ]
+  },
+  {
+    name: "exclude_input_in_output"
+    data_type: TYPE_BOOL
+    dims: [ 1 ]
+    optional: true
+  },
+  {
+    name: "stream"
+    data_type: TYPE_BOOL
+    dims: [ 1 ]
+    optional: true
   }
 ]
 output [

tests/utils.py

@@ -73,16 +73,30 @@ def _infer(model, prompt=None, protocol=None):
             args += ["-i", protocol]
         run(args)
 
-    def _profile(model, backend=None, service_kind=None, endpoint_type=None, url=None):
+    def _profile(model, backend=None, endpoint_type=None, url=None):
         args = ["profile", "-m", model]
         if backend:
             args += ["--backend", backend]
-        if service_kind:
-            args += ["--service-kind", service_kind]
+
         if endpoint_type:
             args += ["--endpoint-type", endpoint_type]
+
         if url:
             args += ["--url", url]
+
+        # Map each model to its corresponding HuggingFace tokenizer.
+        # For the mock model, use a real tokenizer since "mock_llm" is not available on HuggingFace.
+        tokenizer_map = {
+            "mock_llm": "gpt2",
+            "gpt2": "gpt2",
+            "llama-3.1-8b-instruct": "meta-llama/Llama-3.1-8B-Instruct",
+            "llama-2-7b-chat": "meta-llama/Llama-2-7b-chat-hf",
+            "llama-2-7b": "meta-llama/Llama-2-7b-hf",
+        }
+
+        if model in tokenizer_map:
+            args += ["--tokenizer", tokenizer_map[model]]
+
         # NOTE: With default parameters, genai-perf may take upwards of 1m30s or 2m to run,
         # so limit the genai-perf run with --request-count to reduce time for testing purposes.
         args += ["--synthetic-input-tokens-mean", "100", "--", "--request-count", "10"]