Merge pull request #2140 from codeflash-ai/cf-fix-concurrency-measurement

fix: measure individual call durations in sequential phase of concurrency decorator

Author: aseembits93

codeflash/code_utils/codeflash_wrap_decorator.py

@@ -181,13 +181,15 @@ async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
         test_function = os.environ.get("CODEFLASH_TEST_FUNCTION", "")
         loop_index = os.environ.get("CODEFLASH_LOOP_INDEX", "0")
 
-        # Phase 1: Sequential execution timing
+        # Phase 1: Sequential execution timing — sum individual call durations
+        # to exclude event loop scheduling overhead between iterations
+        sequential_time = 0
         gc.disable()
         try:
-            seq_start = time.perf_counter_ns()
             for _ in range(concurrency_factor):
+                _t0 = time.perf_counter_ns()
                 result = await func(*args, **kwargs)
-            sequential_time = time.perf_counter_ns() - seq_start
+                sequential_time += time.perf_counter_ns() - _t0
         finally:
             gc.enable()
 

codeflash/code_utils/instrument_existing_tests.py

@@ -1643,12 +1643,13 @@ async def async_wrapper(*args, **kwargs):
         test_class_name = os.environ.get("CODEFLASH_TEST_CLASS", "")
         test_function = os.environ.get("CODEFLASH_TEST_FUNCTION", "")
         loop_index = os.environ.get("CODEFLASH_LOOP_INDEX", "0")
+        sequential_time = 0
         gc.disable()
         try:
-            seq_start = time.perf_counter_ns()
             for _ in range(concurrency_factor):
+                _t0 = time.perf_counter_ns()
                 result = await func(*args, **kwargs)
-            sequential_time = time.perf_counter_ns() - seq_start
+                sequential_time += time.perf_counter_ns() - _t0
         finally:
             gc.enable()
         gc.disable()

pyproject.toml

@@ -221,9 +221,13 @@ plugins = ["pydantic.mypy"]
 exclude = ["tests/", "code_to_optimize/", "pie_test_set/", "experiments/"]
 
 [[tool.mypy.overrides]]
-module = ["jedi", "jedi.api.classes", "inquirer", "inquirer.themes", "numba"]
+module = ["jedi", "jedi.api.classes", "inquirer", "inquirer.themes", "numba", "dill"]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+module = ["codeflash.code_utils.codeflash_wrap_decorator", "codeflash.code_utils.instrument_existing_tests", "codeflash.optimization.optimizer"]
+disable_error_code = ["attr-defined", "return-value", "no-untyped-call", "no-untyped-def", "arg-type", "assignment", "var-annotated", "no-any-return", "call-overload", "union-attr", "unreachable", "list-item"]
+
 [tool.pydantic-mypy]
 init_forbid_extra = true
 init_typed = true