Pin LiteLLM to patched 1.84.3 release

eval_protocol/litellm_compat.py

@@ -0,0 +1,14 @@
+import asyncio
+
+
+async def allow_litellm_logging_to_start() -> None:
+    """Let LiteLLM claim queued logging callbacks before the event loop can close.
+
+    LiteLLM 1.80+ queues coroutine objects in a process-global logging worker.
+    When pytest replaces a function-scoped event loop before the worker claims a
+    callback, LiteLLM drops that coroutine while rebinding the queue. Two event
+    loop turns let the worker wrap the callback in a task so normal loop
+    shutdown can cancel it cleanly.
+    """
+    await asyncio.sleep(0)
+    await asyncio.sleep(0)

eval_protocol/mcp/execution/policy.py

@@ -18,6 +18,8 @@
 from litellm.caching.in_memory_cache import InMemoryCache
 from litellm.caching.redis_cache import RedisCache
 
+from eval_protocol.litellm_compat import allow_litellm_logging_to_start
+
 from .base_policy import LLMBasePolicy
 
 logger = logging.getLogger(__name__)
@@ -205,6 +207,8 @@ async def _make_llm_call(self, messages: List[Dict[str, Any]], tools: List[Dict[
             else:
                 response = await acompletion(model=self.model_id, **request_params)
 
+            await allow_litellm_logging_to_start()
+
             assert response is not None, "Response is None"
             assert isinstance(response, ModelResponse), "Response should be ModelResponse"
 

eval_protocol/proxy/docker-compose.yml

@@ -15,7 +15,7 @@ services:
 
   # LiteLLM Backend - Handles actual LLM proxying
   litellm-backend:
-    image: litellm/litellm:v1.77.3-stable
+    image: litellm/litellm:v1.84.3
     platform: linux/amd64
     container_name: litellm-backend
     command: ["--config", "/app/config.yaml", "--port", "4000", "--host", "0.0.0.0"]

eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -13,6 +13,7 @@
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 
 from eval_protocol.dataset_logger import default_logger
+from eval_protocol.litellm_compat import allow_litellm_logging_to_start
 from eval_protocol.models import EvaluationRow, Message
 from openai.types import CompletionUsage
 from eval_protocol.pytest.rollout_processor import RolloutProcessor
@@ -129,6 +130,8 @@ async def process_row(row: EvaluationRow) -> EvaluationRow:
             else:
                 response = await acompletion(**request_params)
 
+            await allow_litellm_logging_to_start()
+
             assert response is not None, "Response is None"
             assert isinstance(response, ModelResponse), "Response should be ModelResponse"
             assert isinstance(response.choices[0], Choices), "Response choice should be a Choices"

pyproject.toml

@@ -31,7 +31,7 @@ dependencies = [
     "omegaconf>=2.3.0",
     "httpx>=0.24.0",
     "anthropic>=0.59.0",
-    "litellm<1.75.0",
+    "litellm==1.84.3",
     "pytest>=6.0.0",
     "pytest-asyncio>=0.21.0",
     "peewee>=3.18.2",

tests/test_litellm_event_loop_compatibility.py

@@ -0,0 +1,81 @@
+import asyncio
+
+import litellm
+import pytest
+
+from eval_protocol.dataset_logger import default_logger
+from eval_protocol.litellm_compat import allow_litellm_logging_to_start
+from eval_protocol.mcp.execution.policy import LiteLLMPolicy
+from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.pytest.default_single_turn_rollout_process import SingleTurnRolloutProcessor
+from eval_protocol.pytest.exception_config import get_default_exception_handler_config
+from eval_protocol.pytest.types import RolloutProcessorConfig
+from vendor.tau2.data_model.message import UserMessage
+from vendor.tau2.utils.llm_utils import generate
+
+
+@pytest.mark.parametrize("index", range(4))
+@pytest.mark.asyncio
+async def test_acompletion_across_pytest_event_loops(index: int) -> None:
+    response = await litellm.acompletion(
+        model="openai/gpt-4o-mini",
+        messages=[{"role": "user", "content": "ping"}],
+        api_key="test",
+        mock_response=f"ok-{index}",
+    )
+    await allow_litellm_logging_to_start()
+
+    assert response.choices[0].message.content == f"ok-{index}"
+
+
+@pytest.mark.parametrize(("index", "stream"), [(0, False), (1, True), (2, False), (3, True)])
+@pytest.mark.asyncio
+async def test_single_turn_processor_across_pytest_event_loops(index: int, stream: bool) -> None:
+    config = RolloutProcessorConfig(
+        completion_params={
+            "model": "openai/gpt-4o-mini",
+            "api_key": "test",
+            "mock_response": f"single-turn-{index}",
+            "stream": stream,
+        },
+        mcp_config_path="",
+        semaphore=asyncio.Semaphore(1),
+        server_script_path=None,
+        steps=1,
+        logger=default_logger,
+        exception_handler_config=get_default_exception_handler_config(),
+    )
+    row = EvaluationRow(messages=[Message(role="user", content="ping")])
+
+    result = await SingleTurnRolloutProcessor()([row], config)[0]
+
+    assert result.messages[-1].content == f"single-turn-{index}"
+
+
+@pytest.mark.parametrize(("index", "stream"), [(0, False), (1, True), (2, False), (3, True)])
+@pytest.mark.asyncio
+async def test_litellm_policy_across_pytest_event_loops(index: int, stream: bool) -> None:
+    policy = LiteLLMPolicy(
+        model_id="openai/gpt-4o-mini",
+        use_caching=False,
+        api_key="test",
+        mock_response=f"policy-{index}",
+        stream=stream,
+    )
+
+    result = await policy._make_llm_call([{"role": "user", "content": "ping"}], tools=[])
+
+    assert result["choices"][0]["message"]["content"] == f"policy-{index}"
+
+
+@pytest.mark.parametrize("index", range(2))
+@pytest.mark.asyncio
+async def test_tau2_generate_across_pytest_event_loops(index: int) -> None:
+    result = await generate(
+        model="openai/gpt-4o-mini",
+        messages=[UserMessage(role="user", content="ping")],
+        api_key="test",
+        mock_response=f"tau2-{index}",
+    )
+
+    assert result.content == f"tau2-{index}"

tests/test_litellm_policy_provider_fields.py

@@ -16,7 +16,7 @@ async def test_litellm_policy_surfaces_provider_specific_reasoning_details(monke
     # Define a fake ModelResponse base class and patch the module's ModelResponse
     class FakeModelResponseBase: ...
 
-    policy_mod.ModelResponse = FakeModelResponseBase
+    monkeypatch.setattr(policy_mod, "ModelResponse", FakeModelResponseBase)
 
     async def fake_acompletion(*args, **kwargs):
         # This mimics the LiteLLM Message object shape we rely on in policy._make_llm_call