diff --git a/evalbench/generators/models/grpc_proxy.py b/evalbench/generators/models/grpc_proxy.py index 78e20b55..d8221752 100644 --- a/evalbench/generators/models/grpc_proxy.py +++ b/evalbench/generators/models/grpc_proxy.py @@ -14,6 +14,7 @@ import traceback from generators.models.generator import QueryGenerator from evalproto import eval_request_pb2 +from util.context import rpc_id_var PROXY_QUEUES = {} @@ -22,6 +23,28 @@ class GrpcProxyModel(QueryGenerator): def __init__(self, config): super().__init__(config) self.name = "grpc_proxy" + # The Interact handler puts the session ID into the config. + self.session_id = config.get("session_id") + + def _session_queues(self): + """Returns the queues of the session that owns this request. + + Concurrent sessions must not share queues. A model that has no + session ID uses the RPC ID of the caller, or the only active session. + """ + if self.session_id is not None: + if self.session_id not in PROXY_QUEUES: + raise RuntimeError( + f"The proxy session {self.session_id!r} is not active.") + return PROXY_QUEUES[self.session_id] + rpc_id = rpc_id_var.get() + if rpc_id in PROXY_QUEUES: + return PROXY_QUEUES[rpc_id] + if len(PROXY_QUEUES) == 1: + return next(iter(PROXY_QUEUES.values())) + raise RuntimeError( + f"Cannot select the proxy session: {len(PROXY_QUEUES)} sessions " + "are active and the model has no session ID.") def generate_internal(self, prompt: str) -> str: # This method seems unused in the bidi flow, can be left as pass @@ -39,9 +62,7 @@ def generate(self, eval_output) -> dict: if not PROXY_QUEUES: raise RuntimeError("PROXY_QUEUES is empty!") - # Assuming single session per proxy for now - session_id = list(PROXY_QUEUES.keys())[0] - in_queues_dict, out_queue = PROXY_QUEUES[session_id] + in_queues_dict, out_queue = self._session_queues() def get_val(obj, *keys, default=None): for k in keys: diff --git a/evalbench/test/grpc_proxy_session_test.py b/evalbench/test/grpc_proxy_session_test.py new file mode 100644 index 00000000..9e94453b --- /dev/null +++ b/evalbench/test/grpc_proxy_session_test.py @@ -0,0 +1,82 @@ +"""Tests for the session queues of the gRPC proxy model.""" + +import contextvars +import queue +import threading +import unittest +from types import SimpleNamespace + +from generators.models.grpc_proxy import PROXY_QUEUES, GrpcProxyModel +from util.context import rpc_id_var + + +class GrpcProxySessionTest(unittest.TestCase): + + def setUp(self): + PROXY_QUEUES.clear() + self.addCleanup(PROXY_QUEUES.clear) + self._stop = threading.Event() + self.addCleanup(self._stop.set) + + def _start_session(self, session_id): + """Adds a session and a client thread that answers its requests.""" + inboxes, out_queue = {}, queue.Queue() + PROXY_QUEUES[session_id] = (inboxes, out_queue) + + def client(): + while not self._stop.is_set(): + try: + request = out_queue.get(timeout=0.1) + except queue.Empty: + continue + inboxes[request.conversation_id].put(SimpleNamespace( + generated_sql=f"sql from {session_id}", + generated_nl_response="", + )) + + threading.Thread(target=client, daemon=True).start() + + @staticmethod + def _generate(model, conv_id="c1"): + return model.generate({"id": conv_id, "nl_prompt": "q"})["generated_sql"] + + def test_model_uses_the_queues_of_its_session(self): + self._start_session("a") + self._start_session("b") + + for session_id in ("b", "a"): + model = GrpcProxyModel({"session_id": session_id}) + self.assertEqual(self._generate(model), f"sql from {session_id}") + + def test_model_without_a_session_id_uses_the_rpc_id(self): + self._start_session("a") + self._start_session("b") + model = GrpcProxyModel({}) + + def run(): + rpc_id_var.set("b") + return self._generate(model) + + self.assertEqual(contextvars.copy_context().run(run), "sql from b") + + def test_model_without_identity_uses_the_only_session(self): + self._start_session("a") + + self.assertEqual(self._generate(GrpcProxyModel({})), "sql from a") + + def test_model_without_identity_fails_with_two_sessions(self): + self._start_session("a") + self._start_session("b") + + with self.assertRaisesRegex(RuntimeError, "2 sessions"): + self._generate(GrpcProxyModel({})) + + def test_model_fails_after_its_session_ended(self): + self._start_session("a") + + with self.assertRaisesRegex(RuntimeError, "not active"): + self._generate(GrpcProxyModel({"session_id": "ended"})) + + +if __name__ == "__main__": + unittest.main()