Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 24 additions & 3 deletions evalbench/generators/models/grpc_proxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import traceback
from generators.models.generator import QueryGenerator
from evalproto import eval_request_pb2
from util.context import rpc_id_var

PROXY_QUEUES = {}

Expand All @@ -22,6 +23,28 @@ class GrpcProxyModel(QueryGenerator):
def __init__(self, config):
super().__init__(config)
self.name = "grpc_proxy"
# The Interact handler puts the session ID into the config.
self.session_id = config.get("session_id")

def _session_queues(self):
"""Returns the queues of the session that owns this request.

Concurrent sessions must not share queues. A model that has no
session ID uses the RPC ID of the caller, or the only active session.
"""
if self.session_id is not None:
if self.session_id not in PROXY_QUEUES:
raise RuntimeError(
f"The proxy session {self.session_id!r} is not active.")
return PROXY_QUEUES[self.session_id]
rpc_id = rpc_id_var.get()
if rpc_id in PROXY_QUEUES:
return PROXY_QUEUES[rpc_id]
if len(PROXY_QUEUES) == 1:
return next(iter(PROXY_QUEUES.values()))
raise RuntimeError(
f"Cannot select the proxy session: {len(PROXY_QUEUES)} sessions "
"are active and the model has no session ID.")

def generate_internal(self, prompt: str) -> str:
# This method seems unused in the bidi flow, can be left as pass
Expand All @@ -39,9 +62,7 @@ def generate(self, eval_output) -> dict:
if not PROXY_QUEUES:
raise RuntimeError("PROXY_QUEUES is empty!")

# Assuming single session per proxy for now
session_id = list(PROXY_QUEUES.keys())[0]
in_queues_dict, out_queue = PROXY_QUEUES[session_id]
in_queues_dict, out_queue = self._session_queues()

def get_val(obj, *keys, default=None):
for k in keys:
Expand Down
82 changes: 82 additions & 0 deletions evalbench/test/grpc_proxy_session_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
"""Tests for the session queues of the gRPC proxy model."""

import contextvars
import queue
import threading
import unittest
from types import SimpleNamespace

from generators.models.grpc_proxy import PROXY_QUEUES, GrpcProxyModel
from util.context import rpc_id_var


class GrpcProxySessionTest(unittest.TestCase):

def setUp(self):
PROXY_QUEUES.clear()
self.addCleanup(PROXY_QUEUES.clear)
self._stop = threading.Event()
self.addCleanup(self._stop.set)

def _start_session(self, session_id):
"""Adds a session and a client thread that answers its requests."""
inboxes, out_queue = {}, queue.Queue()
PROXY_QUEUES[session_id] = (inboxes, out_queue)

def client():
while not self._stop.is_set():
try:
request = out_queue.get(timeout=0.1)
except queue.Empty:
continue
inboxes[request.conversation_id].put(SimpleNamespace(
generated_sql=f"sql from {session_id}",
generated_nl_response="",
))

threading.Thread(target=client, daemon=True).start()

@staticmethod
def _generate(model, conv_id="c1"):
return model.generate({"id": conv_id, "nl_prompt": "q"})["generated_sql"]

def test_model_uses_the_queues_of_its_session(self):
self._start_session("a")
self._start_session("b")

for session_id in ("b", "a"):
model = GrpcProxyModel({"session_id": session_id})
self.assertEqual(self._generate(model), f"sql from {session_id}")

def test_model_without_a_session_id_uses_the_rpc_id(self):
self._start_session("a")
self._start_session("b")
model = GrpcProxyModel({})

def run():
rpc_id_var.set("b")
return self._generate(model)

self.assertEqual(contextvars.copy_context().run(run), "sql from b")

def test_model_without_identity_uses_the_only_session(self):
self._start_session("a")

self.assertEqual(self._generate(GrpcProxyModel({})), "sql from a")

def test_model_without_identity_fails_with_two_sessions(self):
self._start_session("a")
self._start_session("b")

with self.assertRaisesRegex(RuntimeError, "2 sessions"):
self._generate(GrpcProxyModel({}))

def test_model_fails_after_its_session_ended(self):
self._start_session("a")

with self.assertRaisesRegex(RuntimeError, "not active"):
self._generate(GrpcProxyModel({"session_id": "ended"}))


if __name__ == "__main__":
unittest.main()
Loading