243 lines
13 KiB
Python
243 lines
13 KiB
Python
"""Offline real-SDK qualification of speech through durable Hope runs."""
|
|
import asyncio
|
|
import json
|
|
import unittest
|
|
import httpx
|
|
from livekit.agents import llm
|
|
from agent.conversation_llm import ConversationLLM
|
|
|
|
class ConversationLLMTests(unittest.IsolatedAsyncioTestCase):
|
|
async def test_single_turn_only_and_completed_public_output(self):
|
|
requests, progress = [], []
|
|
def handler(request):
|
|
requests.append(request)
|
|
if request.method == 'POST':
|
|
return httpx.Response(202, json={'run': {'id': 'run1'}, 'message': {'id': 'user1'}})
|
|
events = [
|
|
('message', {'message': {'id':'a','role':'assistant','status':'streaming','content':'draft'}}),
|
|
('progress', {'message':'PRIVATE TOOL DETAILS'}),
|
|
('message', {'message': {'id':'a','role':'assistant','status':'completed','content':'Final answer'}}),
|
|
('done', {'run': {'id':'run1','status':'completed'}}),
|
|
]
|
|
body = ''.join('event: '+name+'\ndata: '+json.dumps(data)+'\n\n' for name,data in events)
|
|
return httpx.Response(200, text=body, headers={'Content-Type':'text/event-stream'})
|
|
async def feedback(payload): progress.append(payload)
|
|
bridge = ConversationLLM(control_url='https://control.test',secret='internal-secret',room_id='room1',on_progress=feedback)
|
|
await bridge.client.aclose()
|
|
bridge.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
context = llm.ChatContext([
|
|
llm.ChatMessage(role='system',content=['DO NOT FORWARD']),
|
|
llm.ChatMessage(role='user',content=['earlier']),
|
|
llm.ChatMessage(role='assistant',content=['earlier answer']),
|
|
llm.ChatMessage(role='user',content=['latest question']),
|
|
])
|
|
async with bridge.chat(chat_ctx=context) as stream:
|
|
chunks = [chunk async for chunk in stream]
|
|
self.assertEqual([c.delta.content for c in chunks], ['Final answer'])
|
|
payload=json.loads(requests[0].content)
|
|
self.assertEqual(payload['content'],'latest question')
|
|
self.assertEqual(payload['modality'],'voice')
|
|
self.assertNotIn('messages',payload)
|
|
self.assertEqual(requests[0].headers['X-Control-Secret'],'internal-secret')
|
|
self.assertEqual(progress[-1]['status'],'completed')
|
|
self.assertNotIn('PRIVATE',json.dumps(progress))
|
|
await bridge.aclose()
|
|
|
|
async def test_rejected_admission_has_no_answer_or_retry(self):
|
|
calls=[]
|
|
def handler(request):
|
|
calls.append(request)
|
|
return httpx.Response(403,json={'detail':'private diagnostics'})
|
|
async def progress(_): pass
|
|
bridge=ConversationLLM(control_url='https://control.test',secret='s',room_id='r',on_progress=progress)
|
|
await bridge.client.aclose()
|
|
bridge.client=httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
ctx=llm.ChatContext([llm.ChatMessage(role='user',content=['hello'])])
|
|
with self.assertRaises(Exception):
|
|
async with bridge.chat(chat_ctx=ctx) as stream:
|
|
async for _ in stream: pass
|
|
self.assertEqual(len(calls),1)
|
|
await bridge.aclose()
|
|
|
|
async def test_repeated_stream_uses_stable_user_turn_id(self):
|
|
payloads = []
|
|
def handler(request):
|
|
if request.method == 'POST':
|
|
payloads.append(json.loads(request.content))
|
|
return httpx.Response(202, json={'run': {'id': 'offline-stable-run'}})
|
|
body = ('event: message\ndata: {"message":{"role":"assistant","status":"completed","content":"Answer"}}\n\n'
|
|
'event: done\ndata: {"run":{"status":"completed"}}\n\n')
|
|
return httpx.Response(200, text=body)
|
|
async def progress(_): pass
|
|
bridge = ConversationLLM(control_url='https://offline.test', secret='offline', room_id='room', on_progress=progress)
|
|
await bridge.client.aclose()
|
|
bridge.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
context = llm.ChatContext([llm.ChatMessage(role='user', content=['Same physical turn'])])
|
|
try:
|
|
for _ in range(2):
|
|
async with bridge.chat(chat_ctx=context) as stream:
|
|
self.assertEqual([chunk.delta.content async for chunk in stream], ['Answer'])
|
|
self.assertEqual(payloads[0]['idempotency_key'], payloads[1]['idempotency_key'])
|
|
self.assertEqual(payloads[0]['idempotency_key'], context.items[-1].id)
|
|
finally:
|
|
await bridge.aclose()
|
|
|
|
async def test_cancel_uncertain_admission_retries_identical_key_and_cancels(self):
|
|
entered, release = asyncio.Event(), asyncio.Event()
|
|
payloads, cancelled_paths = [], []
|
|
async def handler(request):
|
|
if request.url.path.endswith('/messages'):
|
|
payloads.append(json.loads(request.content))
|
|
if len(payloads) == 1:
|
|
entered.set()
|
|
await release.wait()
|
|
raise httpx.ReadError('Offline lost admission receipt', request=request)
|
|
return httpx.Response(202, json={'run': {'id': 'offline-recovered-id'}})
|
|
if request.url.path.endswith('/cancel'):
|
|
cancelled_paths.append(request.url.path)
|
|
return httpx.Response(200, json={'run': {'status': 'cancelled'}})
|
|
self.fail('Cancelled admission must not attach an answer stream')
|
|
async def progress(_): pass
|
|
bridge = ConversationLLM(control_url='https://offline.test', secret='offline', room_id='bound-room', on_progress=progress)
|
|
await bridge.client.aclose()
|
|
bridge.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
context = llm.ChatContext([llm.ChatMessage(role='user', content=['Uncertain admission'])])
|
|
stream = bridge.chat(chat_ctx=context)
|
|
async def consume():
|
|
async with stream:
|
|
return [chunk async for chunk in stream]
|
|
consumer = asyncio.create_task(consume())
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 2)
|
|
closer = asyncio.create_task(stream.aclose())
|
|
await asyncio.sleep(.02)
|
|
release.set()
|
|
await asyncio.wait_for(closer, 2)
|
|
await asyncio.gather(consumer, return_exceptions=True)
|
|
self.assertEqual(len(payloads), 2)
|
|
self.assertEqual(payloads[0], payloads[1])
|
|
self.assertEqual(cancelled_paths, [
|
|
'/internal/chat/rooms/bound-room/operations/' + context.items[-1].id + '/cancel',
|
|
'/internal/chat/rooms/bound-room/runs/offline-recovered-id/cancel'])
|
|
finally:
|
|
release.set()
|
|
if not consumer.done(): consumer.cancel()
|
|
await asyncio.gather(consumer, return_exceptions=True)
|
|
await bridge.aclose()
|
|
|
|
async def test_cancel_blocked_admission_learns_and_cancels_remote_id(self):
|
|
admission_entered, release_admission, cancelled = asyncio.Event(), asyncio.Event(), asyncio.Event()
|
|
cancellations = []
|
|
async def handler(request):
|
|
if request.url.path.endswith('/messages'):
|
|
admission_entered.set()
|
|
await release_admission.wait()
|
|
return httpx.Response(202, json={'run': {'id': 'offline-admitted-run'}})
|
|
if request.url.path.endswith('/cancel'):
|
|
cancellations.append(request.url.path)
|
|
cancelled.set()
|
|
return httpx.Response(200, json={'run': {'status': 'cancelled'}})
|
|
return httpx.Response(200, text='event: done\ndata: {"run":{"status":"cancelled"}}\n\n')
|
|
async def progress(_): pass
|
|
bridge = ConversationLLM(control_url='https://offline.test', secret='offline', room_id='room', on_progress=progress)
|
|
await bridge.client.aclose()
|
|
bridge.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
context = llm.ChatContext([llm.ChatMessage(role='user', content=['Cancel during admission'])])
|
|
stream = bridge.chat(chat_ctx=context)
|
|
async def consume():
|
|
async with stream:
|
|
return [chunk async for chunk in stream]
|
|
consumer = asyncio.create_task(consume())
|
|
closer = second_closer = None
|
|
try:
|
|
await asyncio.wait_for(admission_entered.wait(), 2)
|
|
# Public SDK cancellation must not orphan an admission already accepted
|
|
# remotely but whose response has not reached the agent.
|
|
closer = asyncio.create_task(stream.aclose())
|
|
await asyncio.sleep(.02)
|
|
second_closer = asyncio.create_task(stream.aclose())
|
|
await asyncio.sleep(.02)
|
|
release_admission.set()
|
|
await asyncio.wait_for(asyncio.gather(closer, second_closer), 2)
|
|
await asyncio.gather(consumer, return_exceptions=True)
|
|
await asyncio.wait_for(cancelled.wait(), .3)
|
|
self.assertEqual(cancellations, [
|
|
'/internal/chat/rooms/room/operations/' + context.items[-1].id + '/cancel',
|
|
'/internal/chat/rooms/room/runs/offline-admitted-run/cancel'])
|
|
finally:
|
|
release_admission.set()
|
|
if closer and not closer.done():
|
|
closer.cancel()
|
|
await asyncio.gather(closer, return_exceptions=True)
|
|
if not consumer.done():
|
|
consumer.cancel()
|
|
await asyncio.gather(consumer, return_exceptions=True)
|
|
await bridge.aclose()
|
|
|
|
async def test_normal_lost_admission_receipt_persists_key_cancellation(self):
|
|
requests, progress_events = [], []
|
|
async def handler(request):
|
|
requests.append(request.url.path)
|
|
if request.url.path.endswith('/messages'):
|
|
raise httpx.ReadError('Offline accepted admission receipt lost', request=request)
|
|
if '/operations/' in request.url.path and request.url.path.endswith('/cancel'):
|
|
return httpx.Response(200, json={'ok': True})
|
|
self.fail('Unknown admission identity must not stream or invent a run stop')
|
|
async def progress(value): progress_events.append(value)
|
|
bridge = ConversationLLM(control_url='https://offline.test', secret='offline', room_id='bound-room', on_progress=progress)
|
|
await bridge.client.aclose()
|
|
bridge.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
context = llm.ChatContext([llm.ChatMessage(role='user', content=['Lost receipt without barge-in'])])
|
|
try:
|
|
with self.assertRaises(Exception):
|
|
async with bridge.chat(chat_ctx=context) as stream:
|
|
async for _ in stream:
|
|
self.fail('No answer is safe after uncertain admission failure')
|
|
self.assertEqual(requests, ['/internal/chat/rooms/bound-room/messages',
|
|
'/internal/chat/rooms/bound-room/operations/' + context.items[-1].id + '/cancel'])
|
|
self.assertEqual(progress_events[-1]['status'], 'failed')
|
|
finally:
|
|
await bridge.aclose()
|
|
|
|
async def test_key_cancel_receipt_prevents_late_rejected_admission_action(self):
|
|
entered, release, tombstoned = asyncio.Event(), asyncio.Event(), asyncio.Event()
|
|
requests, progress_events = [], []
|
|
async def handler(request):
|
|
requests.append(request.url.path)
|
|
if request.url.path.endswith('/messages'):
|
|
entered.set()
|
|
await release.wait()
|
|
self.assertTrue(tombstoned.is_set(), 'Persist operation cancellation before admission settles')
|
|
return httpx.Response(409, json={'detail': 'Operation cancelled'})
|
|
if '/operations/' in request.url.path and request.url.path.endswith('/cancel'):
|
|
tombstoned.set()
|
|
return httpx.Response(200, json={'ok': True})
|
|
self.fail('A rejected cancelled operation must never stream or cancel an invented run')
|
|
async def progress(value): progress_events.append(value)
|
|
bridge = ConversationLLM(control_url='https://offline.test', secret='offline', room_id='bound-room', on_progress=progress)
|
|
await bridge.client.aclose()
|
|
bridge.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
context = llm.ChatContext([llm.ChatMessage(role='user', content=['Cancel before admission'])])
|
|
stream = bridge.chat(chat_ctx=context)
|
|
async def consume():
|
|
async with stream:
|
|
return [chunk async for chunk in stream]
|
|
consumer = asyncio.create_task(consume())
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 2)
|
|
closer = asyncio.create_task(stream.aclose())
|
|
await asyncio.wait_for(tombstoned.wait(), 2)
|
|
release.set()
|
|
await asyncio.wait_for(closer, 2)
|
|
await asyncio.gather(consumer, return_exceptions=True)
|
|
self.assertEqual(requests, ['/internal/chat/rooms/bound-room/messages',
|
|
'/internal/chat/rooms/bound-room/operations/' + context.items[-1].id + '/cancel'])
|
|
self.assertEqual(progress_events[-1]['status'], 'failed')
|
|
finally:
|
|
release.set()
|
|
if not consumer.done(): consumer.cancel()
|
|
await asyncio.gather(consumer, return_exceptions=True)
|
|
await bridge.aclose()
|
|
|
|
if __name__=='__main__': unittest.main()
|