189 lines
7.3 KiB
Python
189 lines
7.3 KiB
Python
"""Offline cancellation/recovery regressions; no HTTP, NAS, secrets or model calls."""
|
|
import asyncio
|
|
import functools
|
|
|
|
import pytest
|
|
|
|
|
|
def offline_async(function):
|
|
@functools.wraps(function)
|
|
def run(*args, **kwargs):
|
|
return asyncio.run(function(*args, **kwargs))
|
|
return run
|
|
|
|
from control.accounts import Accounts, token_hash
|
|
from control.chat_runs import ChatRuns
|
|
from control.chat_store import ChatConflict, ChatStore
|
|
|
|
|
|
class BlockedAdapter:
|
|
available = True
|
|
approvals = False
|
|
|
|
def __init__(self, *, block_start=False, fail_stop=False):
|
|
self.block_start = block_start
|
|
self.fail_stop = fail_stop
|
|
self.start_entered = asyncio.Event()
|
|
self.start_release = asyncio.Event()
|
|
self.stop_entered = asyncio.Event()
|
|
self.stream_entered = asyncio.Event()
|
|
self.stream_release = asyncio.Event()
|
|
self.starts = []
|
|
self.stops = []
|
|
|
|
async def start(self, owner, rid, payload):
|
|
self.starts.append(rid)
|
|
self.start_entered.set()
|
|
if self.block_start:
|
|
await self.start_release.wait()
|
|
return {'run_id': 'offline-upstream'}
|
|
|
|
async def stop(self, owner, upstream):
|
|
self.stops.append(upstream)
|
|
self.stop_entered.set()
|
|
if self.fail_stop:
|
|
raise RuntimeError('Offline simulated unavailable stop')
|
|
|
|
async def status(self, owner, upstream):
|
|
return {'status': 'cancelled' if self.stops and not self.fail_stop else 'running'}
|
|
|
|
async def events(self, owner, upstream):
|
|
self.stream_entered.set()
|
|
await self.stream_release.wait()
|
|
yield {'event': 'run.cancelled'}
|
|
|
|
|
|
@pytest.fixture
|
|
def execution(tmp_path):
|
|
accounts = Accounts(str(tmp_path / 'offline.sqlite'))
|
|
user = accounts.create('review', 'Offline Review', 'user', 'offline-review-password')
|
|
accounts.update(user['id'], status='ready')
|
|
token, _ = accounts.issue_session(user['id'], 3600, user['password_hash'])
|
|
store = ChatStore(accounts)
|
|
cid = store.create_conversation(user['id'])['id']
|
|
operation = store.send(user['id'], cid, 'Offline request', 'offline-key')
|
|
yield accounts, store, user['id'], token, cid, operation['run']['id']
|
|
accounts.close()
|
|
|
|
|
|
def coordinator(execution, adapter, *, upstream=None):
|
|
accounts, store, owner, token, cid, rid = execution
|
|
runs = ChatRuns(accounts, store, adapter=adapter)
|
|
runs.metadata(owner, rid, {'conversation_id': cid, 'payload': {'input': 'Offline request'},
|
|
'session_hash': token_hash(token), 'room': None,
|
|
'upstream_id': upstream, 'assistant_id': None})
|
|
return runs
|
|
|
|
|
|
@offline_async
|
|
async def test_cancel_during_blocked_admission_stops_learned_upstream(execution):
|
|
_, store, owner, _, _, rid = execution
|
|
adapter = BlockedAdapter(block_start=True)
|
|
runs = coordinator(execution, adapter)
|
|
runs.schedule(owner, rid)
|
|
try:
|
|
await asyncio.wait_for(adapter.start_entered.wait(), 2)
|
|
await runs.cancel(owner, rid)
|
|
assert not runs.tasks[rid].done(), 'Do not abort admission before learning its upstream ID'
|
|
adapter.start_release.set()
|
|
await asyncio.wait_for(runs.tasks[rid], 2)
|
|
assert adapter.starts == [rid]
|
|
assert adapter.stops, 'Cancellation must stop the execution admitted during cancellation'
|
|
assert store.get_run(owner, rid)['run']['status'] == 'cancelled'
|
|
finally:
|
|
adapter.start_release.set()
|
|
await runs.stop()
|
|
|
|
|
|
@offline_async
|
|
async def test_stop_failure_keeps_execution_slot_and_durable_intent(execution):
|
|
_, store, owner, _, _, rid = execution
|
|
adapter = BlockedAdapter(fail_stop=True)
|
|
runs = coordinator(execution, adapter, upstream='offline-upstream')
|
|
store.update_run(owner, rid, 'running')
|
|
runs.schedule(owner, rid)
|
|
try:
|
|
await asyncio.wait_for(adapter.stream_entered.wait(), 2)
|
|
await runs.cancel(owner, rid)
|
|
assert runs.metadata(owner, rid).get('stop_requested') is True
|
|
assert store.get_run(owner, rid)['run']['status'] not in runs.store.TERMINAL
|
|
assert runs.locks[owner].locked(), 'Do not admit new account work while old work is unconfirmed'
|
|
assert not runs.tasks[rid].done()
|
|
finally:
|
|
await runs.stop()
|
|
|
|
|
|
@offline_async
|
|
async def test_revoked_session_recovery_stops_remote_before_terminalizing(execution):
|
|
accounts, store, owner, token, _, rid = execution
|
|
adapter = BlockedAdapter()
|
|
runs = coordinator(execution, adapter, upstream='offline-upstream')
|
|
store.update_run(owner, rid, 'running')
|
|
accounts.revoke(token)
|
|
try:
|
|
await asyncio.wait_for(runs.execute(owner, rid), 2)
|
|
assert adapter.starts == [], 'Recovery must never replay a known admission'
|
|
assert adapter.stops, 'A revoked session does not mean its remote execution stopped'
|
|
assert store.get_run(owner, rid)['run']['status'] == 'cancelled'
|
|
finally:
|
|
await runs.stop()
|
|
|
|
|
|
def test_stale_stream_metadata_cannot_erase_cancellation_intent(execution):
|
|
_, _, owner, _, _, rid = execution
|
|
runs = coordinator(execution, BlockedAdapter(), upstream='offline-upstream')
|
|
stale = runs.metadata(owner, rid)
|
|
cancel = runs.metadata(owner, rid)
|
|
cancel['stop_requested'] = True
|
|
runs.metadata(owner, rid, cancel)
|
|
stale['content'] = 'A delta racing with cancellation'
|
|
runs.metadata(owner, rid, stale)
|
|
assert runs.metadata(owner, rid).get('stop_requested') is True
|
|
|
|
|
|
@offline_async
|
|
async def test_restart_after_cancel_during_admission_reconciles_remote(execution):
|
|
_, store, owner, _, _, rid = execution
|
|
adapter = BlockedAdapter(block_start=True, fail_stop=True)
|
|
original = coordinator(execution, adapter)
|
|
original.schedule(owner, rid)
|
|
try:
|
|
await asyncio.wait_for(adapter.start_entered.wait(), 2)
|
|
await original.cancel(owner, rid)
|
|
adapter.start_release.set()
|
|
await asyncio.wait_for(adapter.stop_entered.wait(), 2)
|
|
finally:
|
|
adapter.start_release.set()
|
|
await original.stop()
|
|
# The remote run exists, its stop failed, and the local projection may already
|
|
# be terminal. A restart must still reconcile that durable remote stop intent.
|
|
recovered_adapter = BlockedAdapter()
|
|
recovered = ChatRuns(execution[0], store, adapter=recovered_adapter)
|
|
try:
|
|
await recovered.start()
|
|
await asyncio.wait_for(recovered_adapter.stop_entered.wait(), 2)
|
|
if rid in recovered.tasks:
|
|
await asyncio.wait_for(recovered.tasks[rid], 2)
|
|
assert recovered_adapter.starts == []
|
|
assert store.get_run(owner, rid)['run']['status'] == 'cancelled'
|
|
finally:
|
|
await recovered.stop()
|
|
|
|
|
|
@offline_async
|
|
async def test_delete_cannot_discard_unconfirmed_remote_cancellation(execution):
|
|
_, store, owner, _, cid, rid = execution
|
|
adapter = BlockedAdapter(fail_stop=True)
|
|
runs = coordinator(execution, adapter, upstream='offline-upstream')
|
|
store.update_run(owner, rid, 'running')
|
|
runs.schedule(owner, rid)
|
|
try:
|
|
await asyncio.wait_for(adapter.stream_entered.wait(), 2)
|
|
with pytest.raises(ChatConflict):
|
|
await runs.cancel_conversation(owner, cid)
|
|
store.delete_conversation(owner, cid)
|
|
assert store.get_run(owner, rid)['run']['status'] not in store.TERMINAL
|
|
assert runs.metadata(owner, rid).get('stop_requested') is True
|
|
finally:
|
|
await runs.stop()
|