51 lines
2.0 KiB
Python
51 lines
2.0 KiB
Python
"""The opt-in smoke recorder counts failed attempts and never sends a fifth call."""
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from cybergril.contracts import LLMDelta, LLMMessage
|
|
from scripts.r2_model_smoke import record_provider
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_smoke_evidence_preserves_partial_failure_and_caps_all_attempts(tmp_path):
|
|
class Provider:
|
|
model = "synthetic-test"
|
|
base_url = "http://unused"
|
|
sent = 0
|
|
closed = 0
|
|
|
|
async def stream(self, messages, **kwargs):
|
|
self.sent += 1
|
|
try:
|
|
yield LLMDelta(text="原始片段")
|
|
if self.sent == 1:
|
|
raise OSError("synthetic failure")
|
|
yield LLMDelta(text="完成", finish_reason="stop")
|
|
finally:
|
|
self.closed += 1
|
|
|
|
provider = Provider()
|
|
report = {"calls": []}
|
|
output = tmp_path / "evidence.json"
|
|
messages = [LLMMessage(role="system", content="原始提示"), LLMMessage(role="user", content="合成资料")]
|
|
record_provider(provider, report, output)
|
|
with pytest.raises(OSError):
|
|
async for _ in provider.stream(messages, temperature=.7, max_tokens=1200):
|
|
pass
|
|
for _ in range(3):
|
|
async for _ in provider.stream(messages, temperature=.4, max_tokens=600):
|
|
pass
|
|
with pytest.raises(RuntimeError, match="hard limit"):
|
|
async for _ in provider.stream(messages):
|
|
pass
|
|
evidence = json.loads(output.read_text())
|
|
assert provider.sent == provider.closed == evidence["attempted_model_calls"] == 4
|
|
assert len(evidence["calls"]) == 4
|
|
failed = evidence["calls"][0]
|
|
assert failed["error_type"] == "OSError" and not failed["completed"]
|
|
assert failed["raw_text"] == "原始片段"
|
|
assert failed["messages"] == [{"role": m.role, "content": m.content} for m in messages]
|
|
assert failed["parameters"] == {"temperature": .7, "max_tokens": 1200}
|
|
assert all(c["raw_text"] == "原始片段完成" and c["completed"] for c in evidence["calls"][1:])
|