Files
CyberGril/samples/companion/runtime/server.py
T

254 lines
9.9 KiB
Python

"""Isolated sample gateway. Reuses existing aiohttp; never mutates companion state.
Each speaking turn owns a WebRTC peer. Cancellation retires that peer, cancels
upstream HTTP work, and closes the driver session. A late packet cannot be played
by the next peer. TTS is buffered as one utterance to avoid punctuation gaps.
"""
import argparse
import asyncio
import contextlib
import io
import json
import secrets
import ssl
import time
import wave
from pathlib import Path
import aiohttp
from aiohttp import web
ROOT = Path(__file__).resolve().parents[1]
class Gateway:
def __init__(self, config):
self.config = config
self.sessions = {}
self.http = None
async def start(self, app):
self.http = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=90))
async def stop(self, app):
for token in list(self.sessions):
await self.retire(token)
await self.http.close()
async def driver(self, path, data):
async with self.http.post(self.config["driver"] + path, json=data) as response:
response.raise_for_status()
result = await response.json()
if result.get("code", 0) != 0:
raise RuntimeError(result.get("msg", "driver failed"))
return result
async def retire(self, token):
session = self.sessions.pop(token, None)
if not session:
return
task = session.get("task")
if task and not task.done():
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
# Flush and destroy the peer, including inference/output queues. The
# isolated driver extension closes only this peer, never other services.
await self.driver("/hel714/close", {"sessionid": session["sid"]})
async def offer(self, request):
params = await request.json()
manifest = json.loads((ROOT / "dist/assets/manifest.json").read_text())
clips = manifest.get("talk_clips", [])
if not clips:
raise RuntimeError("当前外观包没有交谈底板,未改用其他人物")
answer = await self.driver(
"/offer", {"sdp": params["sdp"], "type": "offer", "avatar": clips[0]["avatar_id"]}
)
token = secrets.token_urlsafe(24)
self.sessions[token] = {"sid": answer["sessionid"], "task": None, "ready": asyncio.Event()}
return web.json_response({**answer, "token": token})
async def ready(self, request):
session = self.sessions.get((await request.json())["token"])
if not session:
raise web.HTTPConflict(text="会话已取消")
session["ready"].set()
return web.json_response({"ready": True})
async def cancel(self, request):
await self.retire((await request.json())["token"])
return web.json_response({"cancelled": True})
async def reply(self, session, text):
started = time.monotonic()
cfg = self.config
async with self.http.get(cfg["tts"] + "/health") as response:
response.raise_for_status()
health = await response.json()
if not health.get("ok") or cfg["voice"] not in health.get("voices", []):
raise RuntimeError("指定音色不可用;已停止,未回退其他音色")
async with self.http.post(
cfg["llm"],
json={
"model": cfg["model"],
"messages": [
{
"role": "system",
"content": "你是晚晚。当前画面是你在卧室阅读,书名未知。用一两句简短自然的中文回应。不编造书名,不声称有未接入的记忆、天气或感知。",
},
{"role": "user", "content": text},
],
"max_tokens": 100,
},
) as response:
response.raise_for_status()
body = await response.json()
reply = body["choices"][0]["message"]["content"]
llm_ms = round((time.monotonic() - started) * 1000)
pcm = bytearray()
async with self.http.post(
cfg["tts"] + "/tts",
json={
"voice": cfg["voice"],
"text": reply,
},
) as response:
response.raise_for_status()
async for chunk in response.content.iter_chunked(4800):
pcm.extend(chunk)
if len(pcm) > 24000 * 2 * 45:
raise RuntimeError("语音超过样机上限,已停止")
if not pcm:
raise RuntimeError("语音服务未返回音频")
# Generate while the camera transition/ICE handshake runs, but never
# enqueue speech until the browser confirms the media view is ready.
if session.get("ready"):
await asyncio.wait_for(session["ready"].wait(), timeout=20)
wav = io.BytesIO()
with wave.open(wav, "wb") as output:
output.setnchannels(1)
output.setsampwidth(2)
output.setframerate(24000)
output.writeframes(pcm)
form = aiohttp.FormData()
form.add_field("sessionid", session["sid"])
form.add_field("file", wav.getvalue(), filename="utterance.wav", content_type="audio/wav")
async with self.http.post(cfg["driver"] + "/humanaudio", data=form) as response:
response.raise_for_status()
result = await response.json()
if result.get("code") != 0:
raise RuntimeError(result.get("msg", "audio enqueue failed"))
return {
"text": reply,
"llm_ms": llm_ms,
"queued_ms": round((time.monotonic() - started) * 1000),
"audio_seconds": len(pcm) / 48000,
}
async def turn(self, request):
params = await request.json()
session = self.sessions.get(params["token"])
if not session:
raise web.HTTPConflict(text="会话已取消,请重新进入")
if session["task"]:
raise web.HTTPConflict(text="本轮已提交,请先取消")
text = str(params.get("text", "")).strip()
if not text or len(text) > 500:
raise web.HTTPBadRequest(text="请输入1至500字")
task = session["task"] = asyncio.create_task(self.reply(session, text))
try:
result = await task
except asyncio.CancelledError:
raise web.HTTPConflict(text="本轮已取消") from None
return web.json_response(result)
async def asr(self, request):
browser = web.WebSocketResponse(max_msg_size=1024 * 1024)
await browser.prepare(request)
try:
async with self.http.ws_connect(self.config["asr"]) as upstream:
await upstream.send_json(
{
"mode": "2pass",
"wav_name": "hel714",
"is_speaking": True,
"wav_format": "pcm",
"chunk_size": [5, 10, 5],
"audio_fs": 16000,
"itn": True,
}
)
async def receive():
async for message in upstream:
if message.type == aiohttp.WSMsgType.TEXT:
data = json.loads(message.data)
await browser.send_json(data)
await browser.close()
reader = asyncio.create_task(receive())
try:
async for message in browser:
if message.type == aiohttp.WSMsgType.BINARY:
await upstream.send_bytes(message.data)
elif message.type == aiohttp.WSMsgType.TEXT:
# Only endpoint signals are accepted from the client.
await upstream.send_json({"is_speaking": False})
finally:
reader.cancel()
with contextlib.suppress(asyncio.CancelledError):
await reader
except (aiohttp.ClientError, TimeoutError) as error:
if not browser.closed:
await browser.send_json({"error": f"ASR不可达: {type(error).__name__}"})
finally:
await browser.close()
return browser
@web.middleware
async def guard(request, handler):
origin = request.headers.get("Origin")
if origin and origin != f"{request.scheme}://{request.host}":
raise web.HTTPForbidden(text="仅接受同源请求")
try:
return await handler(request)
except (aiohttp.ClientError, TimeoutError, RuntimeError, KeyError) as error:
return web.json_response({"error": str(error)}, status=502)
def make_app(config):
gateway = Gateway(config)
app = web.Application(middlewares=[guard], client_max_size=1024 * 1024)
app.on_startup.append(gateway.start)
app.on_cleanup.append(gateway.stop)
app.router.add_post("/api/offer", gateway.offer)
app.router.add_post("/api/cancel", gateway.cancel)
app.router.add_post("/api/turn", gateway.turn)
app.router.add_post("/api/ready", gateway.ready)
app.router.add_get("/api/asr", gateway.asr)
app.router.add_get("/", lambda _: web.FileResponse(ROOT / "dist/index.html"))
app.router.add_static("/", ROOT / "dist", show_index=False)
return app
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8014)
parser.add_argument("--cert")
parser.add_argument("--key")
args = parser.parse_args()
tls = None
if args.cert:
tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
tls.load_cert_chain(args.cert, args.key)
web.run_app(
make_app(json.loads((ROOT / "runtime/config.json").read_text())),
host=args.host,
port=args.port,
ssl_context=tls,
)