254 lines
9.9 KiB
Python
254 lines
9.9 KiB
Python
"""Isolated sample gateway. Reuses existing aiohttp; never mutates companion state.
|
|
|
|
Each speaking turn owns a WebRTC peer. Cancellation retires that peer, cancels
|
|
upstream HTTP work, and closes the driver session. A late packet cannot be played
|
|
by the next peer. TTS is buffered as one utterance to avoid punctuation gaps.
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import secrets
|
|
import ssl
|
|
import time
|
|
import wave
|
|
from pathlib import Path
|
|
|
|
import aiohttp
|
|
from aiohttp import web
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class Gateway:
|
|
def __init__(self, config):
|
|
self.config = config
|
|
self.sessions = {}
|
|
self.http = None
|
|
|
|
async def start(self, app):
|
|
self.http = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=90))
|
|
|
|
async def stop(self, app):
|
|
for token in list(self.sessions):
|
|
await self.retire(token)
|
|
await self.http.close()
|
|
|
|
async def driver(self, path, data):
|
|
async with self.http.post(self.config["driver"] + path, json=data) as response:
|
|
response.raise_for_status()
|
|
result = await response.json()
|
|
if result.get("code", 0) != 0:
|
|
raise RuntimeError(result.get("msg", "driver failed"))
|
|
return result
|
|
|
|
async def retire(self, token):
|
|
session = self.sessions.pop(token, None)
|
|
if not session:
|
|
return
|
|
task = session.get("task")
|
|
if task and not task.done():
|
|
task.cancel()
|
|
with contextlib.suppress(asyncio.CancelledError):
|
|
await task
|
|
# Flush and destroy the peer, including inference/output queues. The
|
|
# isolated driver extension closes only this peer, never other services.
|
|
await self.driver("/hel714/close", {"sessionid": session["sid"]})
|
|
|
|
async def offer(self, request):
|
|
params = await request.json()
|
|
manifest = json.loads((ROOT / "dist/assets/manifest.json").read_text())
|
|
clips = manifest.get("talk_clips", [])
|
|
if not clips:
|
|
raise RuntimeError("当前外观包没有交谈底板,未改用其他人物")
|
|
answer = await self.driver(
|
|
"/offer", {"sdp": params["sdp"], "type": "offer", "avatar": clips[0]["avatar_id"]}
|
|
)
|
|
token = secrets.token_urlsafe(24)
|
|
self.sessions[token] = {"sid": answer["sessionid"], "task": None, "ready": asyncio.Event()}
|
|
return web.json_response({**answer, "token": token})
|
|
|
|
async def ready(self, request):
|
|
session = self.sessions.get((await request.json())["token"])
|
|
if not session:
|
|
raise web.HTTPConflict(text="会话已取消")
|
|
session["ready"].set()
|
|
return web.json_response({"ready": True})
|
|
|
|
async def cancel(self, request):
|
|
await self.retire((await request.json())["token"])
|
|
return web.json_response({"cancelled": True})
|
|
|
|
async def reply(self, session, text):
|
|
started = time.monotonic()
|
|
cfg = self.config
|
|
async with self.http.get(cfg["tts"] + "/health") as response:
|
|
response.raise_for_status()
|
|
health = await response.json()
|
|
if not health.get("ok") or cfg["voice"] not in health.get("voices", []):
|
|
raise RuntimeError("指定音色不可用;已停止,未回退其他音色")
|
|
async with self.http.post(
|
|
cfg["llm"],
|
|
json={
|
|
"model": cfg["model"],
|
|
"messages": [
|
|
{
|
|
"role": "system",
|
|
"content": "你是晚晚。当前画面是你在卧室阅读,书名未知。用一两句简短自然的中文回应。不编造书名,不声称有未接入的记忆、天气或感知。",
|
|
},
|
|
{"role": "user", "content": text},
|
|
],
|
|
"max_tokens": 100,
|
|
},
|
|
) as response:
|
|
response.raise_for_status()
|
|
body = await response.json()
|
|
reply = body["choices"][0]["message"]["content"]
|
|
llm_ms = round((time.monotonic() - started) * 1000)
|
|
pcm = bytearray()
|
|
async with self.http.post(
|
|
cfg["tts"] + "/tts",
|
|
json={
|
|
"voice": cfg["voice"],
|
|
"text": reply,
|
|
},
|
|
) as response:
|
|
response.raise_for_status()
|
|
async for chunk in response.content.iter_chunked(4800):
|
|
pcm.extend(chunk)
|
|
if len(pcm) > 24000 * 2 * 45:
|
|
raise RuntimeError("语音超过样机上限,已停止")
|
|
if not pcm:
|
|
raise RuntimeError("语音服务未返回音频")
|
|
# Generate while the camera transition/ICE handshake runs, but never
|
|
# enqueue speech until the browser confirms the media view is ready.
|
|
if session.get("ready"):
|
|
await asyncio.wait_for(session["ready"].wait(), timeout=20)
|
|
wav = io.BytesIO()
|
|
with wave.open(wav, "wb") as output:
|
|
output.setnchannels(1)
|
|
output.setsampwidth(2)
|
|
output.setframerate(24000)
|
|
output.writeframes(pcm)
|
|
form = aiohttp.FormData()
|
|
form.add_field("sessionid", session["sid"])
|
|
form.add_field("file", wav.getvalue(), filename="utterance.wav", content_type="audio/wav")
|
|
async with self.http.post(cfg["driver"] + "/humanaudio", data=form) as response:
|
|
response.raise_for_status()
|
|
result = await response.json()
|
|
if result.get("code") != 0:
|
|
raise RuntimeError(result.get("msg", "audio enqueue failed"))
|
|
return {
|
|
"text": reply,
|
|
"llm_ms": llm_ms,
|
|
"queued_ms": round((time.monotonic() - started) * 1000),
|
|
"audio_seconds": len(pcm) / 48000,
|
|
}
|
|
|
|
async def turn(self, request):
|
|
params = await request.json()
|
|
session = self.sessions.get(params["token"])
|
|
if not session:
|
|
raise web.HTTPConflict(text="会话已取消,请重新进入")
|
|
if session["task"]:
|
|
raise web.HTTPConflict(text="本轮已提交,请先取消")
|
|
text = str(params.get("text", "")).strip()
|
|
if not text or len(text) > 500:
|
|
raise web.HTTPBadRequest(text="请输入1至500字")
|
|
task = session["task"] = asyncio.create_task(self.reply(session, text))
|
|
try:
|
|
result = await task
|
|
except asyncio.CancelledError:
|
|
raise web.HTTPConflict(text="本轮已取消") from None
|
|
return web.json_response(result)
|
|
|
|
async def asr(self, request):
|
|
browser = web.WebSocketResponse(max_msg_size=1024 * 1024)
|
|
await browser.prepare(request)
|
|
try:
|
|
async with self.http.ws_connect(self.config["asr"]) as upstream:
|
|
await upstream.send_json(
|
|
{
|
|
"mode": "2pass",
|
|
"wav_name": "hel714",
|
|
"is_speaking": True,
|
|
"wav_format": "pcm",
|
|
"chunk_size": [5, 10, 5],
|
|
"audio_fs": 16000,
|
|
"itn": True,
|
|
}
|
|
)
|
|
|
|
async def receive():
|
|
async for message in upstream:
|
|
if message.type == aiohttp.WSMsgType.TEXT:
|
|
data = json.loads(message.data)
|
|
await browser.send_json(data)
|
|
await browser.close()
|
|
|
|
reader = asyncio.create_task(receive())
|
|
try:
|
|
async for message in browser:
|
|
if message.type == aiohttp.WSMsgType.BINARY:
|
|
await upstream.send_bytes(message.data)
|
|
elif message.type == aiohttp.WSMsgType.TEXT:
|
|
# Only endpoint signals are accepted from the client.
|
|
await upstream.send_json({"is_speaking": False})
|
|
finally:
|
|
reader.cancel()
|
|
with contextlib.suppress(asyncio.CancelledError):
|
|
await reader
|
|
except (aiohttp.ClientError, TimeoutError) as error:
|
|
if not browser.closed:
|
|
await browser.send_json({"error": f"ASR不可达: {type(error).__name__}"})
|
|
finally:
|
|
await browser.close()
|
|
return browser
|
|
|
|
|
|
@web.middleware
|
|
async def guard(request, handler):
|
|
origin = request.headers.get("Origin")
|
|
if origin and origin != f"{request.scheme}://{request.host}":
|
|
raise web.HTTPForbidden(text="仅接受同源请求")
|
|
try:
|
|
return await handler(request)
|
|
except (aiohttp.ClientError, TimeoutError, RuntimeError, KeyError) as error:
|
|
return web.json_response({"error": str(error)}, status=502)
|
|
|
|
|
|
def make_app(config):
|
|
gateway = Gateway(config)
|
|
app = web.Application(middlewares=[guard], client_max_size=1024 * 1024)
|
|
app.on_startup.append(gateway.start)
|
|
app.on_cleanup.append(gateway.stop)
|
|
app.router.add_post("/api/offer", gateway.offer)
|
|
app.router.add_post("/api/cancel", gateway.cancel)
|
|
app.router.add_post("/api/turn", gateway.turn)
|
|
app.router.add_post("/api/ready", gateway.ready)
|
|
app.router.add_get("/api/asr", gateway.asr)
|
|
app.router.add_get("/", lambda _: web.FileResponse(ROOT / "dist/index.html"))
|
|
app.router.add_static("/", ROOT / "dist", show_index=False)
|
|
return app
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", type=int, default=8014)
|
|
parser.add_argument("--cert")
|
|
parser.add_argument("--key")
|
|
args = parser.parse_args()
|
|
tls = None
|
|
if args.cert:
|
|
tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
|
tls.load_cert_chain(args.cert, args.key)
|
|
web.run_app(
|
|
make_app(json.loads((ROOT / "runtime/config.json").read_text())),
|
|
host=args.host,
|
|
port=args.port,
|
|
ssl_context=tls,
|
|
)
|