Files
CyberGril/web/tests/hel898_shots.mjs
T

173 lines
15 KiB
JavaScript

// HEL-898 browser check: her words while she speaks (cg.subtitle contract v2) on the approved conversation. Drives
// headless Edge over CDP against tests/hel878_mock.mjs, with tests/fake-livekit.js in place of the LiveKit client and
// Chromium's fake audio device in place of a microphone: no model, no TTS, no real call. Frames are the worker's v2
// envelopes (docs/voice-subtitle-contract-v2.md). Usage: node tests/hel898_shots.mjs <edge.exe> <outDir> [port].
import { spawn } from 'node:child_process';
import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
const [edge, out, portArg] = process.argv.slice(2);
const port = Number(portArg || 8878), base = `http://127.0.0.1:${port}`, cdpPort = 9498;
mkdirSync(out, { recursive: true });
const profile = join(tmpdir(), `hel898-edge-${Date.now()}`);
const browser = spawn(edge, ['--headless=new', `--remote-debugging-port=${cdpPort}`, `--user-data-dir=${profile}`, '--no-first-run', '--hide-scrollbars', '--force-color-profile=srgb', '--use-fake-device-for-media-stream', '--use-fake-ui-for-media-stream', '--autoplay-policy=no-user-gesture-required', 'about:blank'], { stdio: 'ignore' });
const sleep = ms => new Promise(r => setTimeout(r, ms));
let targets;
for (let i = 0; i < 60 && !targets; i++) { try { targets = await (await fetch(`http://127.0.0.1:${cdpPort}/json`)).json(); } catch { await sleep(250); } }
const ws = new WebSocket(targets.find(t => t.type === 'page').webSocketDebuggerUrl);
await new Promise(r => ws.addEventListener('open', r, { once: true }));
let id = 0; const waiting = new Map();
ws.addEventListener('message', ({ data }) => { const m = JSON.parse(data); if (m.id && waiting.has(m.id)) { const { ok, ko } = waiting.get(m.id); waiting.delete(m.id); m.error ? ko(new Error(m.error.message)) : ok(m.result); } });
const cdp = (method, params = {}) => new Promise((ok, ko) => { const n = ++id; waiting.set(n, { ok, ko }); ws.send(JSON.stringify({ id: n, method, params })); });
const js = async expr => { const r = await cdp('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }); if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'js error'); return r.result.value; };
const results = [];
const check = (name, ok, detail = '') => { results.push(ok); console.log(`${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ` — ${detail}` : ''}`); };
const ctl = async path => (await fetch(`${base}${path}`)).json();
const shot = async name => { const { data } = await cdp('Page.captureScreenshot', { format: 'png' }); writeFileSync(join(out, `${name}.png`), Buffer.from(data, 'base64')); };
const frames = async (name, ms) => { const list = []; const end = Date.now() + ms; while (Date.now() < end) { const { data } = await cdp('Page.captureScreenshot', { format: 'jpeg', quality: 70 }); list.push(data); } list.forEach((d, i) => writeFileSync(join(out, `${name}-${String(i).padStart(2, '0')}.jpg`), Buffer.from(d, 'base64'))); return list.length; };
const click = async (selector, text = '') => js(`(() => { const el = [...document.querySelectorAll(${JSON.stringify(selector)})].find(e => e.offsetParent !== null && e.textContent.includes(${JSON.stringify(text)})); if (!el) return false; el.click(); return true; })()`);
const text = sel => js(`[...document.querySelectorAll(${JSON.stringify(sel)})].map(e => e.innerText).join('\\n')`);
const exists = sel => js(`!!document.querySelector(${JSON.stringify(sel)})`);
// The stage's own text (every line now on it) and the line read out to assistive tech.
const stage = () => js(`document.querySelector('.conversation .stage')?.innerText || ''`);
const said = () => js(`document.querySelector('.conversation .sr')?.textContent || ''`);
const row = () => js(`document.querySelector('.voice-row.real')?.innerText || ''`);
const phase = () => js(`document.querySelector('.voice-row.real')?.dataset.phase || 'off'`);
const lk = expr => js(`window.__lk ? (${expr}) : null`);
const say = frame => lk(`__lk.say(${JSON.stringify(frame)})`);
const her = (kind, speech, extra = {}) => say({ kind, role: 'assistant', text: '', speech_id: speech, ...extra });
const type = t => js(`(() => { const el = document.querySelector('textarea'); const set = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value').set; set.call(el, ${JSON.stringify(t)}); el.dispatchEvent(new Event('input', { bubbles: true })); return true; })()`);
const until = async (fn, ms = 4000) => { const end = Date.now() + ms; while (Date.now() < end) { if (await fn()) return true; await sleep(100); } return false; };
const views = { desktop: { width: 1440, height: 900, deviceScaleFactor: 1, mobile: false }, phone: { width: 390, height: 844, deviceScaleFactor: 2, mobile: true } };
async function view(v) { await cdp('Emulation.setDeviceMetricsOverride', views[v]); await cdp('Emulation.setTouchEmulationEnabled', { enabled: views[v].mobile, maxTouchPoints: 5 }); }
async function boot(theme) {
await cdp('Page.navigate', { url: `${base}/` }); await sleep(500);
await js(`localStorage.setItem('cg.theme', ${JSON.stringify(theme)}); sessionStorage.clear(); true`);
await cdp('Page.navigate', { url: `${base}/` }); await sleep(1600);
}
async function startVoice() {
await click('.voice-button'); await until(() => exists('.modal.open'));
await click('.modal.open .btn-primary', '开始语音');
await until(async () => (await phase()) === 'joining');
await until(async () => (await lk('__lk.rooms.at(-1)?.state')) === 'connected');
await lk('__lk.join()'); await sleep(200);
}
const calls = async () => ctl('/__log');
// One voice turn of yours as the worker leaves it: the stored row, then user_final; waits for it on the stage.
const youSay = async words => {
const { turn } = await ctl('/__voice-turn?stage=user&text=' + encodeURIComponent(words));
await say({ kind: 'user_final', role: 'user', text: words }); await lk(`__lk.state('thinking')`);
await until(async () => (await stage()).includes(words.slice(0, 4)), 6000);
return turn;
};
const rows = async () => ctl('/__rows');
await cdp('Page.enable'); await cdp('Runtime.enable'); await cdp('Network.enable');
const REPLY = '是橘色的那只吗?它最近总在桥下晒太阳,尾巴一甩一甩的,好像整座桥都是它的。你下次路过可以叫它一声,它会回头看你。';
const STEPS = ['是橘色的那只吗?它最近总在桥下晒太阳,', '是橘色的那只吗?它最近总在桥下晒太阳,尾巴一甩一甩的,好像整座桥都是它的。', `${REPLY}`];
for (const [theme, v] of [['moon', 'desktop'], ['fog', 'desktop'], ['moon', 'phone'], ['fog', 'phone']]) {
const tag = `${theme}-${v}`;
await view(v); await boot(theme); await click('.nav-item', '交谈'); await sleep(1400);
await startVoice(); await calls();
// ── 1. A voice turn: your draft and final, then her words growing on the stage before anything is stored. ──
await say({ kind: 'interim', role: 'user', text: '今天下班的路上' }); await sleep(200);
const prior = (await rows()).filter(r => r.text === REPLY).length;
const turn = await youSay('今天下班的路上看到一只猫');
await her('turn_start', `a-${tag}`); await lk(`__lk.state('speaking')`);
const shots = [];
for (const [i, step] of STEPS.entries()) {
await her('assistant_interim', `a-${tag}`, { text: step });
shots.push(frames(`${tag}-m1-growing-${i}`, 450));
await shots.at(-1);
}
await until(async () => (await stage()).includes('晒太阳'), 8000);
check(`${tag}: her reply shows on the stage while she speaks, before it is stored`, (await stage()).includes('晒太阳') && (await rows()).filter(r => r.text === REPLY).length === prior, await stage());
check(`${tag}: your draft stays in the voice line, her words never there`, !(await row()).includes('晒太阳'), await row());
await shot(`${tag}-01-her-words-growing`);
// ── 2. Pause and look back while she is still speaking: reading holds, nothing jumps. ──
await click('.reader-bar .flow'); await sleep(300);
check(`${tag}: pause while she speaks`, await exists('.conversation.reading-paused'));
await click('.reader-bar .icon[aria-label="上一句"]'); await sleep(400);
await shot(`${tag}-02-paused-looking-back`);
await ctl('/__voice-turn?stage=reply&text=' + encodeURIComponent(REPLY));
await her('assistant_final', `a-${tag}`, { text: REPLY, turn_id: `voice-${turn}` }); await her('turn_end', `a-${tag}`, { turn_id: `voice-${turn}` }); await lk(`__lk.state('listening')`);
await sleep(900);
await click('.reader-bar .latest'); await sleep(2500);
await click('.log-btn'); await sleep(700);
const rec = await text('.log-panel .scroll');
check(`${tag}: once stored, the record holds her reply exactly once (re-read, not written by the page)`, rec.split(REPLY).length - 1 === prior + 1, rec.slice(-160).replace(/\n/g, ' '));
await shot(`${tag}-03-record-after-final`);
await click('.log-panel .btn-round'); await sleep(400);
await click('.read-all'); await sleep(500); await shot(`${tag}-04-full-text`); await click('.record.full .btn-round'); await sleep(300);
// ── 3. Interrupted: no final; her line goes at once; late frames of the cut turn change nothing. ──
// Paused reading, then you speak: like sending, your new turn and her answer come in by themselves.
await click('.reader-bar .flow'); await sleep(200);
check(`${tag}: paused before speaking`, await exists('.conversation.reading-paused'));
await youSay('你今天去哪儿了');
check(`${tag}: speaking resumes reading for the new turn`, !(await exists('.conversation.reading-paused')));
await her('turn_start', `b-${tag}`); await lk(`__lk.state('speaking')`);
await her('assistant_interim', `b-${tag}`, { text: '对了,我今天下午去了河边,看到一群白鹭从水面上飞过去了。它们飞得很低,' });
check(`${tag}: second reply growing`, await until(async () => (await stage()).includes('白鹭'), 8000), await stage());
await shot(`${tag}-05-before-cut`);
await say({ kind: 'interim', role: 'user', text: '等一下' });
await her('turn_cancel', `b-${tag}`, { reason: 'interrupted' }); await lk(`__lk.state('listening')`); await ctl('/__voice-turn?stage=cancel');
await sleep(150);
check(`${tag}: interrupted: her unfinished line is gone at once`, !(await stage()).includes('白鹭'), await stage());
await her('assistant_interim', `b-${tag}`, { text: '对了,我今天下午去了河边,看到一群白鹭从水面上飞过去了。它们飞得很低,真好看。' });
await her('assistant_final', `b-${tag}`, { text: '对了,我今天下午去了河边,看到一群白鹭从水面上飞过去了。它们飞得很低,真好看。', turn_id: 'late' });
await sleep(400);
check(`${tag}: late frames of the cut turn stay out`, !(await stage()).includes('白鹭'));
await shot(`${tag}-06-after-cut`);
// ── 4. Text takes over while she speaks: her line goes, your typed words come next. ──
await youSay('我们聊点别的吧');
await her('turn_start', `c-${tag}`); await lk(`__lk.state('speaking')`);
await her('assistant_interim', `c-${tag}`, { text: '那我们换个话题吧,你今天晚饭吃了什么,好不好吃?我今天' });
check(`${tag}: her line before you type`, await until(async () => (await stage()).includes('换个话题'), 8000), await stage());
await type('先等等,我打字说'); await js(`document.querySelector('.send').click(), true`); await sleep(400);
check(`${tag}: text over voice: her line is gone, your words are on the stage`, !(await stage()).includes('换个话题') && (await stage()).includes('先等等'), await stage());
await until(async () => !(await exists('.reader-bar .stop')), 9000); await lk(`__lk.state('listening')`); await sleep(400);
// ── 5. A stored reach-out read aloud: its own text in her place, no request, no read mark, interruptible. ──
const { id: pro } = await ctl('/__proactive?text=' + encodeURIComponent('刚才路过花店,想起你说过喜欢白色的洋桔梗。'));
await calls();
await say({ kind: 'proactive_start', role: 'assistant', text: '刚才路过花店,想起你说过喜欢白色的洋桔梗。', speech_id: `p-${tag}`, message_id: pro }); await lk(`__lk.state('speaking')`);
await sleep(1500);
check(`${tag}: reach-out shown in her place, word for word`, (await stage()).includes('洋桔梗'), await stage());
await shot(`${tag}-07-reach-out`);
await say({ kind: 'interim', role: 'user', text: '是吗' });
await say({ kind: 'proactive_cancel', role: 'assistant', text: '', speech_id: `p-${tag}`, message_id: pro, reason: 'interrupted' }); await lk(`__lk.state('listening')`);
await sleep(200);
const after = await calls();
check(`${tag}: reach-out cut: cleared; the page wrote nothing (no receipt, no read mark, no message)`, !after.some(l => !l.startsWith('GET ')) , after.join(' | '));
// ── 6. Hanging up mid-sentence clears her line with the call. ──
await youSay('我先去睡了');
await her('turn_start', `d-${tag}`); await her('assistant_interim', `d-${tag}`, { text: '晚安,明天见。今天说了好多好多话,你要早点睡哦。我会在窗边等你,' });
check(`${tag}: her line before hanging up`, await until(async () => (await stage()).includes('早点睡'), 8000), await stage());
await click('.voice-row.real .voice-end'); await sleep(400);
check(`${tag}: hung up: her line gone, voice off`, !(await stage()).includes('早点睡') && (await phase()) === 'off', await stage());
await shot(`${tag}-08-ended`);
}
// ── 7. Reduced motion: her words still arrive, without animated entrances. ──
await view('desktop'); await cdp('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-reduced-motion', value: 'reduce' }] });
await boot('moon'); await click('.nav-item', '交谈'); await sleep(1400); await startVoice();
await youSay('慢一点说给我听');
await her('turn_start', 'rm'); await lk(`__lk.state('speaking')`);
for (const step of STEPS) { await her('assistant_interim', 'rm', { text: step }); await sleep(300); }
check('reduced motion: her words arrive', await until(async () => (await stage()).includes('晒太阳'), 8000), await stage());
await frames('reduced-motion-growing', 600);
await click('.voice-row.real .voice-end'); await sleep(300);
ws.close(); browser.kill();
try { rmSync(profile, { recursive: true, force: true }); } catch { /* Edge may still hold the profile. */ }
const failed = results.filter(r => !r).length;
console.log(`${results.length - failed}/${results.length} passed`);
process.exit(failed ? 1 : 0);