217 lines
14 KiB
Python
217 lines
14 KiB
Python
"""Foreground, loopback-only before/current audit and selected production replay.
|
|
|
|
Needs existing Playwright/Pillow/backend dependencies. Production export is read
|
|
only; write/drag assertions run exclusively against this disposable database.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import sqlite3
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from PIL import Image, ImageChops
|
|
from playwright.sync_api import sync_playwright, expect
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--output', type=Path, required=True)
|
|
p.add_argument('--baseline', type=Path, required=True)
|
|
p.add_argument('--references', type=Path, required=True)
|
|
p.add_argument('--deployed', type=Path, required=True)
|
|
p.add_argument('--previous', type=Path, required=True)
|
|
p.add_argument('--only-reference', action='store_true')
|
|
a = p.parse_args()
|
|
out = a.output.resolve(); out.mkdir(parents=True, exist_ok=True)
|
|
report = {'checks': [], 'errors': [], 'geometry': [], 'comparisons': [], 'writes': []}
|
|
API = 'http://127.0.0.1:18383'
|
|
WEB = 'http://127.0.0.1:15383'
|
|
|
|
|
|
def check(name, value):
|
|
report['checks'].append({'name': name, 'pass': bool(value)})
|
|
print('PASS' if value else 'FAIL', name, flush=True)
|
|
assert value, name
|
|
|
|
|
|
def api(path, body=None, method=None):
|
|
r = urllib.request.Request(API + '/api' + path,
|
|
data=None if body is None else json.dumps(body).encode(),
|
|
headers={'Content-Type': 'application/json'},
|
|
method=method or ('GET' if body is None else 'POST'))
|
|
with urllib.request.urlopen(r, timeout=10) as response:
|
|
return json.load(response)
|
|
|
|
|
|
|
|
def geometry(page):
|
|
return page.evaluate('''()=>Object.fromEntries(['.desk','.stage','.composer','.media-summary','.library-layout','.flow-panel','.chat-panel','.svelte-flow__controls','.svelte-flow__minimap'].map(s=>{let e=document.querySelector(s);if(!e)return [s,null];let r=e.getBoundingClientRect(),c=getComputedStyle(e);return [s,{x:r.x,y:r.y,w:r.width,h:r.height,background:c.backgroundColor,color:c.color,font:c.font,padding:c.padding}]}))''')
|
|
|
|
|
|
def seed(runtime):
|
|
db = sqlite3.connect(runtime/'data/calliope.db')
|
|
# Import only the two explicitly selected boards; no credentials/config or
|
|
# unrelated user/project records are copied from the server.
|
|
for board in [4, 5]:
|
|
snap = json.loads((out/f'production-board{board}.json').read_text())
|
|
for table in ['projects','agent_sessions','characters','locations','items','canvas','canvas_node','canvas_edge','agent_messages']:
|
|
for row in snap.get(table, []):
|
|
columns = ','.join('"'+k+'"' for k in row)
|
|
db.execute('INSERT INTO '+table+' ('+columns+') VALUES ('+','.join('?' for _ in row)+')', list(row.values()))
|
|
# An empty, explicitly isolated session for the selected project's UI.
|
|
db.execute("INSERT INTO agent_sessions(id,title,project_id) VALUES(17,'隔离项目会话',7)")
|
|
assets = runtime/'data/assets/uploads'; assets.mkdir(parents=True,exist_ok=True)
|
|
for src, dst in [(ROOT/'calliope-web/static/refract/coast.jpg',assets/'HEL883-fixture.jpg'),(ROOT/'e2e/tiny.mp4',assets/'HEL883-fixture.mp4')]:
|
|
shutil.copyfile(src,dst)
|
|
db.execute("INSERT INTO agent_sessions(id,title) VALUES(18,'HEL883 共享灯箱隔离回归')")
|
|
db.execute("INSERT INTO canvas(id,title,agent_session_id,viewport_json) VALUES(6,'HEL883 测试夹具(非用户画布)',18,?)", (json.dumps({'x':0,'y':0,'zoom':1}),))
|
|
for kind,x in [('image',40),('video',320)]:
|
|
db.execute("INSERT INTO canvas_node(canvas_id,type,title,x,y,status,artifact_path) VALUES(6,?,'HEL883 灯箱夹具',?,140,'done',?)",(kind,x,str(assets/('HEL883-fixture.jpg' if kind=='image' else 'HEL883-fixture.mp4'))))
|
|
db.commit();db.close()
|
|
graph = {'1': {'class_type':'PrimitiveStringMultiline','inputs':{'value':'HEL883 同状态视觉核对'},'_meta':{'title':'Prompt (Input:prompt)'}}, '2': {'class_type':'LoadImage','inputs':{'image':''},'_meta':{'title':'Image (Input:image)'}},'3':{'class_type':'SaveImage','inputs':{'images':['2',0]}}}
|
|
return api('/workflows', {'name':'HEL883 隔离图生图','kind':'image','workflow_json':graph})['id']
|
|
|
|
|
|
def matrix(browser,wf):
|
|
phases=[('base',a.baseline.resolve()),('current',ROOT/'calliope-web/build'),('previous',a.previous.resolve()),('deployed',a.deployed.resolve())]
|
|
if a.only_reference: phases=phases[-1:]
|
|
for phase, build in phases:
|
|
(out/'active-build.txt').write_text(str(build))
|
|
context = browser.new_context(viewport={'width':1440,'height':900})
|
|
context.add_init_script("localStorage.setItem('aigc-lang','zh');localStorage.setItem('aigc-motion','off');localStorage.setItem('aigc-theme','night')")
|
|
mirrors={str(board):json.loads((out/f'production-board{board}.json').read_text())['canvas'][0]['viewport_json'] for board in [4,5]}
|
|
mirrors['6']=json.dumps({'x':0,'y':0,'zoom':1})
|
|
context.add_init_script('Object.entries('+json.dumps(mirrors)+').forEach(([id,v])=>{if(!localStorage.getItem("calliope.canvas.viewport."+id))localStorage.setItem("calliope.canvas.viewport."+id,v)})')
|
|
context.route('https://**/*',lambda r:r.abort())
|
|
page = context.new_page();page.set_default_timeout(12000)
|
|
page.on('pageerror',lambda e:report['errors'].append(str(e)))
|
|
page.on('request',lambda r:report['writes'].append({'phase':phase,'method':r.method,'path':r.url.split('15383')[-1]}) if r.method not in ['GET','HEAD'] else None)
|
|
for label,path,ready in [('create',f'/create?kind=image&wf={wf}','#desk-prompt'),('library','/library','.lt'),('board4','/canvas/4?session=17','.canvas-card'),('board5','/canvas/5?session=16','.flow-panel'),('fixture','/canvas/6?session=18','.canvas-card')]:
|
|
if a.only_reference and label!='fixture': continue
|
|
page.goto(WEB+path);page.locator(ready).first.wait_for()
|
|
page.wait_for_timeout(900)
|
|
for width,height in [(1440,900),(1366,768),(1280,720)]:
|
|
page.set_viewport_size({'width':width,'height':height})
|
|
for theme in ['night','day','grade','bluehour']:
|
|
page.evaluate('(t)=>document.documentElement.dataset.theme=t',theme)
|
|
page.mouse.move(1,1);page.wait_for_timeout(200)
|
|
name=f'{label}-{theme}-{width}'
|
|
page.screenshot(path=str(out/f'{phase}-{name}.png'),animations='disabled')
|
|
if phase=='deployed' and label=='fixture':
|
|
from reference import capture
|
|
capture(page,context,a.references/'canvas-sample.html',out,theme,width)
|
|
report['geometry'].append({'phase':phase,'name':name,'url':page.url,'theme':theme,'motion':page.locator('html').get_attribute('data-motion'),'geometry':geometry(page)})
|
|
check(f'{phase} {label} mounted',True)
|
|
if phase=='deployed':
|
|
for kind in ['image','video']:
|
|
page.goto(WEB+'/canvas/6?session=18')
|
|
hit=page.locator(f'.artifact-node[data-kind="{kind}"] .media-btn')
|
|
hit.wait_for();hit.focus();scroll=page.evaluate('({x:scrollX,y:scrollY,overflow:document.body.style.overflow})')
|
|
page.keyboard.press('Enter')
|
|
box=page.locator('.backdrop');expect(box).to_be_visible()
|
|
check(f'canvas {kind} lightbox actually opened',box.count()==1)
|
|
check(f'canvas {kind} body locked',page.evaluate('document.body.style.overflow')=='hidden')
|
|
if kind=='video':
|
|
video=box.locator('video');expect(video).to_be_visible()
|
|
video.evaluate('(v)=>v.play()');page.wait_for_timeout(150)
|
|
check('canvas video manual playback',video.evaluate('(v)=>!v.paused && v.currentTime>0'))
|
|
video.evaluate('(v)=>v.pause()')
|
|
else:
|
|
expect(box.locator('img')).to_be_visible()
|
|
box.get_by_role('button',name='100%',exact=True).click()
|
|
expect(box.locator('.frame.zoomed')).to_be_visible()
|
|
box.locator('button,a').first.focus();page.keyboard.press('Shift+Tab')
|
|
check(f'{kind} tab stays inside preview',box.evaluate('(e)=>e.contains(document.activeElement)'))
|
|
page.screenshot(path=str(out/f'canvas-{kind}-opened.png'))
|
|
page.keyboard.press('Escape');expect(box).to_have_count(0)
|
|
check(f'canvas {kind} focus restored',hit.evaluate('(e)=>e===document.activeElement'))
|
|
check(f'canvas {kind} scroll restored',scroll==page.evaluate('({x:scrollX,y:scrollY,overflow:document.body.style.overflow})'))
|
|
header=page.locator('.artifact-node[data-kind="image"] header')
|
|
before=next(n for n in api('/canvas/6')['nodes'] if n['type']=='image')
|
|
b=header.bounding_box();page.mouse.move(b['x']+80,b['y']+12);page.mouse.down();page.mouse.move(b['x']+160,b['y']+72,steps=10);page.mouse.up();page.wait_for_timeout(800)
|
|
after=next(n for n in api('/canvas/6')['nodes'] if n['id']==before['id'])
|
|
check('canvas drag persists coordinates in isolated DB',abs(after['x']-before['x'])>30 and abs(after['y']-before['y'])>30)
|
|
page.reload();header.wait_for();page.wait_for_timeout(500)
|
|
check('canvas drag retained after reload',next(n for n in api('/canvas/6')['nodes'] if n['id']==before['id'])['x']==after['x'])
|
|
zoom_before=json.loads(api('/canvas/6')['canvas']['viewport_json'])
|
|
page.locator('.svelte-flow__controls-button').first.click()
|
|
page.wait_for_timeout(800)
|
|
saved=json.loads(api('/canvas/6')['canvas']['viewport_json'])
|
|
check('canvas zoom saves to isolated DB',saved['zoom']!=zoom_before['zoom'])
|
|
check('canvas viewport mirror matches backend',json.loads(page.evaluate("localStorage.getItem('calliope.canvas.viewport.6')"))==saved)
|
|
page.reload();page.locator('.zoom-value').wait_for();page.wait_for_timeout(500)
|
|
check('canvas viewport restored after reload',page.locator('.zoom-value').inner_text()==str(round(saved['zoom']*100))+'%')
|
|
context.close()
|
|
if a.only_reference: return
|
|
for current in sorted(list(out.glob('current-*.png'))+list(out.glob('deployed-*.png'))):
|
|
base=out/(current.name.replace('current-','base-',1) if current.name.startswith('current-') else current.name.replace('deployed-','previous-',1))
|
|
b,c=Image.open(base).convert('RGB'),Image.open(current).convert('RGB')
|
|
diff=ImageChops.difference(b,c)
|
|
pixels=sum(1 for x in diff.get_flattened_data() if x!=(0,0,0))
|
|
report['comparisons'].append({'name':current.stem,'different_pixels':pixels,'total_pixels':b.width*b.height,'bbox':diff.getbbox()})
|
|
pair=Image.new('RGB',(b.width*2,b.height));pair.paste(b,(0,0));pair.paste(c,(b.width,0));pair.save(out/('compare-'+current.name))
|
|
|
|
|
|
for port in [18383,15383]:
|
|
with socket.socket() as s:assert s.connect_ex(('127.0.0.1',port))!=0
|
|
proc=None;web_proc=None
|
|
with tempfile.TemporaryDirectory(prefix='hel883-',dir=out) as temp:
|
|
runtime=Path(temp)
|
|
code='''from pathlib import Path
|
|
from calliope import config
|
|
root=Path(TEMP)
|
|
config.CONFIG_FILE=root/'settings.json'
|
|
config.settings=config.Settings(data_dir=root/'data',assets_dir=root/'data/assets',llm_base_url='http://127.0.0.1:1/v1',comfyui_base_url='http://127.0.0.1:1',dry_run=False)
|
|
config.settings.ensure_storage_dirs()
|
|
from calliope.queue.manager import queue_manager
|
|
queue_manager.paused=True
|
|
from calliope.main import create_app
|
|
import uvicorn
|
|
uvicorn.run(create_app(),host='127.0.0.1',port=18383)
|
|
'''.replace('TEMP',repr(temp))
|
|
with (out/'replay-api.log').open('w') as log:
|
|
try:
|
|
proc=subprocess.Popen(['python3','-c',code],env={**os.environ,'PYTHONPATH':str(ROOT/'calliope-backend/src')},stdout=log,stderr=subprocess.STDOUT)
|
|
for attempt in range(100):
|
|
try: api('/health');break
|
|
except OSError:
|
|
if proc.poll() is not None: raise RuntimeError('API startup failure')
|
|
time.sleep(.1)
|
|
wf=seed(runtime)
|
|
(out/'active-build.txt').write_text(str(a.baseline.resolve()))
|
|
web=(ROOT/'e2e/hel818-web.cjs').read_text().replace('18247','18383').replace('15173','15383').replace("const build=path.resolve('calliope-web/build');", "const marker="+json.dumps(str(out/'active-build.txt'))+";").replace(" let file=path.join(build", " const build=fs.readFileSync(marker,'utf8'); let file=path.join(build")
|
|
web_proc=subprocess.Popen(['node','-e',web],stdout=log,stderr=subprocess.STDOUT)
|
|
for attempt in range(100):
|
|
try: urllib.request.urlopen(WEB,timeout=1);break
|
|
except OSError:time.sleep(.1)
|
|
with sync_playwright() as pw:
|
|
browser=pw.chromium.launch(args=['--no-sandbox'],env={**os.environ,'LD_LIBRARY_PATH':'/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu'})
|
|
try: matrix(browser,wf)
|
|
finally: browser.close()
|
|
check('no JS runtime errors',not report['errors'])
|
|
check('no generation or model call',not any(any(t in r['path'] for t in ['/generate','/prompt-tools','/agent/stream']) for r in report['writes']))
|
|
finally:
|
|
if web_proc:
|
|
daemon=json.loads(subprocess.check_output(['multica','daemon','status','--output','json']))['pid']
|
|
assert web_proc.pid!=daemon
|
|
web_proc.terminate();web_proc.wait(timeout=10)
|
|
report['web_cleanup']={'pid':web_proc.pid,'daemon_pid':daemon,'exit':web_proc.returncode}
|
|
if proc:
|
|
daemon=json.loads(subprocess.check_output(['multica','daemon','status','--output','json']))['pid']
|
|
assert proc.pid!=daemon
|
|
if proc.poll() is None:proc.send_signal(signal.SIGTERM)
|
|
try:proc.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:proc.kill();proc.wait()
|
|
report['cleanup']={'pid':proc.pid,'daemon_pid':daemon,'exit':proc.returncode}
|
|
(out/'replay-report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2))
|
|
report['runtime_removed']=not runtime.exists()
|
|
(out/'replay-report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2))
|