Merge commit '8fef2aa64ec1ee742e8b70b825938d30a0ea7319' into HEAD

This commit is contained in:
morluto
2026-10-14 02:10:22 +08:00
68 changed files with 4366 additions and 294 deletions
+100
View File
@@ -0,0 +1,100 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import {
inspectWebPageInputSchema,
listBrowserTargetsInputSchema,
} from "../../dist/domain/browserObservation.js";
import { compareWebCapturesInputSchema } from "../../dist/domain/webCaptureDiffSchemas.js";
import { artifactCliEvidence } from "./artifact-e2e.mjs";
/** Verify browser-socket discovery and inspection against HTTP discovery. */
export async function verifyBrowserSocketTransport({
provider,
profile,
endpoint,
target,
origin,
}) {
const [port, path] = (
await readFile(join(profile, "DevToolsActivePort"), "utf8")
).split("\n");
const socket = `ws://127.0.0.1:${port}${path}`;
const version = await (await fetch(`${endpoint}/json/version`)).json();
assert.equal(
new URL(version.webSocketDebuggerUrl).pathname,
path,
"DevToolsActivePort and /json/version named different browser sockets",
);
const list = async (cdpEndpoint) => {
const listed = await provider.listTargets(
listBrowserTargetsInputSchema.parse({
cdp_endpoint: cdpEndpoint,
allowed_origins: [origin],
}),
);
if (!listed.ok) throw listed.error;
return listed.value;
};
const throughHttp = await list(endpoint);
const throughSocket = await list(socket);
// Target.getTargets reports attachment state that /json/list may omit.
const withoutAttachment = (targets) =>
targets.map(({ attached: _attached, ...rest }) => rest);
assert.ok(throughSocket.targets.length > 0, "Browser socket listed no page");
assert.deepEqual(
withoutAttachment(throughSocket.targets),
withoutAttachment(throughHttp.targets),
);
assert.ok(
throughSocket.targets.every(
({ attached }) => typeof attached === "boolean",
),
"Browser socket omitted target attachment state",
);
assert.equal(throughSocket.browser.product, throughHttp.browser.product);
const evidence = await artifactCliEvidence("inspect-web-page", socket, [
target,
"--allowed-origins",
origin,
"--observation-ms",
"100",
]);
assert.equal(evidence.normalized_result.target.target_id, target);
assert.ok(
evidence.normalized_result.dom.nodes.length > 0,
"Browser-socket inspection returned no DOM nodes",
);
assert.equal(
evidence.parameters.cdp_endpoint,
`ws://127.0.0.1:${port}/devtools/browser/[REDACTED]`,
);
assert.ok(
!JSON.stringify(evidence).includes(path),
"Browser-socket Evidence retained the socket identifier",
);
const throughHttpInspection = await provider.inspectPage(
inspectWebPageInputSchema.parse({
cdp_endpoint: endpoint,
allowed_origins: [origin],
target_id: target,
observation_ms: 100,
}),
);
if (!throughHttpInspection.ok) throw throughHttpInspection.error;
const compared = await provider.compareCaptures(
compareWebCapturesInputSchema.parse({
before: { inspection: throughHttpInspection.value },
after: { inspection: evidence.normalized_result },
}),
);
if (!compared.ok) throw compared.error;
assert.equal(compared.value.dimensions.dom_structure.status, "unchanged");
assert.equal(compared.value.dimensions.scripts.status, "unchanged");
return {
targets: throughSocket.targets.length,
revision: throughSocket.browser.revision,
};
}
@@ -1,3 +1,4 @@
export function assertCleanLinuxHopperRuntime(
launcherPaths?: readonly string[],
procRoot?: string,
): Promise<void>;
+6 -3
View File
@@ -16,6 +16,7 @@ export const assertCleanLinuxHopperRuntime = async (
? []
: [process.env.HOPPER_LAUNCHER_PATH]),
],
procRoot = "/proc",
) => {
const { stdout } = await execFileAsync("ps", ["-ax", "-o", "command="]);
if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(stdout))
@@ -32,17 +33,19 @@ export const assertCleanLinuxHopperRuntime = async (
// /proc exposes the executed file even when argv[0] names a removed symlink.
// Compare file identity, not a display path or a process name.
let inspectionError;
for (const entry of await readdir("/proc", { withFileTypes: true })) {
for (const entry of await readdir(procRoot, { withFileTypes: true })) {
if (!entry.isDirectory() || !/^\d+$/u.test(entry.name)) continue;
let executable;
try {
executable = await stat(`/proc/${entry.name}/exe`, { bigint: true });
executable = await stat(`${procRoot}/${entry.name}/exe`, {
bigint: true,
});
} catch (error) {
if (error?.code === "ENOENT" || error?.code === "ESRCH") continue;
// Other users' executables may be unreadable. The setup lane launches
// providers as the caller; unreadable caller-owned processes fail closed.
if (error?.code === "EACCES") {
const owner = await stat(`/proc/${entry.name}`).catch((cause) => {
const owner = await stat(`${procRoot}/${entry.name}`).catch((cause) => {
if (cause?.code === "ENOENT") return undefined;
throw cause;
});
+243
View File
@@ -0,0 +1,243 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { isUtf8 } from "node:buffer";
const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
// The native decoder displays strict UTF-8 and retains invalid bytes as null text.
const textOf = (expected) => {
const value = Buffer.from(expected.base64, "base64");
return isUtf8(value) ? value.toString("utf8") : null;
};
const field = (record, pointer) =>
record.binary_fields.find((item) => item.pointer === pointer);
const bytes = (record, pointer) => {
const value = field(record, pointer);
assert.equal(
value?.state,
"retained",
`Missing retained bytes at ${pointer}`,
);
const decoded = Buffer.from(value.content_base64, "base64");
assert.equal(value.bytes, decoded.length);
assert.equal(value.sha256, digest(decoded));
return decoded;
};
const binaryEqual = (record, pointer, expected) => {
const decoded = bytes(record, pointer);
assert.deepEqual(decoded, Buffer.from(expected.base64, "base64"), pointer);
assert.equal(field(record, pointer).sha256, expected.sha256);
assert.equal(field(record, pointer).bytes, expected.bytes);
};
const caseOf = (record) =>
record.reported.request.headers.find(
([name]) => name === "x-lab-case",
)?.[1] ?? "curl";
const byCase = (value, name) => {
const selected = value.records.filter((record) => caseOf(record) === name);
assert.equal(
selected.length,
1,
`Expected exactly one producer record for ${name}`,
);
return selected[0];
};
const connectionH3 = (record, side) => {
const conn = record.reported[side];
assert.equal(conn.transport_protocol, "udp");
assert.equal(bytes(record, `/${side}/alpn`).toString(), "h3");
assert.equal(conn.alpn, "h3", `${side}.alpn readable value`);
assert.equal(typeof conn.timestamp_tls_setup, "number");
};
function assertHeaders(record, side, headers) {
const regular = headers.filter(
([name]) => !Buffer.from(name.base64, "base64").toString().startsWith(":"),
);
assert.equal(record.reported[side].headers.length, regular.length);
for (const [index, pair] of regular.entries()) {
binaryEqual(record, `/${side}/headers/${index}/0`, pair[0]);
binaryEqual(record, `/${side}/headers/${index}/1`, pair[1]);
assert.deepEqual(
record.reported[side].headers[index],
pair.map(textOf),
`${side}.headers[${index}] readable values`,
);
}
}
export function assertEndpointHeaders(actual, expected) {
// The proxy rebuilds pseudo fields from its HTTP state. Compare their values
// by name while retaining the exact order and duplicates of regular headers.
const partition = (headers) => {
const pseudo = new Map();
const regular = [];
for (const pair of headers) {
const name = Buffer.from(pair[0].base64, "base64").toString();
if (name.startsWith(":")) {
assert.equal(pseudo.has(name), false);
pseudo.set(name, pair);
} else regular.push(pair);
}
return { pseudo, regular };
};
assert.deepEqual(partition(actual), partition(expected));
}
export function requireHttp3(record) {
for (const side of ["request", "response"])
assert.equal(record.reported[side].http_version, "HTTP/3");
connectionH3(record, "client_conn");
connectionH3(record, "server_conn");
}
export function verifyNormalRecords(normal, manifest, finalProducer) {
for (const record of normal.records) {
const name = caseOf(record);
const producer = finalProducer.find(
(event) => event.id === record.reported.id,
);
assert.ok(producer, `No actual producer observation for ${name}`);
assert.equal(record.reported.error, null);
assert.equal(producer.error, null);
assert.equal(record.reported.response.status_code, 200);
requireHttp3(record);
for (const side of ["request", "response"]) {
assert.equal(
record.reported[side].http_version,
producer[side].http_version,
);
binaryEqual(record, `/${side}/content`, producer[side].content);
assertHeaders(
record,
side,
manifest[side === "request" ? "requests" : "responses"][name].headers,
);
}
// Native UUIDs must be retained as UUIDs; the producer exports no stream mapping.
for (const side of ["client_conn", "server_conn"])
for (const key of ["id", "tls_version", "timestamp_tls_setup", "error"])
assert.equal(
record.reported[side][key],
producer[side][key],
`${side}.${key} differs from producer observation ${producer.id}`,
);
binaryEqual(record, "/request/content", manifest.requests[name].body);
binaryEqual(record, "/response/content", manifest.responses[name].body);
const pseudo = {
":method": "method",
":scheme": "scheme",
":authority": "authority",
":path": "path",
};
for (const [headerName, headerValue] of manifest.requests[name].headers) {
const decodedName = Buffer.from(headerName.base64, "base64").toString();
if (pseudo[decodedName]) {
binaryEqual(record, `/request/${pseudo[decodedName]}`, headerValue);
assert.equal(
record.reported.request[pseudo[decodedName]],
textOf(headerValue),
`request.${pseudo[decodedName]} readable value`,
);
}
}
}
}
export { bytes, caseOf, byCase };
export function verifyConcurrency(normal, manifest) {
const parallelNames = ["parallel-0", "parallel-1", "parallel-2"];
const sent = manifest.events.filter(
(event) =>
event.event === "request-sent" && parallelNames.includes(event.case),
);
const received = manifest.events.filter(
(event) =>
event.event === "request-received" && parallelNames.includes(event.case),
);
const responses = manifest.events.filter(
(event) =>
event.event === "response-sent" && parallelNames.includes(event.case),
);
assert.equal(new Set(sent.map((event) => event.connection)).size, 1);
assert.equal(new Set(received.map((event) => event.connection)).size, 1);
assert.equal(new Set(sent.map((event) => event.stream_id)).size, 3);
assert.equal(new Set(received.map((event) => event.stream_id)).size, 3);
assert.ok(
Math.max(...received.map((event) => event.monotonic)) <
responses[0].monotonic,
);
assert.deepEqual(
responses.map((event) => event.case),
received.map((event) => event.case).reverse(),
);
for (const side of ["client_conn", "server_conn"])
assert.equal(
new Set(
parallelNames.map((name) => byCase(normal, name).reported[side].id),
).size,
1,
);
const normalConnections = new Set(
[...sent, ...received].map((event) => event.connection),
);
for (const connection of normalConnections) {
const handshakes = manifest.events.filter(
(event) => event.event === "handshake" && event.connection === connection,
);
assert.equal(handshakes.length, 1);
assert.equal(handshakes[0].alpn, "h3");
assert.equal(handshakes[0].quic_version, 1);
}
assert.equal(
new Set(normal.records.map((record) => record.reported.id)).size,
4,
);
assert.equal(sent.length, 3);
const receivedByClient = manifest.events.filter(
(event) =>
event.event === "response-received" &&
sent.some(
(request) =>
request.connection === event.connection &&
request.stream_id === event.stream_id,
),
);
assert.equal(receivedByClient.length, 3);
for (const request of sent) {
const response = receivedByClient.find(
(event) =>
event.connection === request.connection &&
event.stream_id === request.stream_id,
);
assert.ok(
response,
"Each sent request needs an independently observed client response",
);
assert.deepEqual(response.body, manifest.responses[request.case].body);
assert.deepEqual(request.body, manifest.requests[request.case].body);
assertEndpointHeaders(
request.headers,
manifest.requests[request.case].headers,
);
assertEndpointHeaders(
response.headers,
manifest.responses[request.case].headers,
);
}
return {
sent: sent.map((event) => event.case),
completed_at_origin: received.map((event) => event.case),
dispatched_at_origin: responses.map((event) => event.case),
completed_at_client: receivedByClient.map(
(event) =>
sent.find(
(request) =>
request.connection === event.connection &&
request.stream_id === event.stream_id,
).case,
),
saved_in_capture: normal.records
.filter((record) => parallelNames.includes(caseOf(record)))
.map(caseOf),
};
}
@@ -0,0 +1,163 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { test } from "node:test";
import { verifyNormalRecords } from "./http3-capture-assertions.mjs";
function connectionFactsFixture() {
const body = Buffer.from([0, 255, 128, 1]);
const view = (bytes) => ({
base64: bytes.toString("base64"),
bytes: bytes.length,
sha256: createHash("sha256").update(bytes).digest("hex"),
});
const producer = {
id: "flow-id",
error: null,
request: { http_version: "HTTP/3", content: view(body) },
response: { http_version: "HTTP/3", content: view(body) },
client_conn: {
id: "client-id",
transport_protocol: "udp",
tls_version: "QUICv1",
timestamp_tls_setup: 1_700_000_000.125678,
error: null,
},
server_conn: {
id: "server-id",
transport_protocol: "udp",
tls_version: "QUICv1",
timestamp_tls_setup: 1_700_000_000.987654,
error: null,
},
};
const reported = structuredClone(producer);
const requestFields = {
method: "POST",
scheme: "https",
authority: "lab.test:18443",
path: "/echo?case=readable",
};
Object.assign(reported.request, requestFields);
const headers = [
[Buffer.from("x-repeat"), Buffer.from("first-α")],
[Buffer.from("x-repeat"), Buffer.from("second-β")],
[Buffer.from("x-binary"), Buffer.from([255])],
];
for (const side of ["request", "response"])
reported[side].headers = [
["x-repeat", "first-α"],
["x-repeat", "second-β"],
["x-binary", null],
];
reported.client_conn.alpn = "h3";
reported.server_conn.alpn = "h3";
reported.response.status_code = 200;
const binary_fields = [
["/request/content", body],
["/response/content", body],
["/client_conn/alpn", Buffer.from("h3")],
["/server_conn/alpn", Buffer.from("h3")],
...Object.entries(requestFields).map(([name, value]) => [
`/request/${name}`,
Buffer.from(value),
]),
...["request", "response"].flatMap((side) =>
headers.flatMap((pair, index) =>
pair.map((value, part) => [`/${side}/headers/${index}/${part}`, value]),
),
),
].map(([pointer, bytes]) => {
const { base64, ...facts } = view(bytes);
return {
pointer,
representation: "producer-bytes",
state: "retained",
content_base64: base64,
...facts,
};
});
return {
normal: { records: [{ reported, binary_fields }] },
manifest: {
requests: {
curl: {
headers: [
...Object.entries(requestFields).map(([name, value]) => [
view(Buffer.from(`:${name}`)),
view(Buffer.from(value)),
]),
...headers.map((pair) => pair.map(view)),
],
body: view(body),
},
},
responses: {
curl: {
headers: headers.map((pair) => pair.map(view)),
body: view(body),
},
},
},
producer: [producer],
};
}
test("positive assertions retain exact producer connection facts on both legs", () => {
const { normal, manifest, producer } = connectionFactsFixture();
verifyNormalRecords(normal, manifest, producer);
});
for (const [field, value] of [
["method", "GET"],
["scheme", "http"],
["authority", "other.test:18443"],
["path", "/different"],
])
test(`positive assertions reject changed readable request ${field}`, () => {
const { normal, manifest, producer } = connectionFactsFixture();
normal.records[0].reported.request[field] = value;
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
code: "ERR_ASSERTION",
});
});
for (const side of ["client_conn", "server_conn"])
test(`positive assertions reject null readable ${side} ALPN`, () => {
const { normal, manifest, producer } = connectionFactsFixture();
normal.records[0].reported[side].alpn = null;
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
code: "ERR_ASSERTION",
});
});
for (const side of ["request", "response"])
for (const [index, part, value] of [
[0, 0, "x-changed"],
[0, 1, "changed-first"],
[1, 1, "changed-second"],
[2, 1, "�"],
])
test(`positive assertions reject changed readable ${side} header ${index}/${part}`, () => {
const { normal, manifest, producer } = connectionFactsFixture();
normal.records[0].reported[side].headers[index][part] = value;
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
code: "ERR_ASSERTION",
});
});
for (const side of ["client_conn", "server_conn"])
for (const [field, value] of [
["tls_version", null],
["tls_version", "changed-producer-label"],
["error", "changed-connection-error"],
["error", undefined],
["timestamp_tls_setup", 0],
["timestamp_tls_setup", 1_700_000_000],
])
test(`positive assertions reject ${side}.${field} changed to ${String(value)}`, () => {
const { normal, manifest, producer } = connectionFactsFixture();
normal.records[0].reported[side][field] = value;
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
code: "ERR_ASSERTION",
});
});
+143
View File
@@ -0,0 +1,143 @@
import assert from "node:assert/strict";
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { setTimeout } from "node:timers/promises";
import { cleanupOwnedProcessGroup } from "../../dist/process/ProcessOwnership.js";
import { observeOwnedProcessGroups } from "../../dist/process/ProcessOwnershipObservation.js";
export const describeError = (error) => ({
name: error?.name ?? "Error",
message: error?.message ?? String(error),
...(error?.stack ? { stack: error.stack } : {}),
});
export async function attemptAll(actions) {
const failures = [];
for (const [name, action] of actions) {
try {
await action();
} catch (error) {
failures.push({ action: name, ...describeError(error) });
}
}
return failures;
}
async function latest(root, report) {
const temporary = join(root, `.verification-${report.run_id}.tmp`);
await writeFile(temporary, JSON.stringify(report, null, 2), { flag: "wx" });
await rename(temporary, join(root, "verification-latest.json"));
}
async function withLatestLock(root, action) {
// Keep this pathname stable: removing it could split concurrent lock owners.
// SQLite releases its OS-backed lock when the connection/process closes.
const database = new DatabaseSync(join(root, ".verification-lock.sqlite"));
try {
for (;;) {
try {
database.exec("BEGIN EXCLUSIVE");
break;
} catch (error) {
if (
error.code !== "ERR_SQLITE_ERROR" ||
(error.errcode !== 5 && error.errcode !== 6)
)
throw error;
// Yield while another process (or async operation here) owns the lock.
// A synchronous busy timeout would block that operation's fs callbacks.
await setTimeout(10);
}
}
return await action();
} finally {
database.close();
}
}
export async function prepareAttempt(root, inspectExisting, run) {
if (!inspectExisting) await mkdir(root, { mode: 0o700 });
const directory = join(root, "verifications", run.run_id);
await mkdir(directory, { recursive: true, mode: 0o700 });
await withLatestLock(root, async () => {
// Preserve older reports as history, without leaving an ambiguous success
// at the capture root. Registration is the shared marker's ordering point.
try {
await rename(
join(root, "verification.json"),
join(directory, "legacy-verification.json"),
);
} catch (error) {
if (error.code !== "ENOENT") throw error;
}
await latest(root, {
run_id: run.run_id,
status: "running",
started_at: new Date().toISOString(),
directory,
});
});
return { directory, captureRoot: join(root, "captures") };
}
export async function publishAttempt(root, directory, report) {
await writeFile(
join(directory, "result.json"),
JSON.stringify(report, null, 2),
{ flag: "wx" },
);
if (report.status === "passed")
await writeFile(
join(directory, "verification.json"),
JSON.stringify(report, null, 2),
{ flag: "wx" },
);
await withLatestLock(root, async () => {
const current = JSON.parse(
await readFile(join(root, "verification-latest.json"), "utf8"),
);
if (current.run_id !== report.run_id) return;
// Ownership check and replacement share one lock with attempt registration.
// An older attempt still keeps its reports, but cannot replace this marker.
await latest(root, {
run_id: report.run_id,
status: report.status,
directory,
finished_at: report.finished_at,
});
});
}
export async function finishProducerGroups(captureRoot, runId) {
let leases;
try {
leases = (await readFile(join(captureRoot, "process-groups.jsonl"), "utf8"))
.trim()
.split("\n")
.filter(Boolean)
.map(JSON.parse);
} catch (error) {
if (error.code === "ENOENT") return { groups: [], observation: [] };
throw error;
}
for (const lease of leases) assert.equal(lease.runId, runId);
const failures = await attemptAll(
leases.map((lease) => [
`producer-group-${lease.processGroupId}`,
async () => {
const result = await cleanupOwnedProcessGroup(lease);
assert.equal(result.cleaned, true, JSON.stringify(result));
},
]),
);
// Inspect even when one close failed; the groups can outlive their bootloader
// parent and disappear from a plain descendant-only observation.
const observation = [
...(await observeOwnedProcessGroups(
runId,
leases.map((lease) => lease.processGroupId),
)),
];
return { groups: leases, failures, observation };
}
@@ -0,0 +1,433 @@
import assert from "node:assert/strict";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "node:test";
import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { once } from "node:events";
import { setTimeout } from "node:timers/promises";
import { DatabaseSync } from "node:sqlite";
import {
attemptAll,
prepareAttempt,
publishAttempt,
finishProducerGroups,
} from "./http3-verifier-lifecycle.mjs";
import { cleanupOwnedProcessGroup } from "../../dist/process/ProcessOwnership.js";
import { runOwnedCommand } from "../../dist/process/OwnedCommand.js";
import { verifyConcurrency } from "./http3-capture-assertions.mjs";
async function waitForFile(path) {
const signal = AbortSignal.timeout(5_000);
for (;;) {
try {
await readFile(path);
return;
} catch (error) {
if (error.code !== "ENOENT") throw error;
}
await setTimeout(10, undefined, { signal });
}
}
function startAttemptWorker(root, report) {
const worker = `
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { setTimeout } from "node:timers/promises";
import { prepareAttempt, publishAttempt } from ${JSON.stringify(
new URL("./http3-verifier-lifecycle.mjs", import.meta.url).href,
)};
const [root, serialized] = process.argv.slice(1);
const report = JSON.parse(serialized);
const attempt = await prepareAttempt(root, true, report);
await writeFile(join(root, report.run_id + ".ready"), "");
for (;;) {
try { await readFile(join(root, report.run_id + ".finish")); break; }
catch (error) { if (error.code !== "ENOENT") throw error; }
await setTimeout(10);
}
await publishAttempt(root, attempt.directory, report);
`;
return runOwnedCommand(
{
command: process.execPath,
arguments: [
"--input-type=module",
"--eval",
worker,
root,
JSON.stringify(report),
],
cwd: root,
hostEnvironment: process.env,
runId: randomUUID(),
},
{ timeoutMs: 10_000, diagnosticBytes: 4096 },
).then(
(result) => ({ result }),
(error) => ({ error }),
);
}
test("cleanup continues through failed closes and reaches its audit", async () => {
const attempted = [];
const failures = await attemptAll(
["client", "transport", "audit"].map((name) => [
name,
async () => {
attempted.push(name);
if (name !== "audit") throw new Error(`${name} failed`);
},
]),
);
assert.deepEqual(attempted, ["client", "transport", "audit"]);
assert.deepEqual(
failures.map((failure) => failure.action),
["client", "transport"],
);
});
test("a failed reinspection cannot leave the earlier success as the current result", async () => {
const root = await mkdtemp(join(tmpdir(), "rea-http3-results-"));
try {
await writeFile(
join(root, "verification.json"),
'{"status":"passed","run_id":"legacy"}',
);
const first = await prepareAttempt(root, true, { run_id: "first" });
await publishAttempt(root, first.directory, {
run_id: "first",
status: "passed",
});
const next = await prepareAttempt(root, true, { run_id: "next" });
assert.equal(
JSON.parse(await readFile(join(root, "verification-latest.json"))).status,
"running",
);
await publishAttempt(root, next.directory, {
run_id: "next",
status: "failed",
error: { message: "new failure" },
});
assert.equal(
JSON.parse(await readFile(join(root, "verification-latest.json"))).run_id,
"next",
);
assert.equal(
JSON.parse(await readFile(join(root, "verification-latest.json"))).status,
"failed",
);
await assert.rejects(readFile(join(next.directory, "verification.json")), {
code: "ENOENT",
});
await assert.rejects(readFile(join(root, "verification.json")), {
code: "ENOENT",
});
assert.equal(
JSON.parse(await readFile(join(first.directory, "verification.json")))
.status,
"passed",
);
assert.equal(
JSON.parse(
await readFile(join(first.directory, "legacy-verification.json")),
).run_id,
"legacy",
);
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("separate overlapping attempts finishing in reverse order retain the newer failure", async () => {
const root = await mkdtemp(join(tmpdir(), "rea-http3-overlap-"));
const commands = [];
const failed = {
run_id: "newer",
status: "failed",
error: { message: "newer inspection failed" },
cleanup_failures: [{ action: "transport-close", message: "close failed" }],
};
let primaryError;
const start = (report) => {
const command = startAttemptWorker(root, report);
commands.push(command);
return command;
};
try {
const older = start({ run_id: "older", status: "passed" });
await waitForFile(join(root, "older.ready"));
const newer = start(failed);
await waitForFile(join(root, "newer.ready"));
await writeFile(join(root, "newer.finish"), "");
const newerResult = await newer;
if (newerResult.error) throw newerResult.error;
assert.equal(newerResult.result.exitCode, 0);
assert.equal(newerResult.result.signal, null);
const marker = await readFile(join(root, "verification-latest.json"));
await writeFile(join(root, "older.finish"), "");
const olderResult = await older;
if (olderResult.error) throw olderResult.error;
assert.equal(olderResult.result.exitCode, 0);
assert.equal(olderResult.result.signal, null);
assert.deepEqual(
await readFile(join(root, "verification-latest.json")),
marker,
);
assert.deepEqual(
JSON.parse(await readFile(join(root, "verifications/newer/result.json"))),
failed,
);
await assert.rejects(
readFile(join(root, "verifications/newer/verification.json")),
{
code: "ENOENT",
},
);
assert.equal(
JSON.parse(
await readFile(join(root, "verifications/older/verification.json")),
).status,
"passed",
);
} catch (error) {
primaryError = error;
} finally {
// Release both barriers even after an assertion/setup failure. Each command
// retains native supervision through termination before the root is removed.
const failures = await attemptAll(
["older", "newer"].map((name) => [
name,
() => writeFile(join(root, name + ".finish"), ""),
]),
);
const outcomes = await Promise.all(commands);
const retained = outcomes.filter(
({ error }) => error?.cleanupFailure || error?.cleanupResult,
);
if (retained.length)
primaryError = new AggregateError(
[primaryError, ...retained.map(({ error }) => error)].filter(Boolean),
`Worker cleanup failed; retained test evidence at ${root}`,
{ cause: primaryError },
);
else
failures.push(
...(await attemptAll([
["root", () => rm(root, { recursive: true, force: true })],
])),
);
if (failures.length)
primaryError = new AggregateError(
[primaryError, ...failures].filter(Boolean),
"Test cleanup failed",
{ cause: primaryError },
);
}
if (primaryError) throw primaryError;
});
test("publication waits for the lock before reading the current owner", async () => {
const root = await mkdtemp(join(tmpdir(), "rea-http3-lock-"));
let database;
let publication;
let primaryError;
try {
const older = await prepareAttempt(root, true, { run_id: "older" });
database = new DatabaseSync(join(root, ".verification-lock.sqlite"));
database.exec("BEGIN EXCLUSIVE");
publication = publishAttempt(root, older.directory, {
run_id: "older",
status: "passed",
});
await waitForFile(join(older.directory, "verification.json"));
const marker = join(root, "verification-latest.json");
assert.equal(JSON.parse(await readFile(marker)).status, "running");
// This connection represents another marker writer. Its replacement must
// be observed only after the blocked publisher acquires the same lock.
const newer = { run_id: "newer", status: "failed" };
await writeFile(marker, JSON.stringify(newer));
database.close();
database = undefined;
await publication;
assert.deepEqual(JSON.parse(await readFile(marker)), newer);
} catch (error) {
primaryError = error;
} finally {
const failures = await attemptAll([
["lock", () => database?.close()],
[
"publication",
async () => {
if (publication) await publication;
},
],
["root", () => rm(root, { recursive: true, force: true })],
]);
if (failures.length)
primaryError = new AggregateError(
[primaryError, ...failures].filter(Boolean),
"Test cleanup failed",
{ cause: primaryError },
);
}
if (primaryError) throw primaryError;
});
test("a failed marker read releases the lock and preserves the attempt report", async () => {
const root = await mkdtemp(join(tmpdir(), "rea-http3-lock-failure-"));
try {
const older = await prepareAttempt(root, true, { run_id: "older" });
await writeFile(join(root, "verification-latest.json"), "{");
const report = {
run_id: "older",
status: "failed",
error: { message: "inspection failed" },
};
await assert.rejects(
publishAttempt(root, older.directory, report),
SyntaxError,
);
assert.deepEqual(
JSON.parse(await readFile(join(older.directory, "result.json"))),
report,
);
const next = await prepareAttempt(root, true, { run_id: "next" });
await publishAttempt(root, next.directory, {
run_id: "next",
status: "passed",
});
assert.equal(
JSON.parse(await readFile(join(root, "verification-latest.json"))).run_id,
"next",
);
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("dispatch reverses origin completion; client and capture may arrive in other orders", () => {
const names = ["parallel-0", "parallel-1", "parallel-2"];
const header = (text) => ({ base64: Buffer.from(text).toString("base64") });
const messages = Object.fromEntries(
names.map((name) => [
name,
{
headers: [[header("x-lab-case"), header(name)]],
body: { base64: header(name).base64 },
},
]),
);
const event = (kind, index, connection, monotonic) => ({
event: kind,
case: names[index],
stream_id: index * 4,
connection,
monotonic,
...messages[names[index]],
});
const manifest = {
requests: messages,
responses: messages,
events: [
...[0, 1, 2].map((i) => event("request-sent", i, "client", i)),
...[1, 2, 0].map((i) => event("request-received", i, "origin", 10)),
...[0, 2, 1].map((i) => event("response-sent", i, "origin", 20)),
...[2, 0, 1].map((i) => event("response-received", i, "client", 30)),
...["client", "origin"].map((connection) => ({
event: "handshake",
connection,
alpn: "h3",
quic_version: 1,
})),
],
};
const record = (name) => ({
reported: {
id: name,
request: { headers: [["x-lab-case", name]] },
client_conn: { id: "client-uuid" },
server_conn: { id: "origin-uuid" },
},
});
const normal = {
records: [...[1, 0, 2].map((i) => record(names[i])), record("curl")],
};
const orders = verifyConcurrency(normal, manifest);
assert.deepEqual(orders.dispatched_at_origin, [
"parallel-0",
"parallel-2",
"parallel-1",
]);
manifest.events = manifest.events.filter(
(item) => item.event !== "response-sent",
);
manifest.events.push(
...[2, 1, 0].map((i) => event("response-sent", i, "origin", 20)),
);
assert.throws(() => verifyConcurrency(normal, manifest), {
code: "ERR_ASSERTION",
});
});
test("cleanup verifies and retires a separately owned producer group", async () => {
const root = await mkdtemp(join(tmpdir(), "rea-http3-group-"));
const runId = randomUUID();
const child = spawn("/usr/bin/sleep", ["60"], {
detached: true,
env: { ...process.env, REA_PROCESS_RUN_ID: runId },
stdio: "ignore",
});
let primaryError;
let cleaned = false;
try {
await once(child, "spawn");
await writeFile(
join(root, "process-groups.jsonl"),
JSON.stringify({
runId,
leaderPid: child.pid,
processGroupId: child.pid,
}) + "\n",
);
const result = await finishProducerGroups(root, runId);
assert.deepEqual(result.failures, []);
assert.equal(result.observation[0][1].state, "empty");
} catch (error) {
primaryError = error;
} finally {
const failures = await attemptAll([
[
"process",
async () => {
const result = await cleanupOwnedProcessGroup({
runId,
leaderPid: child.pid,
processGroupId: child.pid,
});
assert.equal(
result.cleaned,
true,
`Retained ownership evidence at ${root}: ${JSON.stringify(result)}`,
);
cleaned = true;
},
],
[
"root",
async () => {
if (cleaned) await rm(root, { recursive: true, force: true });
},
],
]);
if (failures.length)
primaryError = new AggregateError(
[primaryError, ...failures].filter(Boolean),
"Test cleanup failed",
{ cause: primaryError },
);
}
if (primaryError) throw primaryError;
});