Merge commit '8fef2aa64ec1ee742e8b70b825938d30a0ea7319' into HEAD
This commit is contained in:
@@ -0,0 +1,100 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import {
|
||||
inspectWebPageInputSchema,
|
||||
listBrowserTargetsInputSchema,
|
||||
} from "../../dist/domain/browserObservation.js";
|
||||
import { compareWebCapturesInputSchema } from "../../dist/domain/webCaptureDiffSchemas.js";
|
||||
import { artifactCliEvidence } from "./artifact-e2e.mjs";
|
||||
|
||||
/** Verify browser-socket discovery and inspection against HTTP discovery. */
|
||||
export async function verifyBrowserSocketTransport({
|
||||
provider,
|
||||
profile,
|
||||
endpoint,
|
||||
target,
|
||||
origin,
|
||||
}) {
|
||||
const [port, path] = (
|
||||
await readFile(join(profile, "DevToolsActivePort"), "utf8")
|
||||
).split("\n");
|
||||
const socket = `ws://127.0.0.1:${port}${path}`;
|
||||
const version = await (await fetch(`${endpoint}/json/version`)).json();
|
||||
assert.equal(
|
||||
new URL(version.webSocketDebuggerUrl).pathname,
|
||||
path,
|
||||
"DevToolsActivePort and /json/version named different browser sockets",
|
||||
);
|
||||
const list = async (cdpEndpoint) => {
|
||||
const listed = await provider.listTargets(
|
||||
listBrowserTargetsInputSchema.parse({
|
||||
cdp_endpoint: cdpEndpoint,
|
||||
allowed_origins: [origin],
|
||||
}),
|
||||
);
|
||||
if (!listed.ok) throw listed.error;
|
||||
return listed.value;
|
||||
};
|
||||
const throughHttp = await list(endpoint);
|
||||
const throughSocket = await list(socket);
|
||||
// Target.getTargets reports attachment state that /json/list may omit.
|
||||
const withoutAttachment = (targets) =>
|
||||
targets.map(({ attached: _attached, ...rest }) => rest);
|
||||
assert.ok(throughSocket.targets.length > 0, "Browser socket listed no page");
|
||||
assert.deepEqual(
|
||||
withoutAttachment(throughSocket.targets),
|
||||
withoutAttachment(throughHttp.targets),
|
||||
);
|
||||
assert.ok(
|
||||
throughSocket.targets.every(
|
||||
({ attached }) => typeof attached === "boolean",
|
||||
),
|
||||
"Browser socket omitted target attachment state",
|
||||
);
|
||||
assert.equal(throughSocket.browser.product, throughHttp.browser.product);
|
||||
|
||||
const evidence = await artifactCliEvidence("inspect-web-page", socket, [
|
||||
target,
|
||||
"--allowed-origins",
|
||||
origin,
|
||||
"--observation-ms",
|
||||
"100",
|
||||
]);
|
||||
assert.equal(evidence.normalized_result.target.target_id, target);
|
||||
assert.ok(
|
||||
evidence.normalized_result.dom.nodes.length > 0,
|
||||
"Browser-socket inspection returned no DOM nodes",
|
||||
);
|
||||
assert.equal(
|
||||
evidence.parameters.cdp_endpoint,
|
||||
`ws://127.0.0.1:${port}/devtools/browser/[REDACTED]`,
|
||||
);
|
||||
assert.ok(
|
||||
!JSON.stringify(evidence).includes(path),
|
||||
"Browser-socket Evidence retained the socket identifier",
|
||||
);
|
||||
const throughHttpInspection = await provider.inspectPage(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: endpoint,
|
||||
allowed_origins: [origin],
|
||||
target_id: target,
|
||||
observation_ms: 100,
|
||||
}),
|
||||
);
|
||||
if (!throughHttpInspection.ok) throw throughHttpInspection.error;
|
||||
const compared = await provider.compareCaptures(
|
||||
compareWebCapturesInputSchema.parse({
|
||||
before: { inspection: throughHttpInspection.value },
|
||||
after: { inspection: evidence.normalized_result },
|
||||
}),
|
||||
);
|
||||
if (!compared.ok) throw compared.error;
|
||||
assert.equal(compared.value.dimensions.dom_structure.status, "unchanged");
|
||||
assert.equal(compared.value.dimensions.scripts.status, "unchanged");
|
||||
return {
|
||||
targets: throughSocket.targets.length,
|
||||
revision: throughSocket.browser.revision,
|
||||
};
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
export function assertCleanLinuxHopperRuntime(
|
||||
launcherPaths?: readonly string[],
|
||||
procRoot?: string,
|
||||
): Promise<void>;
|
||||
|
||||
@@ -16,6 +16,7 @@ export const assertCleanLinuxHopperRuntime = async (
|
||||
? []
|
||||
: [process.env.HOPPER_LAUNCHER_PATH]),
|
||||
],
|
||||
procRoot = "/proc",
|
||||
) => {
|
||||
const { stdout } = await execFileAsync("ps", ["-ax", "-o", "command="]);
|
||||
if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(stdout))
|
||||
@@ -32,17 +33,19 @@ export const assertCleanLinuxHopperRuntime = async (
|
||||
// /proc exposes the executed file even when argv[0] names a removed symlink.
|
||||
// Compare file identity, not a display path or a process name.
|
||||
let inspectionError;
|
||||
for (const entry of await readdir("/proc", { withFileTypes: true })) {
|
||||
for (const entry of await readdir(procRoot, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || !/^\d+$/u.test(entry.name)) continue;
|
||||
let executable;
|
||||
try {
|
||||
executable = await stat(`/proc/${entry.name}/exe`, { bigint: true });
|
||||
executable = await stat(`${procRoot}/${entry.name}/exe`, {
|
||||
bigint: true,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT" || error?.code === "ESRCH") continue;
|
||||
// Other users' executables may be unreadable. The setup lane launches
|
||||
// providers as the caller; unreadable caller-owned processes fail closed.
|
||||
if (error?.code === "EACCES") {
|
||||
const owner = await stat(`/proc/${entry.name}`).catch((cause) => {
|
||||
const owner = await stat(`${procRoot}/${entry.name}`).catch((cause) => {
|
||||
if (cause?.code === "ENOENT") return undefined;
|
||||
throw cause;
|
||||
});
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { isUtf8 } from "node:buffer";
|
||||
const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
||||
// The native decoder displays strict UTF-8 and retains invalid bytes as null text.
|
||||
const textOf = (expected) => {
|
||||
const value = Buffer.from(expected.base64, "base64");
|
||||
return isUtf8(value) ? value.toString("utf8") : null;
|
||||
};
|
||||
const field = (record, pointer) =>
|
||||
record.binary_fields.find((item) => item.pointer === pointer);
|
||||
const bytes = (record, pointer) => {
|
||||
const value = field(record, pointer);
|
||||
assert.equal(
|
||||
value?.state,
|
||||
"retained",
|
||||
`Missing retained bytes at ${pointer}`,
|
||||
);
|
||||
const decoded = Buffer.from(value.content_base64, "base64");
|
||||
assert.equal(value.bytes, decoded.length);
|
||||
assert.equal(value.sha256, digest(decoded));
|
||||
return decoded;
|
||||
};
|
||||
const binaryEqual = (record, pointer, expected) => {
|
||||
const decoded = bytes(record, pointer);
|
||||
assert.deepEqual(decoded, Buffer.from(expected.base64, "base64"), pointer);
|
||||
assert.equal(field(record, pointer).sha256, expected.sha256);
|
||||
assert.equal(field(record, pointer).bytes, expected.bytes);
|
||||
};
|
||||
const caseOf = (record) =>
|
||||
record.reported.request.headers.find(
|
||||
([name]) => name === "x-lab-case",
|
||||
)?.[1] ?? "curl";
|
||||
const byCase = (value, name) => {
|
||||
const selected = value.records.filter((record) => caseOf(record) === name);
|
||||
assert.equal(
|
||||
selected.length,
|
||||
1,
|
||||
`Expected exactly one producer record for ${name}`,
|
||||
);
|
||||
return selected[0];
|
||||
};
|
||||
const connectionH3 = (record, side) => {
|
||||
const conn = record.reported[side];
|
||||
assert.equal(conn.transport_protocol, "udp");
|
||||
assert.equal(bytes(record, `/${side}/alpn`).toString(), "h3");
|
||||
assert.equal(conn.alpn, "h3", `${side}.alpn readable value`);
|
||||
assert.equal(typeof conn.timestamp_tls_setup, "number");
|
||||
};
|
||||
|
||||
function assertHeaders(record, side, headers) {
|
||||
const regular = headers.filter(
|
||||
([name]) => !Buffer.from(name.base64, "base64").toString().startsWith(":"),
|
||||
);
|
||||
assert.equal(record.reported[side].headers.length, regular.length);
|
||||
for (const [index, pair] of regular.entries()) {
|
||||
binaryEqual(record, `/${side}/headers/${index}/0`, pair[0]);
|
||||
binaryEqual(record, `/${side}/headers/${index}/1`, pair[1]);
|
||||
assert.deepEqual(
|
||||
record.reported[side].headers[index],
|
||||
pair.map(textOf),
|
||||
`${side}.headers[${index}] readable values`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function assertEndpointHeaders(actual, expected) {
|
||||
// The proxy rebuilds pseudo fields from its HTTP state. Compare their values
|
||||
// by name while retaining the exact order and duplicates of regular headers.
|
||||
const partition = (headers) => {
|
||||
const pseudo = new Map();
|
||||
const regular = [];
|
||||
for (const pair of headers) {
|
||||
const name = Buffer.from(pair[0].base64, "base64").toString();
|
||||
if (name.startsWith(":")) {
|
||||
assert.equal(pseudo.has(name), false);
|
||||
pseudo.set(name, pair);
|
||||
} else regular.push(pair);
|
||||
}
|
||||
return { pseudo, regular };
|
||||
};
|
||||
assert.deepEqual(partition(actual), partition(expected));
|
||||
}
|
||||
|
||||
export function requireHttp3(record) {
|
||||
for (const side of ["request", "response"])
|
||||
assert.equal(record.reported[side].http_version, "HTTP/3");
|
||||
connectionH3(record, "client_conn");
|
||||
connectionH3(record, "server_conn");
|
||||
}
|
||||
|
||||
export function verifyNormalRecords(normal, manifest, finalProducer) {
|
||||
for (const record of normal.records) {
|
||||
const name = caseOf(record);
|
||||
const producer = finalProducer.find(
|
||||
(event) => event.id === record.reported.id,
|
||||
);
|
||||
assert.ok(producer, `No actual producer observation for ${name}`);
|
||||
assert.equal(record.reported.error, null);
|
||||
assert.equal(producer.error, null);
|
||||
assert.equal(record.reported.response.status_code, 200);
|
||||
requireHttp3(record);
|
||||
for (const side of ["request", "response"]) {
|
||||
assert.equal(
|
||||
record.reported[side].http_version,
|
||||
producer[side].http_version,
|
||||
);
|
||||
binaryEqual(record, `/${side}/content`, producer[side].content);
|
||||
assertHeaders(
|
||||
record,
|
||||
side,
|
||||
manifest[side === "request" ? "requests" : "responses"][name].headers,
|
||||
);
|
||||
}
|
||||
// Native UUIDs must be retained as UUIDs; the producer exports no stream mapping.
|
||||
for (const side of ["client_conn", "server_conn"])
|
||||
for (const key of ["id", "tls_version", "timestamp_tls_setup", "error"])
|
||||
assert.equal(
|
||||
record.reported[side][key],
|
||||
producer[side][key],
|
||||
`${side}.${key} differs from producer observation ${producer.id}`,
|
||||
);
|
||||
binaryEqual(record, "/request/content", manifest.requests[name].body);
|
||||
binaryEqual(record, "/response/content", manifest.responses[name].body);
|
||||
const pseudo = {
|
||||
":method": "method",
|
||||
":scheme": "scheme",
|
||||
":authority": "authority",
|
||||
":path": "path",
|
||||
};
|
||||
for (const [headerName, headerValue] of manifest.requests[name].headers) {
|
||||
const decodedName = Buffer.from(headerName.base64, "base64").toString();
|
||||
if (pseudo[decodedName]) {
|
||||
binaryEqual(record, `/request/${pseudo[decodedName]}`, headerValue);
|
||||
assert.equal(
|
||||
record.reported.request[pseudo[decodedName]],
|
||||
textOf(headerValue),
|
||||
`request.${pseudo[decodedName]} readable value`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export { bytes, caseOf, byCase };
|
||||
|
||||
export function verifyConcurrency(normal, manifest) {
|
||||
const parallelNames = ["parallel-0", "parallel-1", "parallel-2"];
|
||||
const sent = manifest.events.filter(
|
||||
(event) =>
|
||||
event.event === "request-sent" && parallelNames.includes(event.case),
|
||||
);
|
||||
const received = manifest.events.filter(
|
||||
(event) =>
|
||||
event.event === "request-received" && parallelNames.includes(event.case),
|
||||
);
|
||||
const responses = manifest.events.filter(
|
||||
(event) =>
|
||||
event.event === "response-sent" && parallelNames.includes(event.case),
|
||||
);
|
||||
assert.equal(new Set(sent.map((event) => event.connection)).size, 1);
|
||||
assert.equal(new Set(received.map((event) => event.connection)).size, 1);
|
||||
assert.equal(new Set(sent.map((event) => event.stream_id)).size, 3);
|
||||
assert.equal(new Set(received.map((event) => event.stream_id)).size, 3);
|
||||
assert.ok(
|
||||
Math.max(...received.map((event) => event.monotonic)) <
|
||||
responses[0].monotonic,
|
||||
);
|
||||
assert.deepEqual(
|
||||
responses.map((event) => event.case),
|
||||
received.map((event) => event.case).reverse(),
|
||||
);
|
||||
for (const side of ["client_conn", "server_conn"])
|
||||
assert.equal(
|
||||
new Set(
|
||||
parallelNames.map((name) => byCase(normal, name).reported[side].id),
|
||||
).size,
|
||||
1,
|
||||
);
|
||||
const normalConnections = new Set(
|
||||
[...sent, ...received].map((event) => event.connection),
|
||||
);
|
||||
for (const connection of normalConnections) {
|
||||
const handshakes = manifest.events.filter(
|
||||
(event) => event.event === "handshake" && event.connection === connection,
|
||||
);
|
||||
assert.equal(handshakes.length, 1);
|
||||
assert.equal(handshakes[0].alpn, "h3");
|
||||
assert.equal(handshakes[0].quic_version, 1);
|
||||
}
|
||||
assert.equal(
|
||||
new Set(normal.records.map((record) => record.reported.id)).size,
|
||||
4,
|
||||
);
|
||||
assert.equal(sent.length, 3);
|
||||
const receivedByClient = manifest.events.filter(
|
||||
(event) =>
|
||||
event.event === "response-received" &&
|
||||
sent.some(
|
||||
(request) =>
|
||||
request.connection === event.connection &&
|
||||
request.stream_id === event.stream_id,
|
||||
),
|
||||
);
|
||||
assert.equal(receivedByClient.length, 3);
|
||||
for (const request of sent) {
|
||||
const response = receivedByClient.find(
|
||||
(event) =>
|
||||
event.connection === request.connection &&
|
||||
event.stream_id === request.stream_id,
|
||||
);
|
||||
assert.ok(
|
||||
response,
|
||||
"Each sent request needs an independently observed client response",
|
||||
);
|
||||
assert.deepEqual(response.body, manifest.responses[request.case].body);
|
||||
assert.deepEqual(request.body, manifest.requests[request.case].body);
|
||||
assertEndpointHeaders(
|
||||
request.headers,
|
||||
manifest.requests[request.case].headers,
|
||||
);
|
||||
assertEndpointHeaders(
|
||||
response.headers,
|
||||
manifest.responses[request.case].headers,
|
||||
);
|
||||
}
|
||||
return {
|
||||
sent: sent.map((event) => event.case),
|
||||
completed_at_origin: received.map((event) => event.case),
|
||||
dispatched_at_origin: responses.map((event) => event.case),
|
||||
completed_at_client: receivedByClient.map(
|
||||
(event) =>
|
||||
sent.find(
|
||||
(request) =>
|
||||
request.connection === event.connection &&
|
||||
request.stream_id === event.stream_id,
|
||||
).case,
|
||||
),
|
||||
saved_in_capture: normal.records
|
||||
.filter((record) => parallelNames.includes(caseOf(record)))
|
||||
.map(caseOf),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { test } from "node:test";
|
||||
import { verifyNormalRecords } from "./http3-capture-assertions.mjs";
|
||||
|
||||
function connectionFactsFixture() {
|
||||
const body = Buffer.from([0, 255, 128, 1]);
|
||||
const view = (bytes) => ({
|
||||
base64: bytes.toString("base64"),
|
||||
bytes: bytes.length,
|
||||
sha256: createHash("sha256").update(bytes).digest("hex"),
|
||||
});
|
||||
const producer = {
|
||||
id: "flow-id",
|
||||
error: null,
|
||||
request: { http_version: "HTTP/3", content: view(body) },
|
||||
response: { http_version: "HTTP/3", content: view(body) },
|
||||
client_conn: {
|
||||
id: "client-id",
|
||||
transport_protocol: "udp",
|
||||
tls_version: "QUICv1",
|
||||
timestamp_tls_setup: 1_700_000_000.125678,
|
||||
error: null,
|
||||
},
|
||||
server_conn: {
|
||||
id: "server-id",
|
||||
transport_protocol: "udp",
|
||||
tls_version: "QUICv1",
|
||||
timestamp_tls_setup: 1_700_000_000.987654,
|
||||
error: null,
|
||||
},
|
||||
};
|
||||
const reported = structuredClone(producer);
|
||||
const requestFields = {
|
||||
method: "POST",
|
||||
scheme: "https",
|
||||
authority: "lab.test:18443",
|
||||
path: "/echo?case=readable",
|
||||
};
|
||||
Object.assign(reported.request, requestFields);
|
||||
const headers = [
|
||||
[Buffer.from("x-repeat"), Buffer.from("first-α")],
|
||||
[Buffer.from("x-repeat"), Buffer.from("second-β")],
|
||||
[Buffer.from("x-binary"), Buffer.from([255])],
|
||||
];
|
||||
for (const side of ["request", "response"])
|
||||
reported[side].headers = [
|
||||
["x-repeat", "first-α"],
|
||||
["x-repeat", "second-β"],
|
||||
["x-binary", null],
|
||||
];
|
||||
reported.client_conn.alpn = "h3";
|
||||
reported.server_conn.alpn = "h3";
|
||||
reported.response.status_code = 200;
|
||||
const binary_fields = [
|
||||
["/request/content", body],
|
||||
["/response/content", body],
|
||||
["/client_conn/alpn", Buffer.from("h3")],
|
||||
["/server_conn/alpn", Buffer.from("h3")],
|
||||
...Object.entries(requestFields).map(([name, value]) => [
|
||||
`/request/${name}`,
|
||||
Buffer.from(value),
|
||||
]),
|
||||
...["request", "response"].flatMap((side) =>
|
||||
headers.flatMap((pair, index) =>
|
||||
pair.map((value, part) => [`/${side}/headers/${index}/${part}`, value]),
|
||||
),
|
||||
),
|
||||
].map(([pointer, bytes]) => {
|
||||
const { base64, ...facts } = view(bytes);
|
||||
return {
|
||||
pointer,
|
||||
representation: "producer-bytes",
|
||||
state: "retained",
|
||||
content_base64: base64,
|
||||
...facts,
|
||||
};
|
||||
});
|
||||
return {
|
||||
normal: { records: [{ reported, binary_fields }] },
|
||||
manifest: {
|
||||
requests: {
|
||||
curl: {
|
||||
headers: [
|
||||
...Object.entries(requestFields).map(([name, value]) => [
|
||||
view(Buffer.from(`:${name}`)),
|
||||
view(Buffer.from(value)),
|
||||
]),
|
||||
...headers.map((pair) => pair.map(view)),
|
||||
],
|
||||
body: view(body),
|
||||
},
|
||||
},
|
||||
responses: {
|
||||
curl: {
|
||||
headers: headers.map((pair) => pair.map(view)),
|
||||
body: view(body),
|
||||
},
|
||||
},
|
||||
},
|
||||
producer: [producer],
|
||||
};
|
||||
}
|
||||
|
||||
test("positive assertions retain exact producer connection facts on both legs", () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
verifyNormalRecords(normal, manifest, producer);
|
||||
});
|
||||
|
||||
for (const [field, value] of [
|
||||
["method", "GET"],
|
||||
["scheme", "http"],
|
||||
["authority", "other.test:18443"],
|
||||
["path", "/different"],
|
||||
])
|
||||
test(`positive assertions reject changed readable request ${field}`, () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
normal.records[0].reported.request[field] = value;
|
||||
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
|
||||
for (const side of ["client_conn", "server_conn"])
|
||||
test(`positive assertions reject null readable ${side} ALPN`, () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
normal.records[0].reported[side].alpn = null;
|
||||
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
|
||||
for (const side of ["request", "response"])
|
||||
for (const [index, part, value] of [
|
||||
[0, 0, "x-changed"],
|
||||
[0, 1, "changed-first"],
|
||||
[1, 1, "changed-second"],
|
||||
[2, 1, "�"],
|
||||
])
|
||||
test(`positive assertions reject changed readable ${side} header ${index}/${part}`, () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
normal.records[0].reported[side].headers[index][part] = value;
|
||||
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
|
||||
for (const side of ["client_conn", "server_conn"])
|
||||
for (const [field, value] of [
|
||||
["tls_version", null],
|
||||
["tls_version", "changed-producer-label"],
|
||||
["error", "changed-connection-error"],
|
||||
["error", undefined],
|
||||
["timestamp_tls_setup", 0],
|
||||
["timestamp_tls_setup", 1_700_000_000],
|
||||
])
|
||||
test(`positive assertions reject ${side}.${field} changed to ${String(value)}`, () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
normal.records[0].reported[side][field] = value;
|
||||
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,143 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { cleanupOwnedProcessGroup } from "../../dist/process/ProcessOwnership.js";
|
||||
import { observeOwnedProcessGroups } from "../../dist/process/ProcessOwnershipObservation.js";
|
||||
|
||||
export const describeError = (error) => ({
|
||||
name: error?.name ?? "Error",
|
||||
message: error?.message ?? String(error),
|
||||
...(error?.stack ? { stack: error.stack } : {}),
|
||||
});
|
||||
|
||||
export async function attemptAll(actions) {
|
||||
const failures = [];
|
||||
for (const [name, action] of actions) {
|
||||
try {
|
||||
await action();
|
||||
} catch (error) {
|
||||
failures.push({ action: name, ...describeError(error) });
|
||||
}
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
async function latest(root, report) {
|
||||
const temporary = join(root, `.verification-${report.run_id}.tmp`);
|
||||
await writeFile(temporary, JSON.stringify(report, null, 2), { flag: "wx" });
|
||||
await rename(temporary, join(root, "verification-latest.json"));
|
||||
}
|
||||
|
||||
async function withLatestLock(root, action) {
|
||||
// Keep this pathname stable: removing it could split concurrent lock owners.
|
||||
// SQLite releases its OS-backed lock when the connection/process closes.
|
||||
const database = new DatabaseSync(join(root, ".verification-lock.sqlite"));
|
||||
try {
|
||||
for (;;) {
|
||||
try {
|
||||
database.exec("BEGIN EXCLUSIVE");
|
||||
break;
|
||||
} catch (error) {
|
||||
if (
|
||||
error.code !== "ERR_SQLITE_ERROR" ||
|
||||
(error.errcode !== 5 && error.errcode !== 6)
|
||||
)
|
||||
throw error;
|
||||
// Yield while another process (or async operation here) owns the lock.
|
||||
// A synchronous busy timeout would block that operation's fs callbacks.
|
||||
await setTimeout(10);
|
||||
}
|
||||
}
|
||||
return await action();
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function prepareAttempt(root, inspectExisting, run) {
|
||||
if (!inspectExisting) await mkdir(root, { mode: 0o700 });
|
||||
const directory = join(root, "verifications", run.run_id);
|
||||
await mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
await withLatestLock(root, async () => {
|
||||
// Preserve older reports as history, without leaving an ambiguous success
|
||||
// at the capture root. Registration is the shared marker's ordering point.
|
||||
try {
|
||||
await rename(
|
||||
join(root, "verification.json"),
|
||||
join(directory, "legacy-verification.json"),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
await latest(root, {
|
||||
run_id: run.run_id,
|
||||
status: "running",
|
||||
started_at: new Date().toISOString(),
|
||||
directory,
|
||||
});
|
||||
});
|
||||
return { directory, captureRoot: join(root, "captures") };
|
||||
}
|
||||
|
||||
export async function publishAttempt(root, directory, report) {
|
||||
await writeFile(
|
||||
join(directory, "result.json"),
|
||||
JSON.stringify(report, null, 2),
|
||||
{ flag: "wx" },
|
||||
);
|
||||
if (report.status === "passed")
|
||||
await writeFile(
|
||||
join(directory, "verification.json"),
|
||||
JSON.stringify(report, null, 2),
|
||||
{ flag: "wx" },
|
||||
);
|
||||
await withLatestLock(root, async () => {
|
||||
const current = JSON.parse(
|
||||
await readFile(join(root, "verification-latest.json"), "utf8"),
|
||||
);
|
||||
if (current.run_id !== report.run_id) return;
|
||||
// Ownership check and replacement share one lock with attempt registration.
|
||||
// An older attempt still keeps its reports, but cannot replace this marker.
|
||||
await latest(root, {
|
||||
run_id: report.run_id,
|
||||
status: report.status,
|
||||
directory,
|
||||
finished_at: report.finished_at,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function finishProducerGroups(captureRoot, runId) {
|
||||
let leases;
|
||||
try {
|
||||
leases = (await readFile(join(captureRoot, "process-groups.jsonl"), "utf8"))
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map(JSON.parse);
|
||||
} catch (error) {
|
||||
if (error.code === "ENOENT") return { groups: [], observation: [] };
|
||||
throw error;
|
||||
}
|
||||
for (const lease of leases) assert.equal(lease.runId, runId);
|
||||
const failures = await attemptAll(
|
||||
leases.map((lease) => [
|
||||
`producer-group-${lease.processGroupId}`,
|
||||
async () => {
|
||||
const result = await cleanupOwnedProcessGroup(lease);
|
||||
assert.equal(result.cleaned, true, JSON.stringify(result));
|
||||
},
|
||||
]),
|
||||
);
|
||||
// Inspect even when one close failed; the groups can outlive their bootloader
|
||||
// parent and disappear from a plain descendant-only observation.
|
||||
const observation = [
|
||||
...(await observeOwnedProcessGroups(
|
||||
runId,
|
||||
leases.map((lease) => lease.processGroupId),
|
||||
)),
|
||||
];
|
||||
return { groups: leases, failures, observation };
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { spawn } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { once } from "node:events";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
attemptAll,
|
||||
prepareAttempt,
|
||||
publishAttempt,
|
||||
finishProducerGroups,
|
||||
} from "./http3-verifier-lifecycle.mjs";
|
||||
import { cleanupOwnedProcessGroup } from "../../dist/process/ProcessOwnership.js";
|
||||
import { runOwnedCommand } from "../../dist/process/OwnedCommand.js";
|
||||
import { verifyConcurrency } from "./http3-capture-assertions.mjs";
|
||||
|
||||
async function waitForFile(path) {
|
||||
const signal = AbortSignal.timeout(5_000);
|
||||
for (;;) {
|
||||
try {
|
||||
await readFile(path);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
await setTimeout(10, undefined, { signal });
|
||||
}
|
||||
}
|
||||
|
||||
function startAttemptWorker(root, report) {
|
||||
const worker = `
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { prepareAttempt, publishAttempt } from ${JSON.stringify(
|
||||
new URL("./http3-verifier-lifecycle.mjs", import.meta.url).href,
|
||||
)};
|
||||
const [root, serialized] = process.argv.slice(1);
|
||||
const report = JSON.parse(serialized);
|
||||
const attempt = await prepareAttempt(root, true, report);
|
||||
await writeFile(join(root, report.run_id + ".ready"), "");
|
||||
for (;;) {
|
||||
try { await readFile(join(root, report.run_id + ".finish")); break; }
|
||||
catch (error) { if (error.code !== "ENOENT") throw error; }
|
||||
await setTimeout(10);
|
||||
}
|
||||
await publishAttempt(root, attempt.directory, report);
|
||||
`;
|
||||
return runOwnedCommand(
|
||||
{
|
||||
command: process.execPath,
|
||||
arguments: [
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
worker,
|
||||
root,
|
||||
JSON.stringify(report),
|
||||
],
|
||||
cwd: root,
|
||||
hostEnvironment: process.env,
|
||||
runId: randomUUID(),
|
||||
},
|
||||
{ timeoutMs: 10_000, diagnosticBytes: 4096 },
|
||||
).then(
|
||||
(result) => ({ result }),
|
||||
(error) => ({ error }),
|
||||
);
|
||||
}
|
||||
|
||||
test("cleanup continues through failed closes and reaches its audit", async () => {
|
||||
const attempted = [];
|
||||
const failures = await attemptAll(
|
||||
["client", "transport", "audit"].map((name) => [
|
||||
name,
|
||||
async () => {
|
||||
attempted.push(name);
|
||||
if (name !== "audit") throw new Error(`${name} failed`);
|
||||
},
|
||||
]),
|
||||
);
|
||||
assert.deepEqual(attempted, ["client", "transport", "audit"]);
|
||||
assert.deepEqual(
|
||||
failures.map((failure) => failure.action),
|
||||
["client", "transport"],
|
||||
);
|
||||
});
|
||||
|
||||
test("a failed reinspection cannot leave the earlier success as the current result", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-results-"));
|
||||
try {
|
||||
await writeFile(
|
||||
join(root, "verification.json"),
|
||||
'{"status":"passed","run_id":"legacy"}',
|
||||
);
|
||||
const first = await prepareAttempt(root, true, { run_id: "first" });
|
||||
await publishAttempt(root, first.directory, {
|
||||
run_id: "first",
|
||||
status: "passed",
|
||||
});
|
||||
const next = await prepareAttempt(root, true, { run_id: "next" });
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(root, "verification-latest.json"))).status,
|
||||
"running",
|
||||
);
|
||||
await publishAttempt(root, next.directory, {
|
||||
run_id: "next",
|
||||
status: "failed",
|
||||
error: { message: "new failure" },
|
||||
});
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(root, "verification-latest.json"))).run_id,
|
||||
"next",
|
||||
);
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(root, "verification-latest.json"))).status,
|
||||
"failed",
|
||||
);
|
||||
await assert.rejects(readFile(join(next.directory, "verification.json")), {
|
||||
code: "ENOENT",
|
||||
});
|
||||
await assert.rejects(readFile(join(root, "verification.json")), {
|
||||
code: "ENOENT",
|
||||
});
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(first.directory, "verification.json")))
|
||||
.status,
|
||||
"passed",
|
||||
);
|
||||
assert.equal(
|
||||
JSON.parse(
|
||||
await readFile(join(first.directory, "legacy-verification.json")),
|
||||
).run_id,
|
||||
"legacy",
|
||||
);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("separate overlapping attempts finishing in reverse order retain the newer failure", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-overlap-"));
|
||||
const commands = [];
|
||||
const failed = {
|
||||
run_id: "newer",
|
||||
status: "failed",
|
||||
error: { message: "newer inspection failed" },
|
||||
cleanup_failures: [{ action: "transport-close", message: "close failed" }],
|
||||
};
|
||||
let primaryError;
|
||||
const start = (report) => {
|
||||
const command = startAttemptWorker(root, report);
|
||||
commands.push(command);
|
||||
return command;
|
||||
};
|
||||
try {
|
||||
const older = start({ run_id: "older", status: "passed" });
|
||||
await waitForFile(join(root, "older.ready"));
|
||||
const newer = start(failed);
|
||||
await waitForFile(join(root, "newer.ready"));
|
||||
await writeFile(join(root, "newer.finish"), "");
|
||||
const newerResult = await newer;
|
||||
if (newerResult.error) throw newerResult.error;
|
||||
assert.equal(newerResult.result.exitCode, 0);
|
||||
assert.equal(newerResult.result.signal, null);
|
||||
const marker = await readFile(join(root, "verification-latest.json"));
|
||||
await writeFile(join(root, "older.finish"), "");
|
||||
const olderResult = await older;
|
||||
if (olderResult.error) throw olderResult.error;
|
||||
assert.equal(olderResult.result.exitCode, 0);
|
||||
assert.equal(olderResult.result.signal, null);
|
||||
assert.deepEqual(
|
||||
await readFile(join(root, "verification-latest.json")),
|
||||
marker,
|
||||
);
|
||||
assert.deepEqual(
|
||||
JSON.parse(await readFile(join(root, "verifications/newer/result.json"))),
|
||||
failed,
|
||||
);
|
||||
await assert.rejects(
|
||||
readFile(join(root, "verifications/newer/verification.json")),
|
||||
{
|
||||
code: "ENOENT",
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
JSON.parse(
|
||||
await readFile(join(root, "verifications/older/verification.json")),
|
||||
).status,
|
||||
"passed",
|
||||
);
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
} finally {
|
||||
// Release both barriers even after an assertion/setup failure. Each command
|
||||
// retains native supervision through termination before the root is removed.
|
||||
const failures = await attemptAll(
|
||||
["older", "newer"].map((name) => [
|
||||
name,
|
||||
() => writeFile(join(root, name + ".finish"), ""),
|
||||
]),
|
||||
);
|
||||
const outcomes = await Promise.all(commands);
|
||||
const retained = outcomes.filter(
|
||||
({ error }) => error?.cleanupFailure || error?.cleanupResult,
|
||||
);
|
||||
if (retained.length)
|
||||
primaryError = new AggregateError(
|
||||
[primaryError, ...retained.map(({ error }) => error)].filter(Boolean),
|
||||
`Worker cleanup failed; retained test evidence at ${root}`,
|
||||
{ cause: primaryError },
|
||||
);
|
||||
else
|
||||
failures.push(
|
||||
...(await attemptAll([
|
||||
["root", () => rm(root, { recursive: true, force: true })],
|
||||
])),
|
||||
);
|
||||
if (failures.length)
|
||||
primaryError = new AggregateError(
|
||||
[primaryError, ...failures].filter(Boolean),
|
||||
"Test cleanup failed",
|
||||
{ cause: primaryError },
|
||||
);
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
});
|
||||
|
||||
test("publication waits for the lock before reading the current owner", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-lock-"));
|
||||
let database;
|
||||
let publication;
|
||||
let primaryError;
|
||||
try {
|
||||
const older = await prepareAttempt(root, true, { run_id: "older" });
|
||||
database = new DatabaseSync(join(root, ".verification-lock.sqlite"));
|
||||
database.exec("BEGIN EXCLUSIVE");
|
||||
publication = publishAttempt(root, older.directory, {
|
||||
run_id: "older",
|
||||
status: "passed",
|
||||
});
|
||||
await waitForFile(join(older.directory, "verification.json"));
|
||||
const marker = join(root, "verification-latest.json");
|
||||
assert.equal(JSON.parse(await readFile(marker)).status, "running");
|
||||
// This connection represents another marker writer. Its replacement must
|
||||
// be observed only after the blocked publisher acquires the same lock.
|
||||
const newer = { run_id: "newer", status: "failed" };
|
||||
await writeFile(marker, JSON.stringify(newer));
|
||||
database.close();
|
||||
database = undefined;
|
||||
await publication;
|
||||
assert.deepEqual(JSON.parse(await readFile(marker)), newer);
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
} finally {
|
||||
const failures = await attemptAll([
|
||||
["lock", () => database?.close()],
|
||||
[
|
||||
"publication",
|
||||
async () => {
|
||||
if (publication) await publication;
|
||||
},
|
||||
],
|
||||
["root", () => rm(root, { recursive: true, force: true })],
|
||||
]);
|
||||
if (failures.length)
|
||||
primaryError = new AggregateError(
|
||||
[primaryError, ...failures].filter(Boolean),
|
||||
"Test cleanup failed",
|
||||
{ cause: primaryError },
|
||||
);
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
});
|
||||
|
||||
test("a failed marker read releases the lock and preserves the attempt report", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-lock-failure-"));
|
||||
try {
|
||||
const older = await prepareAttempt(root, true, { run_id: "older" });
|
||||
await writeFile(join(root, "verification-latest.json"), "{");
|
||||
const report = {
|
||||
run_id: "older",
|
||||
status: "failed",
|
||||
error: { message: "inspection failed" },
|
||||
};
|
||||
await assert.rejects(
|
||||
publishAttempt(root, older.directory, report),
|
||||
SyntaxError,
|
||||
);
|
||||
assert.deepEqual(
|
||||
JSON.parse(await readFile(join(older.directory, "result.json"))),
|
||||
report,
|
||||
);
|
||||
const next = await prepareAttempt(root, true, { run_id: "next" });
|
||||
await publishAttempt(root, next.directory, {
|
||||
run_id: "next",
|
||||
status: "passed",
|
||||
});
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(root, "verification-latest.json"))).run_id,
|
||||
"next",
|
||||
);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("dispatch reverses origin completion; client and capture may arrive in other orders", () => {
|
||||
const names = ["parallel-0", "parallel-1", "parallel-2"];
|
||||
const header = (text) => ({ base64: Buffer.from(text).toString("base64") });
|
||||
const messages = Object.fromEntries(
|
||||
names.map((name) => [
|
||||
name,
|
||||
{
|
||||
headers: [[header("x-lab-case"), header(name)]],
|
||||
body: { base64: header(name).base64 },
|
||||
},
|
||||
]),
|
||||
);
|
||||
const event = (kind, index, connection, monotonic) => ({
|
||||
event: kind,
|
||||
case: names[index],
|
||||
stream_id: index * 4,
|
||||
connection,
|
||||
monotonic,
|
||||
...messages[names[index]],
|
||||
});
|
||||
const manifest = {
|
||||
requests: messages,
|
||||
responses: messages,
|
||||
events: [
|
||||
...[0, 1, 2].map((i) => event("request-sent", i, "client", i)),
|
||||
...[1, 2, 0].map((i) => event("request-received", i, "origin", 10)),
|
||||
...[0, 2, 1].map((i) => event("response-sent", i, "origin", 20)),
|
||||
...[2, 0, 1].map((i) => event("response-received", i, "client", 30)),
|
||||
...["client", "origin"].map((connection) => ({
|
||||
event: "handshake",
|
||||
connection,
|
||||
alpn: "h3",
|
||||
quic_version: 1,
|
||||
})),
|
||||
],
|
||||
};
|
||||
const record = (name) => ({
|
||||
reported: {
|
||||
id: name,
|
||||
request: { headers: [["x-lab-case", name]] },
|
||||
client_conn: { id: "client-uuid" },
|
||||
server_conn: { id: "origin-uuid" },
|
||||
},
|
||||
});
|
||||
const normal = {
|
||||
records: [...[1, 0, 2].map((i) => record(names[i])), record("curl")],
|
||||
};
|
||||
const orders = verifyConcurrency(normal, manifest);
|
||||
assert.deepEqual(orders.dispatched_at_origin, [
|
||||
"parallel-0",
|
||||
"parallel-2",
|
||||
"parallel-1",
|
||||
]);
|
||||
manifest.events = manifest.events.filter(
|
||||
(item) => item.event !== "response-sent",
|
||||
);
|
||||
manifest.events.push(
|
||||
...[2, 1, 0].map((i) => event("response-sent", i, "origin", 20)),
|
||||
);
|
||||
assert.throws(() => verifyConcurrency(normal, manifest), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
|
||||
test("cleanup verifies and retires a separately owned producer group", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-group-"));
|
||||
const runId = randomUUID();
|
||||
const child = spawn("/usr/bin/sleep", ["60"], {
|
||||
detached: true,
|
||||
env: { ...process.env, REA_PROCESS_RUN_ID: runId },
|
||||
stdio: "ignore",
|
||||
});
|
||||
let primaryError;
|
||||
let cleaned = false;
|
||||
try {
|
||||
await once(child, "spawn");
|
||||
await writeFile(
|
||||
join(root, "process-groups.jsonl"),
|
||||
JSON.stringify({
|
||||
runId,
|
||||
leaderPid: child.pid,
|
||||
processGroupId: child.pid,
|
||||
}) + "\n",
|
||||
);
|
||||
const result = await finishProducerGroups(root, runId);
|
||||
assert.deepEqual(result.failures, []);
|
||||
assert.equal(result.observation[0][1].state, "empty");
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
} finally {
|
||||
const failures = await attemptAll([
|
||||
[
|
||||
"process",
|
||||
async () => {
|
||||
const result = await cleanupOwnedProcessGroup({
|
||||
runId,
|
||||
leaderPid: child.pid,
|
||||
processGroupId: child.pid,
|
||||
});
|
||||
assert.equal(
|
||||
result.cleaned,
|
||||
true,
|
||||
`Retained ownership evidence at ${root}: ${JSON.stringify(result)}`,
|
||||
);
|
||||
cleaned = true;
|
||||
},
|
||||
],
|
||||
[
|
||||
"root",
|
||||
async () => {
|
||||
if (cleaned) await rm(root, { recursive: true, force: true });
|
||||
},
|
||||
],
|
||||
]);
|
||||
if (failures.length)
|
||||
primaryError = new AggregateError(
|
||||
[primaryError, ...failures].filter(Boolean),
|
||||
"Test cleanup failed",
|
||||
{ cause: primaryError },
|
||||
);
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
});
|
||||
Reference in New Issue
Block a user