test(http3): add real MITM capture verification lab (#1997)
* test(http3): add real MITM capture verification lab * test(http3): register the optional verification command * fix(http3): preserve the latest verification attempt owner * test(cli): admit the dyld slice fixture on x64 hosts * test(http3): verify exact producer TLS connection facts * test(http3): verify readable capture fields against observed bytes * fix(comparison): recheck historical calls-only changes (#2223) * fix(comparison): preserve incomplete function call evidence Keep missing, unknown, partial and unresolved call relationships unknown in function comparisons and reconstruction claims. Aggregate IDA caller and callee coverage without overwriting uncertainty. Validate with 144 focused tests, typecheck/lint, docs checks and test discovery. Real providers were not verified; the full headless IDA session suite requires unavailable Windows native controls. * fix(comparison): recheck historical calls-only changes --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> * fix(android): make DEX descriptor pattern portable * fix(ci): isolate Linux Hopper process fixture inventory * fix(http3): retain MCP startup diagnostics --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Lucien <164857476+dakjdakd@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,243 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { isUtf8 } from "node:buffer";
|
||||
const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
||||
// The native decoder displays strict UTF-8 and retains invalid bytes as null text.
|
||||
const textOf = (expected) => {
|
||||
const value = Buffer.from(expected.base64, "base64");
|
||||
return isUtf8(value) ? value.toString("utf8") : null;
|
||||
};
|
||||
const field = (record, pointer) =>
|
||||
record.binary_fields.find((item) => item.pointer === pointer);
|
||||
const bytes = (record, pointer) => {
|
||||
const value = field(record, pointer);
|
||||
assert.equal(
|
||||
value?.state,
|
||||
"retained",
|
||||
`Missing retained bytes at ${pointer}`,
|
||||
);
|
||||
const decoded = Buffer.from(value.content_base64, "base64");
|
||||
assert.equal(value.bytes, decoded.length);
|
||||
assert.equal(value.sha256, digest(decoded));
|
||||
return decoded;
|
||||
};
|
||||
const binaryEqual = (record, pointer, expected) => {
|
||||
const decoded = bytes(record, pointer);
|
||||
assert.deepEqual(decoded, Buffer.from(expected.base64, "base64"), pointer);
|
||||
assert.equal(field(record, pointer).sha256, expected.sha256);
|
||||
assert.equal(field(record, pointer).bytes, expected.bytes);
|
||||
};
|
||||
const caseOf = (record) =>
|
||||
record.reported.request.headers.find(
|
||||
([name]) => name === "x-lab-case",
|
||||
)?.[1] ?? "curl";
|
||||
const byCase = (value, name) => {
|
||||
const selected = value.records.filter((record) => caseOf(record) === name);
|
||||
assert.equal(
|
||||
selected.length,
|
||||
1,
|
||||
`Expected exactly one producer record for ${name}`,
|
||||
);
|
||||
return selected[0];
|
||||
};
|
||||
const connectionH3 = (record, side) => {
|
||||
const conn = record.reported[side];
|
||||
assert.equal(conn.transport_protocol, "udp");
|
||||
assert.equal(bytes(record, `/${side}/alpn`).toString(), "h3");
|
||||
assert.equal(conn.alpn, "h3", `${side}.alpn readable value`);
|
||||
assert.equal(typeof conn.timestamp_tls_setup, "number");
|
||||
};
|
||||
|
||||
function assertHeaders(record, side, headers) {
|
||||
const regular = headers.filter(
|
||||
([name]) => !Buffer.from(name.base64, "base64").toString().startsWith(":"),
|
||||
);
|
||||
assert.equal(record.reported[side].headers.length, regular.length);
|
||||
for (const [index, pair] of regular.entries()) {
|
||||
binaryEqual(record, `/${side}/headers/${index}/0`, pair[0]);
|
||||
binaryEqual(record, `/${side}/headers/${index}/1`, pair[1]);
|
||||
assert.deepEqual(
|
||||
record.reported[side].headers[index],
|
||||
pair.map(textOf),
|
||||
`${side}.headers[${index}] readable values`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function assertEndpointHeaders(actual, expected) {
|
||||
// The proxy rebuilds pseudo fields from its HTTP state. Compare their values
|
||||
// by name while retaining the exact order and duplicates of regular headers.
|
||||
const partition = (headers) => {
|
||||
const pseudo = new Map();
|
||||
const regular = [];
|
||||
for (const pair of headers) {
|
||||
const name = Buffer.from(pair[0].base64, "base64").toString();
|
||||
if (name.startsWith(":")) {
|
||||
assert.equal(pseudo.has(name), false);
|
||||
pseudo.set(name, pair);
|
||||
} else regular.push(pair);
|
||||
}
|
||||
return { pseudo, regular };
|
||||
};
|
||||
assert.deepEqual(partition(actual), partition(expected));
|
||||
}
|
||||
|
||||
export function requireHttp3(record) {
|
||||
for (const side of ["request", "response"])
|
||||
assert.equal(record.reported[side].http_version, "HTTP/3");
|
||||
connectionH3(record, "client_conn");
|
||||
connectionH3(record, "server_conn");
|
||||
}
|
||||
|
||||
export function verifyNormalRecords(normal, manifest, finalProducer) {
|
||||
for (const record of normal.records) {
|
||||
const name = caseOf(record);
|
||||
const producer = finalProducer.find(
|
||||
(event) => event.id === record.reported.id,
|
||||
);
|
||||
assert.ok(producer, `No actual producer observation for ${name}`);
|
||||
assert.equal(record.reported.error, null);
|
||||
assert.equal(producer.error, null);
|
||||
assert.equal(record.reported.response.status_code, 200);
|
||||
requireHttp3(record);
|
||||
for (const side of ["request", "response"]) {
|
||||
assert.equal(
|
||||
record.reported[side].http_version,
|
||||
producer[side].http_version,
|
||||
);
|
||||
binaryEqual(record, `/${side}/content`, producer[side].content);
|
||||
assertHeaders(
|
||||
record,
|
||||
side,
|
||||
manifest[side === "request" ? "requests" : "responses"][name].headers,
|
||||
);
|
||||
}
|
||||
// Native UUIDs must be retained as UUIDs; the producer exports no stream mapping.
|
||||
for (const side of ["client_conn", "server_conn"])
|
||||
for (const key of ["id", "tls_version", "timestamp_tls_setup", "error"])
|
||||
assert.equal(
|
||||
record.reported[side][key],
|
||||
producer[side][key],
|
||||
`${side}.${key} differs from producer observation ${producer.id}`,
|
||||
);
|
||||
binaryEqual(record, "/request/content", manifest.requests[name].body);
|
||||
binaryEqual(record, "/response/content", manifest.responses[name].body);
|
||||
const pseudo = {
|
||||
":method": "method",
|
||||
":scheme": "scheme",
|
||||
":authority": "authority",
|
||||
":path": "path",
|
||||
};
|
||||
for (const [headerName, headerValue] of manifest.requests[name].headers) {
|
||||
const decodedName = Buffer.from(headerName.base64, "base64").toString();
|
||||
if (pseudo[decodedName]) {
|
||||
binaryEqual(record, `/request/${pseudo[decodedName]}`, headerValue);
|
||||
assert.equal(
|
||||
record.reported.request[pseudo[decodedName]],
|
||||
textOf(headerValue),
|
||||
`request.${pseudo[decodedName]} readable value`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export { bytes, caseOf, byCase };
|
||||
|
||||
export function verifyConcurrency(normal, manifest) {
|
||||
const parallelNames = ["parallel-0", "parallel-1", "parallel-2"];
|
||||
const sent = manifest.events.filter(
|
||||
(event) =>
|
||||
event.event === "request-sent" && parallelNames.includes(event.case),
|
||||
);
|
||||
const received = manifest.events.filter(
|
||||
(event) =>
|
||||
event.event === "request-received" && parallelNames.includes(event.case),
|
||||
);
|
||||
const responses = manifest.events.filter(
|
||||
(event) =>
|
||||
event.event === "response-sent" && parallelNames.includes(event.case),
|
||||
);
|
||||
assert.equal(new Set(sent.map((event) => event.connection)).size, 1);
|
||||
assert.equal(new Set(received.map((event) => event.connection)).size, 1);
|
||||
assert.equal(new Set(sent.map((event) => event.stream_id)).size, 3);
|
||||
assert.equal(new Set(received.map((event) => event.stream_id)).size, 3);
|
||||
assert.ok(
|
||||
Math.max(...received.map((event) => event.monotonic)) <
|
||||
responses[0].monotonic,
|
||||
);
|
||||
assert.deepEqual(
|
||||
responses.map((event) => event.case),
|
||||
received.map((event) => event.case).reverse(),
|
||||
);
|
||||
for (const side of ["client_conn", "server_conn"])
|
||||
assert.equal(
|
||||
new Set(
|
||||
parallelNames.map((name) => byCase(normal, name).reported[side].id),
|
||||
).size,
|
||||
1,
|
||||
);
|
||||
const normalConnections = new Set(
|
||||
[...sent, ...received].map((event) => event.connection),
|
||||
);
|
||||
for (const connection of normalConnections) {
|
||||
const handshakes = manifest.events.filter(
|
||||
(event) => event.event === "handshake" && event.connection === connection,
|
||||
);
|
||||
assert.equal(handshakes.length, 1);
|
||||
assert.equal(handshakes[0].alpn, "h3");
|
||||
assert.equal(handshakes[0].quic_version, 1);
|
||||
}
|
||||
assert.equal(
|
||||
new Set(normal.records.map((record) => record.reported.id)).size,
|
||||
4,
|
||||
);
|
||||
assert.equal(sent.length, 3);
|
||||
const receivedByClient = manifest.events.filter(
|
||||
(event) =>
|
||||
event.event === "response-received" &&
|
||||
sent.some(
|
||||
(request) =>
|
||||
request.connection === event.connection &&
|
||||
request.stream_id === event.stream_id,
|
||||
),
|
||||
);
|
||||
assert.equal(receivedByClient.length, 3);
|
||||
for (const request of sent) {
|
||||
const response = receivedByClient.find(
|
||||
(event) =>
|
||||
event.connection === request.connection &&
|
||||
event.stream_id === request.stream_id,
|
||||
);
|
||||
assert.ok(
|
||||
response,
|
||||
"Each sent request needs an independently observed client response",
|
||||
);
|
||||
assert.deepEqual(response.body, manifest.responses[request.case].body);
|
||||
assert.deepEqual(request.body, manifest.requests[request.case].body);
|
||||
assertEndpointHeaders(
|
||||
request.headers,
|
||||
manifest.requests[request.case].headers,
|
||||
);
|
||||
assertEndpointHeaders(
|
||||
response.headers,
|
||||
manifest.responses[request.case].headers,
|
||||
);
|
||||
}
|
||||
return {
|
||||
sent: sent.map((event) => event.case),
|
||||
completed_at_origin: received.map((event) => event.case),
|
||||
dispatched_at_origin: responses.map((event) => event.case),
|
||||
completed_at_client: receivedByClient.map(
|
||||
(event) =>
|
||||
sent.find(
|
||||
(request) =>
|
||||
request.connection === event.connection &&
|
||||
request.stream_id === event.stream_id,
|
||||
).case,
|
||||
),
|
||||
saved_in_capture: normal.records
|
||||
.filter((record) => parallelNames.includes(caseOf(record)))
|
||||
.map(caseOf),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { test } from "node:test";
|
||||
import { verifyNormalRecords } from "./http3-capture-assertions.mjs";
|
||||
|
||||
function connectionFactsFixture() {
|
||||
const body = Buffer.from([0, 255, 128, 1]);
|
||||
const view = (bytes) => ({
|
||||
base64: bytes.toString("base64"),
|
||||
bytes: bytes.length,
|
||||
sha256: createHash("sha256").update(bytes).digest("hex"),
|
||||
});
|
||||
const producer = {
|
||||
id: "flow-id",
|
||||
error: null,
|
||||
request: { http_version: "HTTP/3", content: view(body) },
|
||||
response: { http_version: "HTTP/3", content: view(body) },
|
||||
client_conn: {
|
||||
id: "client-id",
|
||||
transport_protocol: "udp",
|
||||
tls_version: "QUICv1",
|
||||
timestamp_tls_setup: 1_700_000_000.125678,
|
||||
error: null,
|
||||
},
|
||||
server_conn: {
|
||||
id: "server-id",
|
||||
transport_protocol: "udp",
|
||||
tls_version: "QUICv1",
|
||||
timestamp_tls_setup: 1_700_000_000.987654,
|
||||
error: null,
|
||||
},
|
||||
};
|
||||
const reported = structuredClone(producer);
|
||||
const requestFields = {
|
||||
method: "POST",
|
||||
scheme: "https",
|
||||
authority: "lab.test:18443",
|
||||
path: "/echo?case=readable",
|
||||
};
|
||||
Object.assign(reported.request, requestFields);
|
||||
const headers = [
|
||||
[Buffer.from("x-repeat"), Buffer.from("first-α")],
|
||||
[Buffer.from("x-repeat"), Buffer.from("second-β")],
|
||||
[Buffer.from("x-binary"), Buffer.from([255])],
|
||||
];
|
||||
for (const side of ["request", "response"])
|
||||
reported[side].headers = [
|
||||
["x-repeat", "first-α"],
|
||||
["x-repeat", "second-β"],
|
||||
["x-binary", null],
|
||||
];
|
||||
reported.client_conn.alpn = "h3";
|
||||
reported.server_conn.alpn = "h3";
|
||||
reported.response.status_code = 200;
|
||||
const binary_fields = [
|
||||
["/request/content", body],
|
||||
["/response/content", body],
|
||||
["/client_conn/alpn", Buffer.from("h3")],
|
||||
["/server_conn/alpn", Buffer.from("h3")],
|
||||
...Object.entries(requestFields).map(([name, value]) => [
|
||||
`/request/${name}`,
|
||||
Buffer.from(value),
|
||||
]),
|
||||
...["request", "response"].flatMap((side) =>
|
||||
headers.flatMap((pair, index) =>
|
||||
pair.map((value, part) => [`/${side}/headers/${index}/${part}`, value]),
|
||||
),
|
||||
),
|
||||
].map(([pointer, bytes]) => {
|
||||
const { base64, ...facts } = view(bytes);
|
||||
return {
|
||||
pointer,
|
||||
representation: "producer-bytes",
|
||||
state: "retained",
|
||||
content_base64: base64,
|
||||
...facts,
|
||||
};
|
||||
});
|
||||
return {
|
||||
normal: { records: [{ reported, binary_fields }] },
|
||||
manifest: {
|
||||
requests: {
|
||||
curl: {
|
||||
headers: [
|
||||
...Object.entries(requestFields).map(([name, value]) => [
|
||||
view(Buffer.from(`:${name}`)),
|
||||
view(Buffer.from(value)),
|
||||
]),
|
||||
...headers.map((pair) => pair.map(view)),
|
||||
],
|
||||
body: view(body),
|
||||
},
|
||||
},
|
||||
responses: {
|
||||
curl: {
|
||||
headers: headers.map((pair) => pair.map(view)),
|
||||
body: view(body),
|
||||
},
|
||||
},
|
||||
},
|
||||
producer: [producer],
|
||||
};
|
||||
}
|
||||
|
||||
test("positive assertions retain exact producer connection facts on both legs", () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
verifyNormalRecords(normal, manifest, producer);
|
||||
});
|
||||
|
||||
for (const [field, value] of [
|
||||
["method", "GET"],
|
||||
["scheme", "http"],
|
||||
["authority", "other.test:18443"],
|
||||
["path", "/different"],
|
||||
])
|
||||
test(`positive assertions reject changed readable request ${field}`, () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
normal.records[0].reported.request[field] = value;
|
||||
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
|
||||
for (const side of ["client_conn", "server_conn"])
|
||||
test(`positive assertions reject null readable ${side} ALPN`, () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
normal.records[0].reported[side].alpn = null;
|
||||
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
|
||||
for (const side of ["request", "response"])
|
||||
for (const [index, part, value] of [
|
||||
[0, 0, "x-changed"],
|
||||
[0, 1, "changed-first"],
|
||||
[1, 1, "changed-second"],
|
||||
[2, 1, "�"],
|
||||
])
|
||||
test(`positive assertions reject changed readable ${side} header ${index}/${part}`, () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
normal.records[0].reported[side].headers[index][part] = value;
|
||||
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
|
||||
for (const side of ["client_conn", "server_conn"])
|
||||
for (const [field, value] of [
|
||||
["tls_version", null],
|
||||
["tls_version", "changed-producer-label"],
|
||||
["error", "changed-connection-error"],
|
||||
["error", undefined],
|
||||
["timestamp_tls_setup", 0],
|
||||
["timestamp_tls_setup", 1_700_000_000],
|
||||
])
|
||||
test(`positive assertions reject ${side}.${field} changed to ${String(value)}`, () => {
|
||||
const { normal, manifest, producer } = connectionFactsFixture();
|
||||
normal.records[0].reported[side][field] = value;
|
||||
assert.throws(() => verifyNormalRecords(normal, manifest, producer), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,143 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { cleanupOwnedProcessGroup } from "../../dist/process/ProcessOwnership.js";
|
||||
import { observeOwnedProcessGroups } from "../../dist/process/ProcessOwnershipObservation.js";
|
||||
|
||||
export const describeError = (error) => ({
|
||||
name: error?.name ?? "Error",
|
||||
message: error?.message ?? String(error),
|
||||
...(error?.stack ? { stack: error.stack } : {}),
|
||||
});
|
||||
|
||||
export async function attemptAll(actions) {
|
||||
const failures = [];
|
||||
for (const [name, action] of actions) {
|
||||
try {
|
||||
await action();
|
||||
} catch (error) {
|
||||
failures.push({ action: name, ...describeError(error) });
|
||||
}
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
async function latest(root, report) {
|
||||
const temporary = join(root, `.verification-${report.run_id}.tmp`);
|
||||
await writeFile(temporary, JSON.stringify(report, null, 2), { flag: "wx" });
|
||||
await rename(temporary, join(root, "verification-latest.json"));
|
||||
}
|
||||
|
||||
async function withLatestLock(root, action) {
|
||||
// Keep this pathname stable: removing it could split concurrent lock owners.
|
||||
// SQLite releases its OS-backed lock when the connection/process closes.
|
||||
const database = new DatabaseSync(join(root, ".verification-lock.sqlite"));
|
||||
try {
|
||||
for (;;) {
|
||||
try {
|
||||
database.exec("BEGIN EXCLUSIVE");
|
||||
break;
|
||||
} catch (error) {
|
||||
if (
|
||||
error.code !== "ERR_SQLITE_ERROR" ||
|
||||
(error.errcode !== 5 && error.errcode !== 6)
|
||||
)
|
||||
throw error;
|
||||
// Yield while another process (or async operation here) owns the lock.
|
||||
// A synchronous busy timeout would block that operation's fs callbacks.
|
||||
await setTimeout(10);
|
||||
}
|
||||
}
|
||||
return await action();
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function prepareAttempt(root, inspectExisting, run) {
|
||||
if (!inspectExisting) await mkdir(root, { mode: 0o700 });
|
||||
const directory = join(root, "verifications", run.run_id);
|
||||
await mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
await withLatestLock(root, async () => {
|
||||
// Preserve older reports as history, without leaving an ambiguous success
|
||||
// at the capture root. Registration is the shared marker's ordering point.
|
||||
try {
|
||||
await rename(
|
||||
join(root, "verification.json"),
|
||||
join(directory, "legacy-verification.json"),
|
||||
);
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
await latest(root, {
|
||||
run_id: run.run_id,
|
||||
status: "running",
|
||||
started_at: new Date().toISOString(),
|
||||
directory,
|
||||
});
|
||||
});
|
||||
return { directory, captureRoot: join(root, "captures") };
|
||||
}
|
||||
|
||||
export async function publishAttempt(root, directory, report) {
|
||||
await writeFile(
|
||||
join(directory, "result.json"),
|
||||
JSON.stringify(report, null, 2),
|
||||
{ flag: "wx" },
|
||||
);
|
||||
if (report.status === "passed")
|
||||
await writeFile(
|
||||
join(directory, "verification.json"),
|
||||
JSON.stringify(report, null, 2),
|
||||
{ flag: "wx" },
|
||||
);
|
||||
await withLatestLock(root, async () => {
|
||||
const current = JSON.parse(
|
||||
await readFile(join(root, "verification-latest.json"), "utf8"),
|
||||
);
|
||||
if (current.run_id !== report.run_id) return;
|
||||
// Ownership check and replacement share one lock with attempt registration.
|
||||
// An older attempt still keeps its reports, but cannot replace this marker.
|
||||
await latest(root, {
|
||||
run_id: report.run_id,
|
||||
status: report.status,
|
||||
directory,
|
||||
finished_at: report.finished_at,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function finishProducerGroups(captureRoot, runId) {
|
||||
let leases;
|
||||
try {
|
||||
leases = (await readFile(join(captureRoot, "process-groups.jsonl"), "utf8"))
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map(JSON.parse);
|
||||
} catch (error) {
|
||||
if (error.code === "ENOENT") return { groups: [], observation: [] };
|
||||
throw error;
|
||||
}
|
||||
for (const lease of leases) assert.equal(lease.runId, runId);
|
||||
const failures = await attemptAll(
|
||||
leases.map((lease) => [
|
||||
`producer-group-${lease.processGroupId}`,
|
||||
async () => {
|
||||
const result = await cleanupOwnedProcessGroup(lease);
|
||||
assert.equal(result.cleaned, true, JSON.stringify(result));
|
||||
},
|
||||
]),
|
||||
);
|
||||
// Inspect even when one close failed; the groups can outlive their bootloader
|
||||
// parent and disappear from a plain descendant-only observation.
|
||||
const observation = [
|
||||
...(await observeOwnedProcessGroups(
|
||||
runId,
|
||||
leases.map((lease) => lease.processGroupId),
|
||||
)),
|
||||
];
|
||||
return { groups: leases, failures, observation };
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { spawn } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { once } from "node:events";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import {
|
||||
attemptAll,
|
||||
prepareAttempt,
|
||||
publishAttempt,
|
||||
finishProducerGroups,
|
||||
} from "./http3-verifier-lifecycle.mjs";
|
||||
import { cleanupOwnedProcessGroup } from "../../dist/process/ProcessOwnership.js";
|
||||
import { runOwnedCommand } from "../../dist/process/OwnedCommand.js";
|
||||
import { verifyConcurrency } from "./http3-capture-assertions.mjs";
|
||||
|
||||
async function waitForFile(path) {
|
||||
const signal = AbortSignal.timeout(5_000);
|
||||
for (;;) {
|
||||
try {
|
||||
await readFile(path);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
await setTimeout(10, undefined, { signal });
|
||||
}
|
||||
}
|
||||
|
||||
function startAttemptWorker(root, report) {
|
||||
const worker = `
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout } from "node:timers/promises";
|
||||
import { prepareAttempt, publishAttempt } from ${JSON.stringify(
|
||||
new URL("./http3-verifier-lifecycle.mjs", import.meta.url).href,
|
||||
)};
|
||||
const [root, serialized] = process.argv.slice(1);
|
||||
const report = JSON.parse(serialized);
|
||||
const attempt = await prepareAttempt(root, true, report);
|
||||
await writeFile(join(root, report.run_id + ".ready"), "");
|
||||
for (;;) {
|
||||
try { await readFile(join(root, report.run_id + ".finish")); break; }
|
||||
catch (error) { if (error.code !== "ENOENT") throw error; }
|
||||
await setTimeout(10);
|
||||
}
|
||||
await publishAttempt(root, attempt.directory, report);
|
||||
`;
|
||||
return runOwnedCommand(
|
||||
{
|
||||
command: process.execPath,
|
||||
arguments: [
|
||||
"--input-type=module",
|
||||
"--eval",
|
||||
worker,
|
||||
root,
|
||||
JSON.stringify(report),
|
||||
],
|
||||
cwd: root,
|
||||
hostEnvironment: process.env,
|
||||
runId: randomUUID(),
|
||||
},
|
||||
{ timeoutMs: 10_000, diagnosticBytes: 4096 },
|
||||
).then(
|
||||
(result) => ({ result }),
|
||||
(error) => ({ error }),
|
||||
);
|
||||
}
|
||||
|
||||
test("cleanup continues through failed closes and reaches its audit", async () => {
|
||||
const attempted = [];
|
||||
const failures = await attemptAll(
|
||||
["client", "transport", "audit"].map((name) => [
|
||||
name,
|
||||
async () => {
|
||||
attempted.push(name);
|
||||
if (name !== "audit") throw new Error(`${name} failed`);
|
||||
},
|
||||
]),
|
||||
);
|
||||
assert.deepEqual(attempted, ["client", "transport", "audit"]);
|
||||
assert.deepEqual(
|
||||
failures.map((failure) => failure.action),
|
||||
["client", "transport"],
|
||||
);
|
||||
});
|
||||
|
||||
test("a failed reinspection cannot leave the earlier success as the current result", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-results-"));
|
||||
try {
|
||||
await writeFile(
|
||||
join(root, "verification.json"),
|
||||
'{"status":"passed","run_id":"legacy"}',
|
||||
);
|
||||
const first = await prepareAttempt(root, true, { run_id: "first" });
|
||||
await publishAttempt(root, first.directory, {
|
||||
run_id: "first",
|
||||
status: "passed",
|
||||
});
|
||||
const next = await prepareAttempt(root, true, { run_id: "next" });
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(root, "verification-latest.json"))).status,
|
||||
"running",
|
||||
);
|
||||
await publishAttempt(root, next.directory, {
|
||||
run_id: "next",
|
||||
status: "failed",
|
||||
error: { message: "new failure" },
|
||||
});
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(root, "verification-latest.json"))).run_id,
|
||||
"next",
|
||||
);
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(root, "verification-latest.json"))).status,
|
||||
"failed",
|
||||
);
|
||||
await assert.rejects(readFile(join(next.directory, "verification.json")), {
|
||||
code: "ENOENT",
|
||||
});
|
||||
await assert.rejects(readFile(join(root, "verification.json")), {
|
||||
code: "ENOENT",
|
||||
});
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(first.directory, "verification.json")))
|
||||
.status,
|
||||
"passed",
|
||||
);
|
||||
assert.equal(
|
||||
JSON.parse(
|
||||
await readFile(join(first.directory, "legacy-verification.json")),
|
||||
).run_id,
|
||||
"legacy",
|
||||
);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("separate overlapping attempts finishing in reverse order retain the newer failure", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-overlap-"));
|
||||
const commands = [];
|
||||
const failed = {
|
||||
run_id: "newer",
|
||||
status: "failed",
|
||||
error: { message: "newer inspection failed" },
|
||||
cleanup_failures: [{ action: "transport-close", message: "close failed" }],
|
||||
};
|
||||
let primaryError;
|
||||
const start = (report) => {
|
||||
const command = startAttemptWorker(root, report);
|
||||
commands.push(command);
|
||||
return command;
|
||||
};
|
||||
try {
|
||||
const older = start({ run_id: "older", status: "passed" });
|
||||
await waitForFile(join(root, "older.ready"));
|
||||
const newer = start(failed);
|
||||
await waitForFile(join(root, "newer.ready"));
|
||||
await writeFile(join(root, "newer.finish"), "");
|
||||
const newerResult = await newer;
|
||||
if (newerResult.error) throw newerResult.error;
|
||||
assert.equal(newerResult.result.exitCode, 0);
|
||||
assert.equal(newerResult.result.signal, null);
|
||||
const marker = await readFile(join(root, "verification-latest.json"));
|
||||
await writeFile(join(root, "older.finish"), "");
|
||||
const olderResult = await older;
|
||||
if (olderResult.error) throw olderResult.error;
|
||||
assert.equal(olderResult.result.exitCode, 0);
|
||||
assert.equal(olderResult.result.signal, null);
|
||||
assert.deepEqual(
|
||||
await readFile(join(root, "verification-latest.json")),
|
||||
marker,
|
||||
);
|
||||
assert.deepEqual(
|
||||
JSON.parse(await readFile(join(root, "verifications/newer/result.json"))),
|
||||
failed,
|
||||
);
|
||||
await assert.rejects(
|
||||
readFile(join(root, "verifications/newer/verification.json")),
|
||||
{
|
||||
code: "ENOENT",
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
JSON.parse(
|
||||
await readFile(join(root, "verifications/older/verification.json")),
|
||||
).status,
|
||||
"passed",
|
||||
);
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
} finally {
|
||||
// Release both barriers even after an assertion/setup failure. Each command
|
||||
// retains native supervision through termination before the root is removed.
|
||||
const failures = await attemptAll(
|
||||
["older", "newer"].map((name) => [
|
||||
name,
|
||||
() => writeFile(join(root, name + ".finish"), ""),
|
||||
]),
|
||||
);
|
||||
const outcomes = await Promise.all(commands);
|
||||
const retained = outcomes.filter(
|
||||
({ error }) => error?.cleanupFailure || error?.cleanupResult,
|
||||
);
|
||||
if (retained.length)
|
||||
primaryError = new AggregateError(
|
||||
[primaryError, ...retained.map(({ error }) => error)].filter(Boolean),
|
||||
`Worker cleanup failed; retained test evidence at ${root}`,
|
||||
{ cause: primaryError },
|
||||
);
|
||||
else
|
||||
failures.push(
|
||||
...(await attemptAll([
|
||||
["root", () => rm(root, { recursive: true, force: true })],
|
||||
])),
|
||||
);
|
||||
if (failures.length)
|
||||
primaryError = new AggregateError(
|
||||
[primaryError, ...failures].filter(Boolean),
|
||||
"Test cleanup failed",
|
||||
{ cause: primaryError },
|
||||
);
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
});
|
||||
|
||||
test("publication waits for the lock before reading the current owner", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-lock-"));
|
||||
let database;
|
||||
let publication;
|
||||
let primaryError;
|
||||
try {
|
||||
const older = await prepareAttempt(root, true, { run_id: "older" });
|
||||
database = new DatabaseSync(join(root, ".verification-lock.sqlite"));
|
||||
database.exec("BEGIN EXCLUSIVE");
|
||||
publication = publishAttempt(root, older.directory, {
|
||||
run_id: "older",
|
||||
status: "passed",
|
||||
});
|
||||
await waitForFile(join(older.directory, "verification.json"));
|
||||
const marker = join(root, "verification-latest.json");
|
||||
assert.equal(JSON.parse(await readFile(marker)).status, "running");
|
||||
// This connection represents another marker writer. Its replacement must
|
||||
// be observed only after the blocked publisher acquires the same lock.
|
||||
const newer = { run_id: "newer", status: "failed" };
|
||||
await writeFile(marker, JSON.stringify(newer));
|
||||
database.close();
|
||||
database = undefined;
|
||||
await publication;
|
||||
assert.deepEqual(JSON.parse(await readFile(marker)), newer);
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
} finally {
|
||||
const failures = await attemptAll([
|
||||
["lock", () => database?.close()],
|
||||
[
|
||||
"publication",
|
||||
async () => {
|
||||
if (publication) await publication;
|
||||
},
|
||||
],
|
||||
["root", () => rm(root, { recursive: true, force: true })],
|
||||
]);
|
||||
if (failures.length)
|
||||
primaryError = new AggregateError(
|
||||
[primaryError, ...failures].filter(Boolean),
|
||||
"Test cleanup failed",
|
||||
{ cause: primaryError },
|
||||
);
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
});
|
||||
|
||||
test("a failed marker read releases the lock and preserves the attempt report", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-lock-failure-"));
|
||||
try {
|
||||
const older = await prepareAttempt(root, true, { run_id: "older" });
|
||||
await writeFile(join(root, "verification-latest.json"), "{");
|
||||
const report = {
|
||||
run_id: "older",
|
||||
status: "failed",
|
||||
error: { message: "inspection failed" },
|
||||
};
|
||||
await assert.rejects(
|
||||
publishAttempt(root, older.directory, report),
|
||||
SyntaxError,
|
||||
);
|
||||
assert.deepEqual(
|
||||
JSON.parse(await readFile(join(older.directory, "result.json"))),
|
||||
report,
|
||||
);
|
||||
const next = await prepareAttempt(root, true, { run_id: "next" });
|
||||
await publishAttempt(root, next.directory, {
|
||||
run_id: "next",
|
||||
status: "passed",
|
||||
});
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(join(root, "verification-latest.json"))).run_id,
|
||||
"next",
|
||||
);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("dispatch reverses origin completion; client and capture may arrive in other orders", () => {
|
||||
const names = ["parallel-0", "parallel-1", "parallel-2"];
|
||||
const header = (text) => ({ base64: Buffer.from(text).toString("base64") });
|
||||
const messages = Object.fromEntries(
|
||||
names.map((name) => [
|
||||
name,
|
||||
{
|
||||
headers: [[header("x-lab-case"), header(name)]],
|
||||
body: { base64: header(name).base64 },
|
||||
},
|
||||
]),
|
||||
);
|
||||
const event = (kind, index, connection, monotonic) => ({
|
||||
event: kind,
|
||||
case: names[index],
|
||||
stream_id: index * 4,
|
||||
connection,
|
||||
monotonic,
|
||||
...messages[names[index]],
|
||||
});
|
||||
const manifest = {
|
||||
requests: messages,
|
||||
responses: messages,
|
||||
events: [
|
||||
...[0, 1, 2].map((i) => event("request-sent", i, "client", i)),
|
||||
...[1, 2, 0].map((i) => event("request-received", i, "origin", 10)),
|
||||
...[0, 2, 1].map((i) => event("response-sent", i, "origin", 20)),
|
||||
...[2, 0, 1].map((i) => event("response-received", i, "client", 30)),
|
||||
...["client", "origin"].map((connection) => ({
|
||||
event: "handshake",
|
||||
connection,
|
||||
alpn: "h3",
|
||||
quic_version: 1,
|
||||
})),
|
||||
],
|
||||
};
|
||||
const record = (name) => ({
|
||||
reported: {
|
||||
id: name,
|
||||
request: { headers: [["x-lab-case", name]] },
|
||||
client_conn: { id: "client-uuid" },
|
||||
server_conn: { id: "origin-uuid" },
|
||||
},
|
||||
});
|
||||
const normal = {
|
||||
records: [...[1, 0, 2].map((i) => record(names[i])), record("curl")],
|
||||
};
|
||||
const orders = verifyConcurrency(normal, manifest);
|
||||
assert.deepEqual(orders.dispatched_at_origin, [
|
||||
"parallel-0",
|
||||
"parallel-2",
|
||||
"parallel-1",
|
||||
]);
|
||||
manifest.events = manifest.events.filter(
|
||||
(item) => item.event !== "response-sent",
|
||||
);
|
||||
manifest.events.push(
|
||||
...[2, 1, 0].map((i) => event("response-sent", i, "origin", 20)),
|
||||
);
|
||||
assert.throws(() => verifyConcurrency(normal, manifest), {
|
||||
code: "ERR_ASSERTION",
|
||||
});
|
||||
});
|
||||
|
||||
test("cleanup verifies and retires a separately owned producer group", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-http3-group-"));
|
||||
const runId = randomUUID();
|
||||
const child = spawn("/usr/bin/sleep", ["60"], {
|
||||
detached: true,
|
||||
env: { ...process.env, REA_PROCESS_RUN_ID: runId },
|
||||
stdio: "ignore",
|
||||
});
|
||||
let primaryError;
|
||||
let cleaned = false;
|
||||
try {
|
||||
await once(child, "spawn");
|
||||
await writeFile(
|
||||
join(root, "process-groups.jsonl"),
|
||||
JSON.stringify({
|
||||
runId,
|
||||
leaderPid: child.pid,
|
||||
processGroupId: child.pid,
|
||||
}) + "\n",
|
||||
);
|
||||
const result = await finishProducerGroups(root, runId);
|
||||
assert.deepEqual(result.failures, []);
|
||||
assert.equal(result.observation[0][1].state, "empty");
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
} finally {
|
||||
const failures = await attemptAll([
|
||||
[
|
||||
"process",
|
||||
async () => {
|
||||
const result = await cleanupOwnedProcessGroup({
|
||||
runId,
|
||||
leaderPid: child.pid,
|
||||
processGroupId: child.pid,
|
||||
});
|
||||
assert.equal(
|
||||
result.cleaned,
|
||||
true,
|
||||
`Retained ownership evidence at ${root}: ${JSON.stringify(result)}`,
|
||||
);
|
||||
cleaned = true;
|
||||
},
|
||||
],
|
||||
[
|
||||
"root",
|
||||
async () => {
|
||||
if (cleaned) await rm(root, { recursive: true, force: true });
|
||||
},
|
||||
],
|
||||
]);
|
||||
if (failures.length)
|
||||
primaryError = new AggregateError(
|
||||
[primaryError, ...failures].filter(Boolean),
|
||||
"Test cleanup failed",
|
||||
{ cause: primaryError },
|
||||
);
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
});
|
||||
Reference in New Issue
Block a user