test: remove stale limits and handoff expectations

This commit is contained in:
morluto
2026-09-29 17:00:16 +08:00
parent ecf3ed8bdb
commit acf33d6df2
22 changed files with 206 additions and 156 deletions
+2 -1
View File
@@ -59,7 +59,8 @@ describe("guided prompt contracts", () => {
expect(rendered).toContain(
"optional starting points, not a required sequence",
);
expect(rendered).toContain("## Suggested steps");
expect(rendered).toContain("## Optional starting points");
expect(rendered).toContain("\n- ");
expect(rendered).toContain("Observations");
expect(rendered).toContain("Inference");
expect(rendered).toContain("Unknowns");
+67
View File
@@ -0,0 +1,67 @@
import { describe, expect, it } from "vitest";
import { buildSyntheticJavaScriptApplicationGraph } from "./javascriptApplicationGraph.fixture.js";
import type { JsonValue } from "./jsonValue.js";
import { findApplicationFeatureSeeds } from "./javascriptFeatureSeed.js";
describe("application feature seed property matching", () => {
it("finds strings after the former property count ceiling", () => {
const node = graphNodeWithProperties({
values: Array.from({ length: 513 }, (_, index) => `value-${index}`),
});
expect(
findApplicationFeatureSeeds([node], {
kind: "string",
value: "value-512",
match: "exact",
case_sensitive: true,
}),
).toEqual([
{
node_id: node.node_id,
kind: node.kind,
basis: "property",
field: "observations[0].properties.values[512]",
},
]);
});
it("finds strings deeper than the former traversal cutoff", () => {
const node = graphNodeWithProperties({
one: {
two: { three: { four: { five: { six: { seven: "deep-value" } } } } },
},
});
expect(
findApplicationFeatureSeeds([node], {
kind: "string",
value: "deep-value",
match: "exact",
case_sensitive: true,
}),
).toEqual([
{
node_id: node.node_id,
kind: node.kind,
basis: "property",
field: "observations[0].properties.one.two.three.four.five.six.seven",
},
]);
});
});
const graphNodeWithProperties = (
properties: Readonly<Record<string, JsonValue>>,
) => {
const template = buildSyntheticJavaScriptApplicationGraph().nodes[0];
if (template === undefined) throw new Error("Expected fixture graph node");
return {
...template,
observations: template.observations.map((observation) => ({
...observation,
properties,
})),
};
};
@@ -12,7 +12,7 @@ import { analyzeJavaScriptApplication } from "../../../src/application/JavaScrip
import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascriptApplicationAnalysis.js";
describe("application workflow MCP parity", () => {
it("compares exact parser export shapes through full Evidence and session IDs", async () => {
it("compares exact parser export shapes with inline Evidence", async () => {
const root = await createTestTempDirectory("rea-export-shape-mcp-");
const leftRoot = join(root, "left");
const rightRoot = join(root, "right");
@@ -80,22 +80,6 @@ describe("application workflow MCP parity", () => {
]),
);
const byId = await client.callTool({
name: "compare_javascript_export_shapes",
arguments: {
left: left.value.evidence_id,
right: right.value.evidence_id,
...selectors,
},
});
expect(byId.isError).not.toBe(true);
expect(byId.structuredContent).toMatchObject({
result: {
evidence_links: [left.value.evidence_id, right.value.evidence_id],
changes: [expect.objectContaining({ path: "/depth" })],
},
});
const analyzed = javascriptApplicationAnalysisResultSchema.parse(
left.value.normalized_result,
);
@@ -136,7 +136,7 @@ describe("JavaScript application workflows", () => {
});
describe("JavaScript application comparison workflows", () => {
it("accepts complete native evidence arrays and rejects comparison limit bags", () => {
it("accepts complete inline Evidence and native observation arrays", () => {
const native = Array.from({ length: 65 }, (_, index) =>
createEvidence(
{
@@ -153,32 +153,47 @@ describe("JavaScript application comparison workflows", () => {
},
),
);
const applicationEvidence = (side: string) =>
createEvidence(
undefined,
{ id: "fixture", name: "Fixture", version: "1" },
{
predicateType: "rea.javascript-application-analysis",
operation: "analyze_javascript_application",
parameters: { side },
result: {},
},
);
const input = {
left: `ev_${"a".repeat(64)}`,
right: `ev_${"b".repeat(64)}`,
left: applicationEvidence("left"),
right: applicationEvidence("right"),
left_native_observations: native,
};
expect(
compareApplicationVersionsRequestSchema.safeParse(input).success,
).toBe(true);
expect(
compareApplicationVersionsRequestSchema.safeParse({
...input,
limits: { max_comparison_items: 1 },
}).success,
).toBe(false);
expect(
compareApplicationVersionsRequestSchema.safeParse({
left_evidence_id: `ev_${"a".repeat(64)}`,
right_evidence_id: `ev_${"b".repeat(64)}`,
}).success,
).toBe(false);
});
it("uses one consistent Evidence reference shape for export comparisons", () => {
const request = {
left: `ev_${"a".repeat(64)}`,
right: `ev_${"b".repeat(64)}`,
left: createEvidence(
undefined,
{ id: "fixture", name: "Fixture", version: "1" },
{
operation: "analyze_javascript_application",
parameters: { side: "left" },
result: {},
},
),
right: createEvidence(
undefined,
{ id: "fixture", name: "Fixture", version: "1" },
{
operation: "analyze_javascript_application",
parameters: { side: "right" },
result: {},
},
),
left_module_path: "parser.mjs",
left_export_name: "default",
right_module_path: "parser.mjs",
@@ -187,13 +202,6 @@ describe("JavaScript application comparison workflows", () => {
expect(
compareJavaScriptExportShapesRequestSchema.safeParse(request).success,
).toBe(true);
expect(
compareJavaScriptExportShapesRequestSchema.safeParse({
...request,
left_evidence_id: request.left,
left: undefined,
}).success,
).toBe(false);
});
});
-1
View File
@@ -201,7 +201,6 @@ describe("Electron CLI runtime reconciliation", () => {
expect(missing).toMatchObject({
input_path: missingPath,
input_reason: "read-failed",
maximum_input_bytes: 64 * 1_024 * 1_024,
});
},
INTEGRATION_TEST_TIMEOUT_MS,
+2 -17
View File
@@ -6,22 +6,7 @@ import { describe, expect, it } from "vitest";
import { parseCliJsonInput } from "../../../src/cliJsonInput.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const MAX_BYTES = 64 * 1_024 * 1_024;
describe("bounded CLI JSON input", () => {
it("rejects oversized inline UTF-8 before returning or echoing its content", async () => {
const input = `"${"é".repeat(MAX_BYTES / 2)}"`;
expect(input.length).toBeLessThan(MAX_BYTES);
const result = await parseCliJsonInput(input, "test-input");
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.error).toMatchObject({
input_reason: "too-large",
maximum_input_bytes: MAX_BYTES,
});
expect(JSON.stringify(result.error).length).toBeLessThan(1_024);
});
describe("CLI JSON input", () => {
it("preserves JSON scalars and distinguishes malformed inline text from files", async () => {
for (const value of [null, false, 0, "text", [], {}])
expect(
@@ -39,7 +24,7 @@ describe("bounded CLI JSON input", () => {
});
expect(await parseCliJsonInput(root, "test-input")).toMatchObject({
ok: false,
error: { input_reason: "not-file" },
error: { input_reason: "read-failed" },
});
});
});
@@ -82,7 +82,6 @@ describe("Ghidra MCP evidence parity", () => {
normalized_result: expect.stringContaining("return 42"),
limitations: expect.arrayContaining([
expect.stringContaining("not text-equivalent to Hopper"),
expect.stringContaining("30-second native deadline"),
]),
});
+8 -4
View File
@@ -10,6 +10,7 @@ import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComp
import { PROCESS_CAPTURE_REFERENCE } from "../../../src/contracts/investigationExamples.js";
import { createEvidence } from "../../../src/domain/evidence.js";
import { createArtifactInspection } from "../../../src/domain/artifactInspection.js";
import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js";
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
import { createPromptCompletionSource } from "../../../src/server/promptCompletion.js";
import { observed } from "../../fixtures/analysisExecution.js";
@@ -76,6 +77,9 @@ describe("guided prompt completion from investigation records", () => {
const inventoryInspection = createArtifactInspection(
ARTIFACT_COMPARISON_EXAMPLE.left,
);
const inventory = artifactInventoryResultSchema.parse(
ARTIFACT_COMPARISON_EXAMPLE.left.normalized_result,
);
const inspectionEvidence = createEvidence(undefined, fixtureProvider, {
operation: "inspect_artifact",
parameters: {},
@@ -111,11 +115,11 @@ describe("guided prompt completion from investigation records", () => {
PROCESS_CAPTURE_REFERENCE.evidence_id,
]);
expect(await completion.complete("manifest", "agm_")).toEqual([
`agm_${"0".repeat(64)}`,
]);
expect(await completion.complete("occurrence", "occ_")).toEqual([
`occ_${"0".repeat(64)}`,
inventory.manifest.manifest_id,
]);
expect(await completion.complete("occurrence", "occ_")).toEqual(
inventory.occurrences.map(({ occurrence_id }) => occurrence_id),
);
expect(await completion.complete("unknown", "unk_")).toEqual([
unknown.value.unknown_id,
]);
+2 -1
View File
@@ -70,7 +70,8 @@ describe("guided prompts over MCP", () => {
if (content?.type !== "text") throw new Error("missing prompt text");
expect(content.text).toContain('"feature":"license validation"');
expect(content.text).toContain("`trace_feature`");
expect(content.text).toContain("incomplete pagination");
expect(content.text).toContain("Use Evidence returned inline");
expect(content.text).toContain("do not fetch a bundle or resource");
const audit = await client.getPrompt({
name: "audit_residual_unknowns",
arguments: { audit_scope: "all active release blockers" },
+2 -4
View File
@@ -239,11 +239,9 @@ describe("live server identity over MCP", () => {
const assertLiveIdentity = async (client: Client): Promise<void> => {
const instructions = client.getInstructions();
expect(instructions).toContain(
"For standalone JavaScript or ASAR analysis, use analyze_javascript_application",
);
expect(instructions).toContain(
"For an archive or application package, open_binary(path) binds the active target; use inspect_artifact when its graph and findings help",
"Use the tool that directly answers the question",
);
expect(instructions).toContain("Tool results include inline Evidence");
const identity = await client.callTool({
name: "binary_session",
arguments: {},
+7 -6
View File
@@ -188,10 +188,11 @@ it("records approved capture disagreement as a contradicted unknown", async () =
const mismatched = await client.callTool({
name: "compare_process_captures",
arguments: {
left_evidence_id: leftEvidence.evidence_id,
left: right,
right_evidence_id: rightEvidence.evidence_id,
right,
left: {
...leftEvidence,
normalized_result: rightEvidence.normalized_result,
},
right: rightEvidence,
},
});
expect(mismatched.isError).toBe(true);
@@ -199,8 +200,8 @@ it("records approved capture disagreement as a contradicted unknown", async () =
const compared = await client.callTool({
name: "compare_process_captures",
arguments: {
left_evidence_id: leftEvidence.evidence_id,
right_evidence_id: rightEvidence.evidence_id,
left: leftEvidence,
right: rightEvidence,
},
});
expect(compared.isError).not.toBe(true);
+5 -5
View File
@@ -49,26 +49,26 @@ it("advertises the native function discovery-to-dossier workflow", async () => {
});
expect(client.getInstructions()).toContain(
"use search_procedures or list_procedures to choose a function",
"Search or list symbols when discovery is needed",
);
expect(client.getInstructions()).toContain(
"Call binary_overview when target metadata or inventory counts are useful",
"analyze_function provides a function dossier",
);
const tools = new Map(
(await client.listTools()).tools.map((tool) => [tool.name, tool]),
);
expect(tools.get("binary_overview")?.description).toContain(
"Use search_procedures, list_procedures, or analyze_function directly when you already know which procedure to inspect",
"Return native-binary metadata",
);
expect(tools.get("analyze_function")?.description).toContain(
"typically selected from search_procedures or list_procedures",
"Build a dossier for one native function",
);
expect(tools.get("analyze_function")?.inputSchema.properties).toHaveProperty(
"procedure",
);
expect(tools.get("inspect_native_api")?.description).toContain(
"use analyze_function for a general function dossier",
"Analyze a native API boundary in one function",
);
});
+1 -2
View File
@@ -116,12 +116,11 @@ describe("target-free MCP workflow", () => {
const beforeTools = (await mcp.listTools()).tools;
const beforeNames = beforeTools.map(({ name }) => name);
expect(mcp.getInstructions()).toContain(
"For an archive or application package, open_binary(path) binds the active target; use inspect_artifact when its graph and findings help",
"Use the tool that directly answers the question",
);
expect(
beforeTools.find(({ name }) => name === "inspect_artifact")?.description,
).toContain("Returns the complete content-addressed artifact graph");
expect(beforeNames).not.toContain("inventory_artifact");
expect(beforeNames).toContain("open_binary");
expect(beforeNames).toContain("binary_session");
expect(beforeNames).toContain("current_document");
-2
View File
@@ -65,7 +65,6 @@ describe("tool result projection", () => {
strategy: "user-mount-namespace",
fallback_reason: null,
xvfb_stderr_bytes: 512,
xvfb_stderr_truncated: false,
}),
),
contract,
@@ -79,7 +78,6 @@ describe("tool result projection", () => {
mount_read_only: true,
wsl: true,
strategy: "user-mount-namespace",
xvfb_stderr_truncated: false,
},
},
},
@@ -227,7 +227,6 @@ const electronRuntimeEvidence = (
const sourceIncluded = options.sourceIncluded ?? true;
const input = inspectElectronPageInputSchema.parse({
cdp_endpoint: "http://127.0.0.1:9223",
allowed_file_roots: [root],
target_id: targetId,
observation_ms: 100,
include_script_sources: sourceIncluded,
+11 -7
View File
@@ -159,13 +159,13 @@ describe("provider process runtime and wait primitives", () => {
});
describe("provider process output and cleanup primitives", () => {
it("captures exact byte totals while bounding retained stdout and stderr", async () => {
const child = spawnProviderProcessFixture("burst", 4_096);
it("retains complete stdout and stderr beyond the former fixed ceiling", async () => {
const outputBytes = 70_000;
const child = spawnProviderProcessFixture("burst", outputBytes);
const diagnostics: ProviderProcessDiagnostic[] = [];
const supervisor = new ProviderProcessSupervisor(
{ process: child, ownsProcessLifetime: true },
{
maxOutputBytesPerStream: 64,
onDiagnostic: (event) => diagnostics.push(event),
},
);
@@ -173,13 +173,17 @@ describe("provider process output and cleanup primitives", () => {
await expect(supervisor.waitForExit(2_000)).resolves.toBe(true);
const snapshot = supervisor.snapshot();
expect(snapshot).toMatchObject({
stdout: { bytes: 4_096, retainedBytes: 64, truncated: true },
stderr: { bytes: 4_096, retainedBytes: 64, truncated: true },
stdout: {
bytes: outputBytes,
},
stderr: {
bytes: outputBytes,
},
exitCode: 23,
signal: null,
});
expect(snapshot.stdout.text).toBe("o".repeat(64));
expect(snapshot.stderr.text).toBe("e".repeat(64));
expect(snapshot.stdout.text).toBe("o".repeat(outputBytes));
expect(snapshot.stderr.text).toBe("e".repeat(outputBytes));
expect(diagnostics).toContainEqual(
expect.objectContaining({ type: "exit", code: 23 }),
);
@@ -43,30 +43,6 @@ describe("runtime executable diagnostics", () => {
expect(pathNodes.every(({ healthy }) => healthy)).toBe(true);
});
it("reports every PATH candidate beyond the former per-tool cap", async () => {
const directories = await Promise.all(
Array.from({ length: 17 }, () => temporaryRoot()),
);
await Promise.all(
directories.map((directory) =>
symlink(process.execPath, join(directory, "node")),
),
);
const inventory = await inspectRuntimeExecutables({
platform: process.platform,
path: directories.join(delimiter),
launcherNode: process.execPath,
});
expect(
inventory.candidates.filter(
({ tool, path_index: pathIndex }) =>
tool === "node" && pathIndex !== null,
),
).toHaveLength(17);
});
it("distinguishes a healthy primary runtime from a broken shadowed candidate", async () => {
const healthy = await temporaryRoot();
const broken = await temporaryRoot();
@@ -169,6 +145,51 @@ describe("runtime executable diagnostics", () => {
});
});
describe("complete runtime diagnostics", () => {
it("reports every PATH candidate", async () => {
const directories = await Promise.all(
Array.from({ length: 17 }, () => temporaryRoot()),
);
await Promise.all(
directories.map((directory) =>
symlink(process.execPath, join(directory, "node")),
),
);
const inventory = await inspectRuntimeExecutables({
platform: process.platform,
path: directories.join(delimiter),
launcherNode: process.execPath,
});
expect(
inventory.candidates.filter(
({ tool, path_index: pathIndex }) =>
tool === "node" && pathIndex !== null,
),
).toHaveLength(17);
});
it("preserves complete diagnostics", async () => {
const root = await temporaryRoot();
await executable(
join(root, "node"),
"#!/bin/sh\nprintf '%070000d' 0 >&2\nexit 17\n",
);
const inventory = await inspectRuntimeExecutables({
platform: process.platform,
path: root,
launcherNode: process.execPath,
});
const candidate = inventory.candidates.find(
({ tool, path_index: pathIndex }) => tool === "node" && pathIndex === 0,
);
expect(candidate?.failure?.stderr).toHaveLength(70_000);
});
});
const temporaryRoot = async (): Promise<string> => {
const root = await createTestTempDirectory("rea-runtime-diagnostics-");
roots.push(root);
@@ -115,7 +115,6 @@ class DiagnosticExitingLauncher implements BridgeLauncher {
strategy: "direct",
fallback_reason: null,
xvfb_stderr_bytes: 100,
xvfb_stderr_truncated: false,
};
const line = `${LINUX_PRIVATE_DISPLAY_DIAGNOSTIC_PREFIX}${JSON.stringify(diagnostic)}\n`;
return Promise.resolve(
@@ -245,7 +244,7 @@ describe("HopperClient startup failures", () => {
});
});
it("preserves the bounded private-display diagnostic from an adapter exit", async () => {
it("preserves the private-display diagnostic from an adapter exit", async () => {
const client = new HopperClient({
launcher: new DiagnosticExitingLauncher(),
startupTimeoutMs: 10_000,
@@ -51,7 +51,6 @@ const diagnostic = (overrides: DiagnosticOverrides = {}) => {
strategy: "direct",
fallback_reason: null,
xvfb_stderr_bytes: 0,
xvfb_stderr_truncated: false,
} as const;
return overrides.status === "error"
? ({ ...common, ...overrides } satisfies HopperStartupDiagnostic)
@@ -71,7 +70,6 @@ const processResult = (
exitCode: value.status === "ready" ? 0 : failureExit(value.failure_code),
stderr: `${LINUX_PRIVATE_DISPLAY_DIAGNOSTIC_PREFIX}${JSON.stringify(value)}\n`,
stderrBytes: 0,
stderrTruncated: false,
cleanupIncomplete: false,
...overrides,
});
@@ -166,37 +164,20 @@ describe("Linux private display selection", () => {
},
);
it.each([
[
"malformed",
`${LINUX_PRIVATE_DISPLAY_DIAGNOSTIC_PREFIX}not-json`,
false,
"diagnostic_malformed",
],
[
"oversized",
`${LINUX_PRIVATE_DISPLAY_DIAGNOSTIC_PREFIX}${"x".repeat(70_000)}`,
true,
"diagnostic_truncated",
],
] as const)(
"rejects %s helper diagnostics",
async (_name, stderr, stderrTruncated, reason) => {
const runProbe: LinuxPrivateDisplayProbeRunner = () =>
Promise.resolve({
...processResult(diagnostic()),
exitCode: 70,
stderr,
stderrTruncated,
});
await expect(
selectLinuxPrivateDisplayStrategy({ helperPath, runProbe }),
).resolves.toMatchObject({
ok: false,
diagnostic: { reason },
it("rejects malformed helper diagnostics", async () => {
const runProbe: LinuxPrivateDisplayProbeRunner = () =>
Promise.resolve({
...processResult(diagnostic()),
exitCode: 70,
stderr: `${LINUX_PRIVATE_DISPLAY_DIAGNOSTIC_PREFIX}not-json`,
});
},
);
await expect(
selectLinuxPrivateDisplayStrategy({ helperPath, runProbe }),
).resolves.toMatchObject({
ok: false,
diagnostic: { reason: "diagnostic_malformed" },
});
});
it("reports missing helper dependencies without exposing raw stderr", () => {
for (const [option] of [
@@ -209,7 +190,7 @@ describe("Linux private display selection", () => {
{ encoding: "utf8" },
);
expect(result.status).toBe(79);
const parsed = parseLinuxPrivateDisplayDiagnostic(result.stderr, false);
const parsed = parseLinuxPrivateDisplayDiagnostic(result.stderr);
expect(parsed).toMatchObject({
ok: true,
value: {
@@ -11,12 +11,13 @@ import {
describe("native DMG artifact reader", () => {
it("uses plist attachment metadata and detaches returned devices", async () => {
if (process.platform !== "darwin") return;
const calls: string[][] = [];
const commandOptions: unknown[] = [];
const host: NativeDmgHost = {
async run(arguments_) {
async run(arguments_, _signal, options) {
const args = [...arguments_];
calls.push(args);
commandOptions.push(options);
if (args[0] !== "attach") return { stdout: "", exitCode: 0 };
const mountRoot = args[args.indexOf("-mountroot") + 1];
if (mountRoot === undefined) throw new Error("missing mount root");
@@ -48,10 +49,12 @@ describe("native DMG artifact reader", () => {
await reader.close();
expect(calls).toContainEqual(["verify", "/tmp/image.dmg"]);
expect(calls).toContainEqual(["detach", "/dev/disk-fixture"]);
expect(commandOptions[0]).toBeUndefined();
expect(commandOptions[1]).toBeUndefined();
expect(commandOptions[2]).toEqual({ timeoutMs: 120_000 });
});
it("rejects non-zero results and surfaces detach failure during attach cleanup", async () => {
if (process.platform !== "darwin") return;
await expect(
NativeDmgArtifactReader.create("/tmp/image.dmg", undefined, {
run: () => Promise.resolve({ stdout: "", exitCode: 1 }),
@@ -350,7 +350,6 @@ const electronRuntimeEvidence = (
const sourceIncluded = options.sourceIncluded ?? true;
const input = inspectElectronPageInputSchema.parse({
cdp_endpoint: "http://127.0.0.1:9223",
allowed_file_roots: [root],
target_id: targetId,
observation_ms: 100,
include_script_sources: sourceIncluded,