Files
rea/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts
T
eb69763d27 fix(process): name the reason two captures cannot be compared (#1080)
* fix(process): name the reason two captures cannot be compared

compare-process-captures rejected captures whose comparison contracts
differ, but the reason never reached the caller. The CLI printed
"Process command could not complete... run `rea doctor`". MCP wrapped
the error in an AnalysisInputError with no issues ("Analysis input is
invalid. Check the arguments"). A stale capture under
max_capture_age_ms got the same treatment.

Throw typed input issues that name the differing contract fields, or
the stale side with its completion time and age, and let
runDerivedOperation pass AnalysisInputError through unchanged. Document
what the comparison contract covers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(process): report CLI trace-spec issues under the MCP input path

The CLI reported an invalid trace specification under operation
"compare-process-captures" with root-relative issue paths such as
["events"]. MCP reports the same input as compare_process_captures with
paths under trace_spec. Use the canonical operation name and prefix
trace_spec so both surfaces describe the same input identically.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(process): name the side and constraint for unusable capture Evidence

Adjacent compare_process_captures failures still lost their reason.
Over MCP, Evidence from another workflow, tampered Evidence and an
invalid capture result all became "Evidence is invalid or has changed".
The CLI said only "Capture evidence is malformed".

Parse each side separately. Evidence from another workflow is an input
issue on left or right that names the operation and predicate it has.
Tampered Evidence and invalid capture results stay integrity
mismatches, and their message now carries the validation detail.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 00:04:35 +08:00

211 lines
7.2 KiB
TypeScript

import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { expect, it, onTestFinished } from "vitest";
import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js";
import { PROCESS_PROVIDER } from "../../../src/application/process/ProcessEvidence.js";
import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/contracts/process/processCaptureExample.js";
import { createEvidence } from "../../../src/domain/evidence.js";
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
import { digestProcessCommitment } from "../../../src/domain/process/processCapture.js";
import { createServer } from "../../../src/server/createServer.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const captureEvidence = (
side: "left" | "right",
comparisonContract: Readonly<Record<string, string>> = {},
) =>
createEvidence(undefined, PROCESS_PROVIDER, {
predicateType: "rea.process-capture",
operation: "capture_process_scenario",
parameters: { side },
result: jsonValueSchema.parse({
...EMPTY_PROCESS_CAPTURE_EXAMPLE,
manifest: {
...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest,
comparison_contract: comparisonContract,
comparison_contract_sha256: digestProcessCommitment(comparisonContract),
},
}),
confidence: "observed",
authority: "controlled-replay",
});
const left = captureEvidence("left", { working_directory: "/work/reference" });
const right = captureEvidence("right", {
working_directory: "/work/reconstruction",
});
const connect = async () => {
const session = createTestBinarySession(() => {
throw new Error("Process comparison must not launch a provider");
});
const server = createServer(session, session);
const client = new Client({ name: "process-comparison-test", version: "1" });
onTestFinished(async () => {
await Promise.allSettled([client.close(), server.close(), session.close()]);
});
const [clientTransport, serverTransport] =
InMemoryTransport.createLinkedPair();
await server.connect(serverTransport);
await client.connect(clientTransport);
return client;
};
it("names differing comparison contract fields through MCP and the CLI", async () => {
const client = await connect();
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left, right },
});
expect(response.isError).toBe(true);
const issue = {
path: ["right"],
reason: "invalid_value",
message: expect.stringContaining(
"these scenario fields differ: working_directory",
),
expected: ["working_directory"],
};
expect(response.structuredContent).toMatchObject({
error: { code: "invalid_request", details: { issues: [issue] } },
});
const root = await createTestTempDirectory("rea-process-contract-");
const leftPath = join(root, "left.json");
const rightPath = join(root, "right.json");
await writeFile(leftPath, JSON.stringify(left));
await writeFile(rightPath, JSON.stringify(right));
expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({
error: "Process command failed",
code: "invalid_request",
details: { operation: "compare_process_captures", issues: [issue] },
});
});
it("names the stale capture that exceeds max_capture_age_ms", async () => {
const client = await connect();
const response = await client.callTool({
name: "compare_process_captures",
arguments: {
left: captureEvidence("left"),
right: captureEvidence("right"),
max_capture_age_ms: 1000,
},
});
expect(response.isError).toBe(true);
expect(response.structuredContent).toMatchObject({
error: {
code: "invalid_request",
details: {
issues: [
{
path: ["max_capture_age_ms"],
reason: "out_of_range",
message: expect.stringContaining(
`left completed at ${EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest.completed_at}`,
),
},
],
},
},
});
});
it("reports CLI trace specification issues like the MCP trace_spec input", async () => {
const root = await createTestTempDirectory("rea-process-trace-spec-");
const capturePath = join(root, "capture.json");
const specPath = join(root, "spec.json");
await writeFile(capturePath, JSON.stringify(captureEvidence("left")));
await writeFile(specPath, JSON.stringify({ language: {} }));
expect(
await compareProcessEvidenceFiles(capturePath, capturePath, specPath),
).toMatchObject({
code: "invalid_request",
details: {
operation: "compare_process_captures",
issues: expect.arrayContaining([
expect.objectContaining({ path: ["trace_spec", "events"] }),
]),
},
});
});
it("names the side and constraint when Evidence is not a usable capture", async () => {
const client = await connect();
const capture = captureEvidence("right");
const unrelated = createEvidence(undefined, PROCESS_PROVIDER, {
operation: "other",
parameters: {},
result: {},
});
const wrongKind = await client.callTool({
name: "compare_process_captures",
arguments: { left: unrelated, right: capture },
});
expect(wrongKind.structuredContent).toMatchObject({
error: {
code: "invalid_request",
details: {
issues: [
{
path: ["left"],
reason: "invalid_value",
message: expect.stringContaining(
"the left Evidence has operation other",
),
},
],
},
},
});
const tampered = await client.callTool({
name: "compare_process_captures",
arguments: {
left: capture,
right: { ...capture, parameters: { side: "changed" } },
},
});
expect(tampered.structuredContent).toMatchObject({
error: {
code: "evidence_integrity_mismatch",
message: expect.stringContaining(
"The right Evidence failed validation (Evidence semantic identifier does not match its record)",
),
},
});
const invalidResult = createEvidence(undefined, PROCESS_PROVIDER, {
predicateType: "rea.process-capture",
operation: "capture_process_scenario",
parameters: {},
result: { manifest: {} },
});
const malformed = await client.callTool({
name: "compare_process_captures",
arguments: { left: capture, right: invalidResult },
});
expect(malformed.structuredContent).toMatchObject({
error: {
code: "evidence_integrity_mismatch",
message: expect.stringContaining(
"The right Evidence has an invalid process capture result (at ",
),
},
});
const root = await createTestTempDirectory("rea-process-malformed-");
const leftPath = join(root, "left.json");
const rightPath = join(root, "right.json");
await writeFile(leftPath, JSON.stringify(capture));
await writeFile(rightPath, JSON.stringify(invalidResult));
expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({
category: "invalid_input",
message: expect.stringContaining("Capture evidence is malformed (at "),
});
});