fix(setup): reject invalid OMP profiles before fallback writes (#1562)
This commit is contained in:
@@ -0,0 +1,441 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { homedir, tmpdir } from "node:os";
|
||||
import { join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
assert(
|
||||
process.platform !== "win32",
|
||||
"Native OMP verifier requires POSIX; Windows is unverified",
|
||||
);
|
||||
const repo = fileURLToPath(new URL("..", import.meta.url));
|
||||
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
const command = process.env.REA_VERIFY_OMP_COMMAND ?? "omp";
|
||||
const mode = process.argv[2] ?? "call";
|
||||
assert(
|
||||
["chat", "call"].includes(mode),
|
||||
"Usage: verify-omp-client.mjs [chat|call]",
|
||||
);
|
||||
const verifier = createVerifierRun();
|
||||
const lab = await mkdtemp(join(tmpdir(), "rea-omp-client-"));
|
||||
const account = join(lab, "account");
|
||||
const agent = join(lab, "selected agent");
|
||||
const workspace = join(lab, "workspace");
|
||||
const configPath = join(agent, "mcp.json");
|
||||
const skillDirectory = join(
|
||||
account,
|
||||
".agents",
|
||||
"skills",
|
||||
"reverse-engineer-anything",
|
||||
);
|
||||
await Promise.all([
|
||||
mkdir(account),
|
||||
mkdir(agent),
|
||||
mkdir(workspace),
|
||||
...["data", "state", "cache"].map((name) =>
|
||||
mkdir(join(lab, name, "omp"), { recursive: true }),
|
||||
),
|
||||
]);
|
||||
await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
|
||||
// OMP's global config root uses native homedir independently of the agent override.
|
||||
// A documented relative PI_CONFIG_DIR redirects it without changing HOME.
|
||||
const environment = {
|
||||
PATH: process.env.PATH,
|
||||
NODE_OPTIONS: process.env.NODE_OPTIONS,
|
||||
USERPROFILE: account,
|
||||
PI_CODING_AGENT_DIR: agent,
|
||||
PI_CONFIG_DIR: relative(homedir(), join(lab, "native-config")),
|
||||
OMP_PROFILE: "",
|
||||
XDG_DATA_HOME: join(lab, "data"),
|
||||
XDG_STATE_HOME: join(lab, "state"),
|
||||
XDG_CACHE_HOME: join(lab, "cache"),
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
|
||||
);
|
||||
const expectedDevices = TOOL_CONTRACTS.map(
|
||||
(tool) => "mcp__rea_" + tool.name,
|
||||
).sort();
|
||||
const analysisDevice = "xd://mcp__rea_analyze_javascript_application";
|
||||
let clientVersion;
|
||||
let expectedSkillBody;
|
||||
let skillLoaded = false;
|
||||
let contractRead = false;
|
||||
let evidenceSeen = false;
|
||||
let artifactDigest;
|
||||
let finalSeen = false;
|
||||
let catalogMounted = false;
|
||||
let invalidProfileRejected = false;
|
||||
let failure;
|
||||
const containsText = (value, text) => {
|
||||
if (typeof value === "string") return value.includes(text);
|
||||
return (
|
||||
value !== null &&
|
||||
typeof value === "object" &&
|
||||
Object.values(value).some((item) => containsText(item, text))
|
||||
);
|
||||
};
|
||||
const call = (index, name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: `call_omp_${index}`,
|
||||
type: "function",
|
||||
function: { name, arguments: JSON.stringify(args) },
|
||||
},
|
||||
],
|
||||
});
|
||||
const fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: "omp",
|
||||
onRequest: async ({ body, tools, requestIndex }) => {
|
||||
assert(tools.some((tool) => tool.name === "read"));
|
||||
assert(tools.some((tool) => tool.name === "write"));
|
||||
const instructions = body.messages.filter(
|
||||
(message) => message.role === "system",
|
||||
);
|
||||
for (const device of expectedDevices)
|
||||
assert(
|
||||
containsText(instructions, device),
|
||||
`OMP must advertise mounted device ${device}`,
|
||||
);
|
||||
if (mode === "chat") {
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
}
|
||||
const replies = body.messages.filter((message) => message.role === "tool");
|
||||
if (requestIndex === 1)
|
||||
return call(requestIndex, "read", {
|
||||
i: "Load REA skill",
|
||||
path: "skill://reverse-engineer-anything:raw",
|
||||
});
|
||||
assert(
|
||||
expectedSkillBody !== undefined &&
|
||||
containsText(replies, expectedSkillBody),
|
||||
"OMP must return the complete installed skill body",
|
||||
);
|
||||
skillLoaded = true;
|
||||
if (requestIndex === 2)
|
||||
return call(requestIndex, "read", {
|
||||
i: "Inspect REA native tool contract",
|
||||
path: analysisDevice,
|
||||
});
|
||||
assert(containsText(replies, "# mcp__rea_analyze_javascript_application"));
|
||||
assert(
|
||||
containsText(replies, "input_path") &&
|
||||
containsText(replies, '"directory"'),
|
||||
);
|
||||
contractRead = true;
|
||||
if (requestIndex === 3)
|
||||
return call(requestIndex, "write", {
|
||||
path: analysisDevice,
|
||||
content: JSON.stringify({ input_path: target, format: "directory" }),
|
||||
});
|
||||
const evidenceReply = replies.find(
|
||||
(reply) =>
|
||||
typeof reply.content === "string" &&
|
||||
reply.content.startsWith('{"evidence_id":'),
|
||||
);
|
||||
assert(
|
||||
evidenceReply,
|
||||
"OMP must deliver the full analysis result to the next model turn",
|
||||
);
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(JSON.parse(evidenceReply.content));
|
||||
assert.equal(evidence.subject.local_path, target);
|
||||
const shape = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.find(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(shape?.return_shape_coverage?.projection_complete);
|
||||
const fields = shape.static_return_shapes.flatMap((item) => item.fields);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
);
|
||||
const bytes = JSON.stringify(evidence);
|
||||
artifactDigest = createHash("sha256").update(bytes).digest("hex");
|
||||
await writeFile(join(lab, "omp-evidence.json"), bytes);
|
||||
evidenceSeen = true;
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
},
|
||||
});
|
||||
const original = JSON.stringify(
|
||||
{
|
||||
mcpServers: { other: { command: "unrelated-server", enabled: false } },
|
||||
disabledServers: ["rea", "other"],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
);
|
||||
await writeFile(configPath, original);
|
||||
await writeFile(
|
||||
join(agent, "config.yml"),
|
||||
JSON.stringify({
|
||||
skills: { customDirectories: [join(account, ".agents", "skills")] },
|
||||
retry: { fallbackChains: { default: [] } },
|
||||
}),
|
||||
);
|
||||
await writeFile(
|
||||
join(agent, "models.yml"),
|
||||
JSON.stringify({
|
||||
providers: {
|
||||
"rea-fixture": {
|
||||
baseUrl: fixture.baseUrl,
|
||||
apiKey: "local-fixture-key",
|
||||
api: "openai-completions",
|
||||
models: [
|
||||
{
|
||||
id: "rea-client-fixture",
|
||||
name: "REA local fixture",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1000000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
const rea = async (args, env = environment) =>
|
||||
exec(
|
||||
process.execPath,
|
||||
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
|
||||
{ env, cwd: workspace, timeout: 60_000 },
|
||||
);
|
||||
try {
|
||||
clientVersion = (
|
||||
await exec(command, ["--version"], { env: environment, timeout: 30_000 })
|
||||
).stdout.trim();
|
||||
const invalidEnvironment = { ...environment, OMP_PROFILE: "Bad Name" };
|
||||
const nativeRejected = await exec(
|
||||
command,
|
||||
["--print", "--no-session", "--no-title", "Hello"],
|
||||
{ env: invalidEnvironment, cwd: workspace, timeout: 30_000 },
|
||||
).then(
|
||||
() => undefined,
|
||||
(cause) => cause,
|
||||
);
|
||||
assert(
|
||||
nativeRejected?.code === 1 &&
|
||||
nativeRejected.stderr.includes('Invalid OMP profile "Bad Name"'),
|
||||
);
|
||||
await writeFile(
|
||||
join(lab, "invalid-profile-native.stderr"),
|
||||
nativeRejected.stderr,
|
||||
);
|
||||
const setupRejected = await rea(
|
||||
["setup", "--client", "omp", "--dry-run"],
|
||||
invalidEnvironment,
|
||||
).then(
|
||||
() => undefined,
|
||||
(cause) => cause,
|
||||
);
|
||||
assert(typeof setupRejected?.code === "number" && setupRejected.code !== 0);
|
||||
const rejected = JSON.parse(setupRejected.stdout);
|
||||
assert.equal(rejected.status, "needs_human");
|
||||
assert.deepEqual(rejected.plannedActions, []);
|
||||
assert.deepEqual(rejected.appliedActions, []);
|
||||
assert(rejected.remediation.includes('Invalid OMP profile "Bad Name"'));
|
||||
assert.equal(await readFile(configPath, "utf8"), original);
|
||||
await writeFile(
|
||||
join(lab, "invalid-profile-setup.json"),
|
||||
setupRejected.stdout,
|
||||
);
|
||||
invalidProfileRejected = true;
|
||||
for (const args of [
|
||||
["setup", "--client", "omp", "--dry-run"],
|
||||
["setup", "--client", "omp", "--yes"],
|
||||
["doctor", "--client", "omp", "--skill"],
|
||||
]) {
|
||||
const result = await rea(args);
|
||||
await writeFile(
|
||||
join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
|
||||
result.stdout,
|
||||
);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
if (args[0] === "doctor") assert.equal(parsed.healthy, true);
|
||||
else {
|
||||
assert.equal(
|
||||
parsed.status,
|
||||
args.includes("--dry-run") ? "planned" : "ready",
|
||||
);
|
||||
if (args.includes("--dry-run")) {
|
||||
assert(
|
||||
parsed.plannedActions.some((action) => action.target === configPath),
|
||||
);
|
||||
assert(
|
||||
parsed.plannedActions.some(
|
||||
(action) => action.target === skillDirectory,
|
||||
),
|
||||
);
|
||||
assert(
|
||||
parsed.plannedActions.every(
|
||||
(action) =>
|
||||
action.target === configPath ||
|
||||
action.target === skillDirectory ||
|
||||
action.target.startsWith(skillDirectory + "/"),
|
||||
),
|
||||
"Refuse setup writes outside the owned config/skill",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
const configured = await readFile(configPath, "utf8");
|
||||
const registration = JSON.parse(configured);
|
||||
assert.equal(registration.mcpServers.other.command, "unrelated-server");
|
||||
assert.deepEqual(registration.disabledServers, ["other"]);
|
||||
const repeated = JSON.parse(
|
||||
(await rea(["setup", "--client", "omp", "--yes"])).stdout,
|
||||
);
|
||||
assert.deepEqual(repeated.appliedActions, []);
|
||||
assert.equal(await readFile(configPath, "utf8"), configured);
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
expectedSkillBody = await readFile(join(skillDirectory, "SKILL.md"), "utf8");
|
||||
const execution = exec(
|
||||
command,
|
||||
[
|
||||
"--mode",
|
||||
"json",
|
||||
"--print",
|
||||
"--no-session",
|
||||
"--no-title",
|
||||
"--no-lsp",
|
||||
"--no-pty",
|
||||
"--approval-mode",
|
||||
"yolo",
|
||||
"--model",
|
||||
"rea-fixture/rea-client-fixture",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use the REA skill to analyze ${target}.`,
|
||||
],
|
||||
{
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
},
|
||||
);
|
||||
execution.child.stdin.end();
|
||||
const response = await execution;
|
||||
await writeFile(join(lab, `${mode}.stdout`), response.stdout);
|
||||
await writeFile(join(lab, `${mode}.stderr`), response.stderr);
|
||||
assert.equal(fixture.failure, undefined);
|
||||
const events = response.stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
const mounted = events
|
||||
.filter(
|
||||
(event) =>
|
||||
event.type === "notice" && event.message?.startsWith("xd://: mounted "),
|
||||
)
|
||||
.flatMap((event) =>
|
||||
event.message.slice("xd://: mounted ".length).split(", "),
|
||||
)
|
||||
.filter((name) => name.startsWith("mcp__rea_"));
|
||||
assert.deepEqual(
|
||||
mounted.sort(),
|
||||
expectedDevices,
|
||||
"Native OMP must mount the complete REA catalog",
|
||||
);
|
||||
catalogMounted = true;
|
||||
assert(
|
||||
!events.some(
|
||||
(event) => event.type === "tool_execution_end" && event.isError,
|
||||
),
|
||||
);
|
||||
assert(
|
||||
events.some(
|
||||
(event) =>
|
||||
event.type === "message_end" &&
|
||||
event.message?.role === "assistant" &&
|
||||
event.message.stopReason === "stop" &&
|
||||
containsText(event.message.content, "REA_CLIENT_COMPATIBILITY_OK"),
|
||||
),
|
||||
"OMP must finish with the expected native assistant message",
|
||||
);
|
||||
assert(
|
||||
events.some((event) => event.type === "agent_end" && event.isTerminal),
|
||||
);
|
||||
assert(finalSeen);
|
||||
if (mode === "call") assert(skillLoaded && contractRead && evidenceSeen);
|
||||
} catch (cause) {
|
||||
failure = cause;
|
||||
if (typeof cause?.stdout === "string")
|
||||
await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
|
||||
if (typeof cause?.stderr === "string")
|
||||
await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
|
||||
} finally {
|
||||
await fixture.close();
|
||||
const lineage = await completeVerifierRun(verifier);
|
||||
const receipt = {
|
||||
client: "OMP",
|
||||
clientVersion,
|
||||
mode,
|
||||
passed: failure === undefined,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
},
|
||||
modelFixture:
|
||||
"loopback OpenAI completions/SSE, caller-declared one-million-token model and synthetic usage; no live model claim",
|
||||
discovery:
|
||||
"explicit isolated default agent/global config/XDG roots and custom skill directory; named profiles, default OS-home skill discovery and Windows unverified",
|
||||
interface:
|
||||
"default native xd:// device metadata with read/write dispatch; complete JSON schemas are not forwarded as model functions",
|
||||
catalogMounted,
|
||||
catalogSize: expectedDevices.length,
|
||||
invalidProfileRejected,
|
||||
skillLoaded,
|
||||
contractRead,
|
||||
evidenceSeen,
|
||||
artifactDigest,
|
||||
finalSeen,
|
||||
requests: fixture.requests,
|
||||
probes: fixture.probes,
|
||||
fixtureFailure: fixture.failure,
|
||||
verifier: lineage,
|
||||
};
|
||||
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
|
||||
console.log(JSON.stringify(receipt));
|
||||
}
|
||||
if (failure !== undefined) throw failure;
|
||||
Reference in New Issue
Block a user