From ba098f24e59efbd567ae8438f65e31af0f0b8050 Mon Sep 17 00:00:00 2001
From: morluto <76467478+morluto@users.noreply.github.com>
Date: Wed, 29 Jul 2026 17:36:51 +0000
Subject: [PATCH] test(eval): cover navigation and address tool routing
---
scripts/verify-agent-experience.mjs | 89 ++++++++++++++++++-----------
1 file changed, 55 insertions(+), 34 deletions(-)
diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs
index e643f5fdb..378d4031f 100644
--- a/scripts/verify-agent-experience.mjs
+++ b/scripts/verify-agent-experience.mjs
@@ -34,7 +34,16 @@ try {
skillDestination,
{ recursive: true },
);
- const targets = await createTargets(fixtureRoot);
+ const requestedScenarioIds = new Set(
+ (process.env.REA_AGENT_EVAL_SCENARIOS ?? "")
+ .split(",")
+ .map((value) => value.trim())
+ .filter((value) => value.length > 0),
+ );
+ const targets = await createTargets(
+ fixtureRoot,
+ requestedScenarioIds.size === 0 || requestedScenarioIds.has("managed"),
+ );
const allScenarios = [
{
id: "native",
@@ -94,13 +103,23 @@ try {
prompt:
"A user-owned page is already open at http://127.0.0.1:3000 through the approved local debugging endpoint http://127.0.0.1:9222. Inspect what is available without navigating or executing page code, and report any authority or availability limits.",
},
+ {
+ id: "navigation-context",
+ expectedFirstTool: "get_navigation_context",
+ requiresEvidence: false,
+ requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
+ prompt:
+ "Report the selected document, current address, and containing procedure from the current native-analysis session in one coherent request. If no native session is available, report that limitation plainly.",
+ },
+ {
+ id: "address-context",
+ expectedFirstTool: "inspect_address_context",
+ requiresEvidence: false,
+ requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
+ prompt:
+ "Inspect explicit address 0x401000 in the current native-analysis session. Report its analyzed name, containing procedure, regular comment, inline comment, and matching bookmarks. If no native session is available, report that limitation plainly.",
+ },
];
- const requestedScenarioIds = new Set(
- (process.env.REA_AGENT_EVAL_SCENARIOS ?? "")
- .split(",")
- .map((value) => value.trim())
- .filter((value) => value.length > 0),
- );
const scenarios =
requestedScenarioIds.size === 0
? allScenarios
@@ -234,7 +253,7 @@ try {
await rm(evaluationRoot, { recursive: true, force: true });
}
-async function createTargets(root) {
+async function createTargets(root, includeManaged) {
const javascript = join(root, "desktop-app");
await mkdir(join(javascript, "renderer"), { recursive: true });
await Promise.all([
@@ -275,33 +294,35 @@ async function createTargets(root) {
),
]);
- const managedProject = join(root, "managed-project");
const managedOutput = join(root, "managed-output");
- await mkdir(managedProject, { recursive: true });
- await Promise.all([
- writeFile(
- join(managedProject, "AgentEval.csproj"),
- 'Exenet8.0enableenable\n',
- ),
- writeFile(
- join(managedProject, "Program.cs"),
- 'namespace AgentEval; public sealed record Profile(string Id); public static class Program { public static void Main() => Console.WriteLine(new Profile("fixture")); }\n',
- ),
- ]);
- await runProcess(
- "dotnet",
- [
- "build",
- join(managedProject, "AgentEval.csproj"),
- "--configuration",
- "Release",
- "--output",
- managedOutput,
- "--nologo",
- ],
- root,
- 120_000,
- );
+ if (includeManaged) {
+ const managedProject = join(root, "managed-project");
+ await mkdir(managedProject, { recursive: true });
+ await Promise.all([
+ writeFile(
+ join(managedProject, "AgentEval.csproj"),
+ 'Exenet8.0enableenable\n',
+ ),
+ writeFile(
+ join(managedProject, "Program.cs"),
+ 'namespace AgentEval; public sealed record Profile(string Id); public static class Program { public static void Main() => Console.WriteLine(new Profile("fixture")); }\n',
+ ),
+ ]);
+ await runProcess(
+ "dotnet",
+ [
+ "build",
+ join(managedProject, "AgentEval.csproj"),
+ "--configuration",
+ "Release",
+ "--output",
+ managedOutput,
+ "--nologo",
+ ],
+ root,
+ 120_000,
+ );
+ }
return {
native: "/bin/true",
javascript: javascriptAsar,