test(evaluation): grade managed agent fixture facts (#1924)

* test(evaluation): grade managed agent fixture facts

* test(evaluation): cover distinct managed artifact deliveries

* fix(evaluation): require unfiltered managed type inventories

* fix(evaluation): preserve selected entry point grounding

---------

Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
This commit is contained in:
Huamnanx
2026-10-13 23:34:17 +08:00
committed by GitHub
co-authored by morluto
parent 7dcd23c005
commit ffd8c9c03d
5 changed files with 506 additions and 10 deletions
+9 -1
View File
@@ -1,4 +1,5 @@
import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
import {
chmod,
copyFile,
@@ -259,7 +260,7 @@ try {
const execution = await runCodex(
answerInstructions.length === 0
? scenario.prompt
: `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial application analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
: `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial inspection or analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
scenario.approvedTools ?? [],
);
const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;
@@ -440,6 +441,13 @@ async function createTargets(root, includeManaged) {
...fixtures,
largeJavascript,
managed: join(managedOutput, "AgentEval.dll"),
...(includeManaged
? {
managedSha256: createHash("sha256")
.update(await readFile(join(managedOutput, "AgentEval.dll")))
.digest("hex"),
}
: {}),
};
}