chore: integrate upstream verification fixture owners

Integrate repaired Ghidra installation fixtures and the shared CI report invocation owner. Remove the superseded local report-environment override. Preserve Windows caller-evidence content checks through retry/teardown and add the upstream distinct-object identity assertion.

All ten PowerPC admission fixture tests and the original blob-report case pass with GITHUB_STEP_SUMMARY set; seven unrelated cases unselected. No real-Ghidra result is claimed from the fixture host. Original experiment files and measured runtime/data remain unchanged.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>

experiments/analysis-efficiency/README.md

@@ -19,6 +19,10 @@ Follow-up: [SQLite existing-control measurements](sqlite-existing-controls.md)
 tests QGIS's shipped GeoPackage through the existing column selection, preserving
 full selected-table schema and typed cells before considering another API.
 
+Follow-up: [small real PCAP recordings](pcap-small-captures.md) compares existing
+complete delivery with summary/pages. Preserve the negative SDK/model input
+results: compact captures do not justify another production mechanism.
+
 Follow-up: [native PE/DOS entry views and real model verification](native-entry-views.md)
 records the post-merge native continuation, including a small-function case where
 four selected replies cost more bytes than the complete dossier.

experiments/analysis-efficiency/agent-trial.py

@@ -180,7 +180,7 @@ def main():
                  'inspect_android_package', 'search_android_classes',
                  'inspect_android_class', 'inspect_android_method',
                  'inspect_firmware_regions', 'inspect_managed_artifact',
-                 'inspect_sqlite_database',
+                 'inspect_sqlite_database', 'inspect_pcap_capture',
                  'inspect_managed_members', 'inspect_managed_native_boundaries',
                  'inspect_artifact', 'inspect_wasm_artifact',
                  'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',

experiments/analysis-efficiency/compare.mjs

@@ -70,6 +70,44 @@ for (const [side, answer] of [
       value: answer.evidence_ids,
     });
 }
+// PCAP delivery observations preserve each actual receipt's identity, limitations
+// and view coverage. Summary/page scopes differ from complete source scope;
+// retain them separately rather than equating page exhaustion with capture
+// coverage. Repeated equivalent deliveries must also match every observation
+// field except the receipt and parent IDs, which identify their own invocations.
+if (
+  ["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes(
+    left.artifact,
+  )
+) {
+  for (const [side, answer] of [
+    ["left", a],
+    ["right", b],
+  ]) {
+    if (
+      !Array.isArray(answer.delivery_observations) ||
+      !answer.delivery_observations.length
+    )
+      throw new Error("Missing original PCAP delivery observations");
+    runSpecific.push({
+      side,
+      path: ["delivery_observations"],
+      value: answer.delivery_observations,
+    });
+    if (left.delivery === right.delivery) {
+      answer.delivery_observations = answer.delivery_observations.map(
+        (observation) =>
+          Object.fromEntries(
+            Object.entries(observation).filter(
+              ([key]) => key !== "evidence_id" && key !== "parent_evidence_id",
+            ),
+          ),
+      );
+    } else {
+      delete answer.delivery_observations;
+    }
+  }
+}
 // Binwalk assigns a new opaque region UUID in each invocation. Preserve both
 // original lookup values in this report; compare the observed region facts
 // separately. The product and original replies retain those IDs unchanged.

experiments/analysis-efficiency/corpus.json

@@ -263,6 +263,34 @@
         "url": "https://raw.githubusercontent.com/qgis/QGIS/e1223b8e9e71fc50ee995ed556f711c54ccc101b/resources/data/world_map.gpkg",
         "derivation": "unmodified official resource file; streaming Git blob and SHA-256 verification"
       }
+    },
+    {
+      "id": "wireshark-http.cap",
+      "file": "http.cap",
+      "bytes": 25803,
+      "sha256": "25a72bdf10339f2c29916920c8b9501d294923108de8f29b19aba7cc001ab60d",
+      "provenance": {
+        "project": "Wireshark public sample captures",
+        "source_page": "https://wiki.wireshark.org/SampleCaptures",
+        "url": "https://wiki.wireshark.org/uploads/27707187aeb30df68e70c8fb9d614981/http.cap",
+        "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/27707187aeb30df68e70c8fb9d614981/http.cap",
+        "upstream_expected_digest": null,
+        "derivation": "Unmodified capture published by official Wireshark wiki. Local digest pins downloaded bytes; no independently published upstream digest is claimed."
+      }
+    },
+    {
+      "id": "wireshark-http_redirects.pcapng",
+      "file": "http_redirects.pcapng",
+      "bytes": 47804,
+      "sha256": "ff432a6b71d06cd20720f7763e584014edd0a43c3b55981c682d1b7640207abc",
+      "provenance": {
+        "project": "Wireshark public sample captures",
+        "source_page": "https://wiki.wireshark.org/SampleCaptures",
+        "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/http_redirects.pcapng",
+        "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/http_redirects.pcapng",
+        "upstream_expected_digest": null,
+        "derivation": "Unmodified capture published by official Wireshark wiki. Local digest pins downloaded bytes; no independently published upstream digest is claimed."
+      }
     }
   ]
 }

experiments/analysis-efficiency/measure.mjs

@@ -523,7 +523,207 @@ const inspectEvmTarget = async () => {
   };
 };
 
+const PCAP_EVIDENCE_ID = /^ev_[a-f0-9]{64}$/u;
+const pcapObservation = (evidence, requestedView, parseEvidence) => {
+  // Authenticate with the selected runtime's owner, preserving the original
+  // envelope for measurement and provenance rather than reconstructing it.
+  parseEvidence(evidence);
+  const evidenceId = evidence.evidence_id;
+  const value = normalized(evidence);
+  for (const key of ["limitations", "unknowns"])
+    if (
+      Object.hasOwn(value, key) &&
+      (!Array.isArray(value[key]) ||
+        !value[key].every((item) => typeof item === "string"))
+    )
+      throw new Error("PCAP normalized provenance lacks a valid string list");
+  const target =
+    value.kind === "summary"
+      ? value.summary?.target
+      : value.kind === "pcap"
+        ? value.artifact
+        : value.target;
+  if (
+    evidence.subject?.digest?.sha256 !== inputSha ||
+    evidence.subject?.local_path !== values.path ||
+    target?.sha256 !== inputSha ||
+    target?.path !== values.path ||
+    (value.kind === "summary" &&
+      (value.artifact?.sha256 !== inputSha ||
+        value.artifact?.path !== values.path))
+  )
+    throw new Error("PCAP Evidence does not identify the selected capture");
+  // Summary delivery delegates to a retained view, whose Evidence operation is
+  // inspect_analysis_view even when inspect_pcap_capture was requested.
+  const operation = requestedView
+    ? "inspect_analysis_view"
+    : "inspect_pcap_capture";
+  if (evidence.operation !== operation)
+    throw new Error(
+      "PCAP Evidence operation differs from the requested delivery",
+    );
+  if (
+    requestedView &&
+    (typeof value.parent_evidence_id !== "string" ||
+      !PCAP_EVIDENCE_ID.test(value.parent_evidence_id) ||
+      value.parent_evidence_id === evidenceId ||
+      !Array.isArray(evidence.evidence_links) ||
+      !evidence.evidence_links.includes(value.parent_evidence_id))
+  )
+    throw new Error("PCAP retained parent lacks a distinct Evidence link");
+  if (
+    requestedView &&
+    value.parent_digest !== value.parent_evidence_id.slice(3)
+  )
+    throw new Error("PCAP retained parent digest differs from its identity");
+  const parameters = requestedView
+    ? { parent_evidence_id: value.parent_evidence_id, view: requestedView }
+    : { path: values.path };
+  if (
+    !isDeepStrictEqual(evidence.parameters, parameters) ||
+    (requestedView && !isDeepStrictEqual(value.view, requestedView))
+  )
+    throw new Error("PCAP Evidence parameters or view differ from the request");
+  return {
+    evidence_id: evidenceId,
+    limitations: evidence.limitations,
+    ...Object.fromEntries(
+      [
+        "kind",
+        "view",
+        "parent_evidence_id",
+        "coverage",
+        "limitations",
+        "unknowns",
+      ]
+        .filter((key) => Object.hasOwn(value, key) && key !== "limitations")
+        .map((key) => [key, value[key]]),
+    ),
+    ...(Object.hasOwn(value, "limitations")
+      ? { normalized_limitations: value.limitations }
+      : {}),
+  };
+};
+const inspectPcapTarget = async () => {
+  const { parseEvidence } = await import(
+    pathToFileURL(resolve(values["runtime-root"], "dist/domain/evidence.js"))
+      .href
+  );
+  const input = {
+    path: values.path,
+    ...(values.delivery === "summary" ? { detail: "summary" } : {}),
+  };
+  const definition = catalogDefinitions.get("inspect_pcap_capture");
+  const ajv = new Ajv2020({ strict: false, validateFormats: false });
+  if (
+    !definition ||
+    !ajv.validateSchema(definition.inputSchema) ||
+    !ajv.validate(definition.inputSchema, input)
+  )
+    throw new Error("Selected PCAP request is not advertised by this runtime");
+  const inspected = await call("inspect_pcap_capture", input);
+  const n = normalized(inspected);
+  const observations = [
+    pcapObservation(
+      inspected,
+      values.delivery === "summary" ? { kind: "summary" } : undefined,
+      parseEvidence,
+    ),
+  ];
+  let capture, uris;
+  if (values.delivery === "summary") {
+    if (n.kind !== "summary" || n.parent_operation !== "inspect_pcap_capture")
+      throw new Error("Runtime returned no requested PCAP summary");
+    if (
+      !isDeepStrictEqual(n.coverage, {
+        status: "complete-within-view",
+        examined: 1,
+        total: 1,
+        next_offset: null,
+        exhausted: true,
+      })
+    )
+      throw new Error("PCAP summary coverage differs from its complete view");
+    capture = n.summary;
+    const parentEvidenceId = observations[0].parent_evidence_id;
+    const view = { kind: "pcap", facet: "http-uris", offset: 0, limit: 2 };
+    const selected = await call("inspect_analysis_view", {
+      source: { kind: "retained-evidence", evidence_id: parentEvidenceId },
+      view,
+    });
+    const page = normalized(selected);
+    const observation = pcapObservation(selected, view, parseEvidence);
+    const total = capture.retained_counts.http_request_uris;
+    const examined = Math.min(view.limit, total);
+    const expectedCoverage = {
+      status: "page",
+      examined,
+      total,
+      next_offset: examined < total ? examined : null,
+      exhausted: examined >= total,
+    };
+    if (
+      page.parent_evidence_id !== parentEvidenceId ||
+      page.parent_operation !== "inspect_pcap_capture" ||
+      page.kind !== "pcap" ||
+      page.page.facet !== "http-uris" ||
+      page.source_coverage !== capture.coverage ||
+      !isDeepStrictEqual(page.coverage, expectedCoverage)
+    )
+      throw new Error(
+        "PCAP page identity, scope or coverage differs from its parent",
+      );
+    uris = page.page.items;
+    if (
+      observations.some((item) => item.evidence_id === observation.evidence_id)
+    )
+      throw new Error("PCAP retained read reused an Evidence identity");
+    observations.push(observation);
+  } else {
+    const {
+      endpoints,
+      dns_queries,
+      tls_sni_values,
+      protocol_distribution,
+      http_request_uris,
+      unattributed_dns_addresses,
+      diagnostics,
+      ...metadata
+    } = n;
+    capture = {
+      ...metadata,
+      retained_counts: {
+        endpoints: endpoints.length,
+        dns_queries: dns_queries.length,
+        tls_sni_values: tls_sni_values.length,
+        protocol_distribution: protocol_distribution.length,
+        http_request_uris: http_request_uris.length,
+        unattributed_dns_addresses: unattributed_dns_addresses.length,
+      },
+      diagnostics_code_units:
+        diagnostics === undefined
+          ? null
+          : {
+              tshark: diagnostics.tshark_stderr.length,
+              capinfos: diagnostics.capinfos_stderr.length,
+            },
+    };
+    uris = http_request_uris.slice(0, 2);
+  }
+  if (uris.length !== Math.min(2, capture.retained_counts.http_request_uris))
+    throw new Error("Runtime did not return the requested retained URI prefix");
+  return {
+    artifact_sha256: targetSha(inspected),
+    capture,
+    http_request_uris_first2: uris,
+    delivery_observations: observations,
+    evidence_ids: observations.map((item) => item.evidence_id),
+  };
+};
+
 const workflows = {
+  "wireshark-http.cap": inspectPcapTarget,
+  "wireshark-http_redirects.pcapng": inspectPcapTarget,
   "qgis-world-map.gpkg": async () => {
     const columns = ["fid", "NAME", "ISO_A3"];
     const input = {

experiments/analysis-efficiency/pcap-small-captures.md

@@ -0,0 +1,291 @@
+# Small real PCAP recordings: keep complete delivery
+
+Continues [#2016](https://github.com/morluto/rea/issues/2016) after the QGIS
+SQLite round. Measured main is pinned to
+`3ae826ceba6f25b08bce50bcb51001209bc16b57`, with 23 intervening commits.
+These include bounded PCAP summaries/pages, authenticated artifact subjects,
+unknown-snapshot reuse and shared transcript grading. This round measures existing
+PCAP delivery choices; it adds no production, API, default, cache or skill change.
+
+## Corpus and question
+
+Use two unmodified recordings published on the official
+[Wireshark sample-capture page](https://wiki.wireshark.org/SampleCaptures):
+
+| Recording                               |  Bytes | SHA-256                                                            |
+| --------------------------------------- | -----: | ------------------------------------------------------------------ |
+| HTTP request/response, `http.cap`       | 25,803 | `25a72bdf10339f2c29916920c8b9501d294923108de8f29b19aba7cc001ab60d` |
+| HTTP redirects, `http_redirects.pcapng` | 47,804 | `ff432a6b71d06cd20720f7763e584014edd0a43c3b55981c682d1b7640207abc` |
+
+`corpus.json` preserves original/resolved URLs. Local digests pin downloaded
+bytes; no independently published upstream digest or immutable source commit is
+claimed. These are upstream saved application-protocol recordings, not generated
+REA fixtures. Recorded clients and current deployments are unknown. No new
+traffic is captured and no recorded target/URL is executed or fetched.
+
+The question asks for complete returned identity/client/scalar metadata, every
+observed count and source coverage, six retained inventory counts, explicit nulls,
+diagnostic UTF-16 lengths, and the first two retained HTTP URIs in original order.
+Preserve each used receipt's limitations and identity. Summary/page scope and
+unknowns remain distinct from source coverage; exhaustion does not establish
+network completeness. Other inventory lists and diagnostic text are unrequested.
+
+Compare default `{path}` with existing `{path, detail: "summary"}` plus a retained
+`http-uris` page, offset 0, limit 2. Ordinary selected-field
+`inspect_packet_capture` remains a separate boundary, not a substitute claim
+about complete inventories. `questions/pcap-http.txt` contains the fixed question.
+
+## SDK measurements
+
+Frozen dist/bridge file-table SHA-256:
+`db050545e94dc305a48042e06df96ac6c64b21ee6c5c4707c787d5e46df79e47`.
+Linux x64, Node 24.18.0, SDK client/server 2.3.1, installed Wireshark 4.0.17.
+Explicit commands are `/usr/bin/tshark`, `/usr/bin/capinfos`, `/usr/bin/prlimit`;
+private receipts retain executable hashes. Each workflow starts a fresh server
+and uses actual named advertised definitions/schema validation. Host page caches
+are not flushed. Both tools read the owned snapshot.
+
+Eight trials run complete/summary/summary/complete per recording. Both pairs
+match selected capture facts, URI prefixes, settings and original source bytes.
+Each delivery's exact limitations, identities and coverage remain separately
+recorded, rather than equating source coverage with page exhaustion.
+
+| Trial                | Reply bytes | SDK seconds | Payload CPU seconds | Kernel peak bytes |
+| -------------------- | ----------: | ----------: | ------------------: | ----------------: |
+| HTTP complete-1      |       7,601 |       4.349 |               5.336 |       611,778,560 |
+| HTTP summary-1       |      17,540 |       4.972 |               5.973 |       601,661,440 |
+| HTTP summary-2       |      17,540 |       5.101 |               6.092 |       611,438,592 |
+| HTTP complete-2      |       7,601 |       4.463 |               5.503 |       617,148,416 |
+| Redirects complete-1 |       7,535 |       4.670 |               5.641 |       610,627,584 |
+| Redirects summary-1  |      17,240 |       5.099 |               6.104 |       615,522,304 |
+| Redirects summary-2  |      17,240 |       5.060 |               6.056 |       610,385,920 |
+| Redirects complete-2 |       7,535 |       4.466 |               5.458 |       608,141,312 |
+
+Bytes are compact client-result JSON summed over one complete or two summary/page
+calls, not model tokens or raw wire bytes. Summary/page expands replies to more
+than twice the complete result on both recordings. All four SDK pairs are slower
+and use more payload CPU; memory differences are small/inconsistent. CPU includes
+driver/catalog work outside the SDK workflow timer.
+
+Reproduce with `measure.mjs --artifact wireshark-http.cap` or
+`wireshark-http_redirects.pcapng`, pinned path/runtime/revision, explicit provider
+JSON, `--schemas selected`, and `--delivery complete|summary`. Use `compare.mjs`
+on completed output directories. Original replies, catalogs, comparison receipts
+and exact measured driver source stay private. Repeated deliveries in the same
+mode also compare the ordered receipt observations, including limitations, view
+parameters, coverage and unknowns; only receipt/parent IDs are invocation-specific.
+Comparator regressions run with `npm run test:local -- tests/evaluation/analysisEfficiencyComparison.test.ts`
+using synthetic measurement directories; they do not establish provider behavior.
+
+## Actual model tasks
+
+Four actual Codex CLI 0.161.0 tasks use requested `gpt-5.6-terra` high, the same
+redirects question/runtime/artifact/provider settings, direct mode, two exact
+enabled tools and equal context/discovery capture flags. The requested alias is
+not verified server-resolved model identity. Equal cohort instructions select
+complete or summary plus one URI page. The experiment driver applies its existing
+private per-tool approval policy to `inspect_pcap_capture`; global configuration
+and REA authority are unchanged.
+
+| Trial      |  Input | Cached | Uncached | Output | CLI wall seconds | MCP calls | Facts |
+| ---------- | -----: | -----: | -------: | -----: | ---------------: | --------: | ----- |
+| complete-1 | 38,947 |  9,984 |   28,963 |  2,119 |           69.469 |         1 | pass  |
+| summary-1  | 58,779 | 38,144 |   20,635 |  3,241 |          141.304 |         2 | pass  |
+| summary-2  | 58,584 | 38,144 |   20,440 |  4,261 |           50.916 |         2 | pass  |
+| complete-2 | 38,917 | 23,040 |   15,877 |  2,813 |          106.896 |         1 | pass  |
+
+Summary/page total input **increases 50.92% / 50.54%** and output increases
+52.95% / 51.48%. Uncached input is mixed: 28.75% lower / 28.74% higher. Time is
+mixed: 103.41% slower / 52.37% faster. No cause is inferred for caching or timing
+variation. Cached input is included in total; each CLI reports one final aggregate
+usage. Output includes 1,149 / 1,162 / 2,196 / 1,837 reasoning tokens.
+
+All calls succeed without extras/retries. Events contain only the selected MCP
+tools and agent messages. All source-digest and private-profile cleanup checks
+pass. Model payload CPU seconds/kernel peaks: 7.818/791,060,480;
+9.793/752,705,536; 7.685/757,891,072; 7.999/750,993,408. These do not establish
+a general CPU/memory gain.
+
+The first summary initially fails the grader because it uses explicit equivalent
+labels and a URI object retaining page/source coverage. Its facts are correct.
+The grading boundary accepts these known forms while checking exact types,
+null presence, original limitations and extra scopes against actual receipts.
+Conflicting duplicates and unsupported claims remain failures. Each cited
+receipt's own ID must be reported; its parent lookup ID is checked separately.
+Original answers/reviews/grader source remain intact. No model is rerun. A grader
+review pause separates the pairs; it is not task time or a cold-cache claim.
+
+## Verification and decision
+
+An independent stdlib raw PCAP/PCAPNG/Ethernet/IP/TCP/complete-HTTP-header parser
+agrees on 43/271 packets, per-packet endpoint counts, first URI prefixes and 2/23
+distinct HTTP URIs. It invokes no Wireshark. DNS associations, protocol-stack
+dissection, nullable timestamps and live-network semantics are not independently
+validated.
+
+Both real CLI complete results match full MCP normalized results, subjects,
+parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained
+normalized facts and subjects; source forms/caller parameters remain distinct.
+Source bytes stay unchanged. Thirty-four context/grading regressions pass, including
+cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID
+lists that reject parent lookups, duplicates and nested representations. URI
+prefix coverage must come from its cited page, even if an uncited page has the
+same items. The SDK driver also checks the returned view against the exact
+requested kind, facet, offset and limit. Five real-answer
+mutations—omitted null, false page exhaustion, contradictory source scope,
+unsupported absence claim and non-string receipt identity—fail correctly.
+Malformed identities produce failed facts rather than crashing the grader.
+JS syntax, Python compilation and
+relevant standard lint pass.
+
+A fresh real SDK summary/URI smoke matches the original selected facts. Three
+private consumer fault injections alter the returned view's offset, limit or
+presence after a real SDK reply; the driver rejects each. These establish the
+experiment consumer's guard, not provider malformed-reply behavior. The initial
+injection setup loaded the separate CommonJS client and did not inject a fault;
+that discarded setup attempt remains private alongside the corrected ESM runs.
+Original measured driver sources, token totals and timing data remain unchanged.
+
+The driver also requires a distinct Evidence receipt for the retained URI page.
+A private SDK consumer injection reuses the summary's response ID after a real
+MCP reply: the preceding driver accepts it, the corrected driver rejects it
+without writing an answer, and an unmodified real reply still passes. This
+checks receipt uniqueness before recording observations; it does not establish
+that a provider actually returned duplicate IDs.
+
+The page guard checks the complete expected coverage object, including status,
+continuation and exhaustion. Private actual-reply injections demonstrate that
+the preceding driver accepts inconsistent coverage, while the corrected driver
+rejects wrong status, next offset or exhaustion. Unmodified exhausted and
+continuable URI pages still pass. The grader also requires each cited receipt's
+subject digest and local path to identify the reference capture, and uses the
+reference page's verified coverage. Thirty-four grading/context regressions pass,
+including missing or malformed subjects and faithful but inconsistent coverage;
+all four original model answers still pass factual regrading. Strict resource
+grades remain false where terminal whole-scope counters are unknown.
+
+PCAP observation production now validates both the Evidence subject and
+normalized target against the selected path/digest for complete, summary and
+page replies. Retained parents must be declared in each view receipt's Evidence
+links and differ from that response's own ID. The grader enforces the same link
+relationship. Private actual-reply injections and normal complete/summary
+controls exercise this boundary; these do not establish a provider defect.
+
+The grader preserves each receipt's producing tool while indexing it; conflicting
+tool/envelope ownership for one ID remains ambiguous, while identical repeated
+receipts are allowed. Complete delivery requires the capture tool/operation.
+Summary delivery uses the capture tool but correctly delegates its Evidence
+operation to `inspect_analysis_view`; the URI page uses that view tool/operation.
+Both the grader and SDK observation producer validate this actual contract.
+Wrong-tool/operation regressions fail before the correction and pass afterward;
+private SDK operation faults are rejected while real complete/summary controls
+and all four stored model factual grades still pass. No operation is rewritten
+to make the tool name and delegated Evidence operation identical.
+
+The receipt index parses the required envelope/nested-view containers before
+allowing lookup, so missing or malformed normalized results/limitations become
+failed facts with a written report. Empty observation lists also remain failed
+facts and never enter citation checks that require a first receipt. Coverage uses type-sensitive JSON comparison
+to distinguish booleans, integers and floats. Receipt-identity conflict checks
+use the same type-sensitive rule, so a scalar type change cannot hide in a
+repeated ID. Existing PCAP scope-alias checks already preserve JSON types.
+Requested and cited URI views use the same type-sensitive JSON rule, rejecting
+boolean or floating-point offsets/limits even when an answer faithfully copies
+them. The observation producer rejects missing, null or malformed response IDs
+before recording either capture or page receipts. Private real-reply injections
+exercise these ID failures alongside normal complete/summary controls; they do
+not establish malformed provider behavior.
+Completed-call arguments stay with their producing tool and receipt in the
+grader's identity index. Cited envelope parameters and actual call arguments
+must match the selected path or retained parent/view, including delegated
+summary parameters. The observation producer validates both summary and URI
+views and envelope parameters at their common boundary. Private real-reply
+faults reproduce missing-parameter/summary-view acceptance before the fix and
+are rejected afterward; normal complete/summary controls still pass.
+Negative regressions reproduce
+prior crashes and false passes; all original real-model factual grades still
+pass without model reruns. The SDK workflow is unchanged by these grader fixes.
+
+**Decision: keep existing complete defaults; add no production optimization.**
+For these compact recordings and this question, summary/page costs more total
+input in both real-model pairs. No stable speed or CPU/memory benefit justifies
+another API, automatic delivery switch, cache or plugin layer. Existing pages
+remain available; their benefit needs evidence for the actual question/inventory.
+Large captures, other protocols, Windows/macOS/Bun Wireshark and other model
+workloads remain unverified here.
+
+Heavy work stays serial with one CPU, 3 GiB early stop/4 GiB hard memory and no
+swap. Build kernel peak: 2,575,732,736 bytes. Payload cleanup and zero phase limit
+events are verified; missing terminal whole-scope counters remain unknown, so
+strict whole-resource grades stay false. No raw capture, trace, credential,
+runtime or result JSON is committed. No unrelated REA product bug is identified
+or fixed, and no release is performed.
+
+Review follow-up binds the cited URI page to its exact requested facet/range,
+reported values and the summary retained parent. Small synthetic trial receipts
+exercise the entire grading entrypoint, including faithful but wrong-page/range/
+parent citations, missing nulls, false scope and malformed identities. These are
+grader regressions, not real-provider evidence. All four stored actual model
+answers still pass; no model reruns are required. Transcript readers close their
+owned file handles after grading.
+
+The SDK driver and factual grader require summary coverage to describe its one
+complete view and require each present normalized limitations/unknowns field to
+be a string array. Missing fields and valid empty arrays remain distinct.
+Controlled consumer mutations over actual SDK replies reproduce the previously
+accepted inconsistent summary coverage and verify rejection after the guard;
+complete and summary controls still pass. These mutations do not establish a
+malformed-response defect in the provider. The four stored model answers still
+pass factual regrading without another model run.
+
+Summary views must preserve both the outer artifact path/digest and the nested
+summary target, as well as the Evidence subject. The full grader rejects
+faithful receipts with missing or incorrect outer artifact identities. Private
+mutations over actual SDK replies reproduce the previous false success and
+verify rejection after the shared observation guard; normal deliveries and all
+four original model answers retain their recorded facts.
+
+Declared retained parent digests must match the digest encoded in the linked
+parent Evidence ID for both summary and page views. SDK acceptance and factual
+grading reject mismatches and missing digests while preserving complete
+deliveries. Before/after private actual-reply mutations verify both view roles;
+unmodified deliveries and four stored original-model factual regrades pass.
+
+The SDK observation boundary additionally authenticates each complete, summary
+and page envelope with the selected runtime’s existing `parseEvidence` owner.
+It preserves the original envelope, without duplicating semantic-ID hashing.
+Format-valid swapped-ID mutations over actual SDK replies pass before this
+guard and fail afterward; unmodified complete/summary controls pass. The model
+grader tests selected facts against cited transcript receipts; its synthetic
+fixtures do not establish Evidence authentication. The original real-model
+receipts are independently authenticated against the pinned runtime.
+
+The grader separates truthful cited facts from equivalence of the measured
+workflow. PCAP workflow checks require the planned ordered capture call (complete)
+or summary capture followed by the URI-page call, with exact arguments and
+completed status. Extra, reversed, failed or undecodable attempts fail workflow
+acceptance even when the final cited facts are correct; their actual cost and
+errors remain recorded. All four original model trials pass both factual and
+workflow checks without model reruns. Strict resource grades remain unchanged.
+
+Each planned call must also decode as an Evidence object with its own cited
+response ID. An unreadable planned reply cannot independently pass workflow
+checks merely because tool names, arguments and terminal status match. The
+regression reproduces the previous workflow-only false success; original model
+factual/workflow grades remain unchanged.
+
+Workflow citation cardinality must equal the planned call count before binding
+per-call IDs. A shorter or longer answer list cannot silently shorten Python
+zip’s expected plan; missing URI calls and extra citations are rejected.
+Original complete/summary model workflows continue to pass unchanged.
+
+The six comparator CLI regressions now live in the normally discovered Vitest
+evaluation suite, so routine local/PR/CI gates execute them. The original six
+scenarios and CLI assertions are retained; the former manual Node suite is
+removed. They remain synthetic consumer checks, not provider evidence.
+
+Failed calls with scalar/list replies or malformed nested resource-error
+containers are parsed defensively. The grader writes its report, preserves the
+original failure and any supported cited facts, and rejects workflow acceptance
+instead of crashing. Genuine structured resource Evidence references remain
+observable; no malformed producer behavior is inferred from synthetic cases.

experiments/analysis-efficiency/questions/pcap-http.txt

@@ -0,0 +1 @@
+Statically inspect the supplied saved Wireshark capture at {{artifact_path}} using only registered REA inspect_pcap_capture and inspect_analysis_view. Report its original client/target identity and every returned scalar metadata/count/coverage field exactly, preserving explicit nulls. Include retained_counts for endpoints, dns_queries, tls_sni_values, protocol_distribution, http_request_uris and unattributed_dns_addresses; derive array lengths only when reading complete Evidence. Include diagnostics_code_units as exact UTF-16 lengths of retained capinfos/tshark stderr, or null if diagnostics are unrecorded. In capture, omit inventory arrays and diagnostic text; do not omit their observed counts or change source coverage. Report the first two retained HTTP URIs in original order, or all if fewer; do not request other inventory pages. Report delivery_observations in dependency order for the Evidence used: evidence_id, outer limitations, and each present normalized kind, view, parent_evidence_id, coverage, unknowns and normalized_limitations. Preserve presence, explicit nulls, scalar types and original limitation order. Distinguish source coverage from URI-page exhaustion. Final JSON fields: artifact_sha256, capture, http_request_uris_first2, delivery_observations, evidence_ids. Include only actual response IDs. Saved packet/URI observations do not establish current deployment, process identity, missing traffic, decrypted payloads or current DNS resolution. Do not use shell, host file tools, network, other agents, traffic acquisition or execute recorded targets. Stop once these selected facts are supported.

experiments/analysis-efficiency/review-model.py

@@ -67,9 +67,47 @@ def scope_with_verified_counts(value, expected_scope, returned_counts):
     return scope, checks
 
 
+def pcap_observation_fields(value):
+    """Equivalent explicit labels are accepted; conflicting duplicates remain failures."""
+    if not isinstance(value, dict):
+        return value
+    result = dict(value)
+    for label, canonical in {
+        'outer_limitations': 'limitations', 'normalized_kind': 'kind',
+        'normalized_view': 'view', 'normalized_parent_evidence_id': 'parent_evidence_id',
+        'normalized_coverage': 'coverage', 'normalized_unknowns': 'unknowns',
+        'normalized_source_coverage': 'source_coverage',
+    }.items():
+        if label not in result:
+            continue
+        if canonical in result and json.dumps(result[label], sort_keys=True) != json.dumps(result[canonical], sort_keys=True):
+            return {**result, 'conflicting_fields': [label, canonical]}
+        result[canonical] = result.pop(label)
+    return result
+
+
+def pcap_complete_capture(value):
+    """Project the selected metadata/counts from the cited complete receipt."""
+    inventories = ['endpoints', 'dns_queries', 'tls_sni_values', 'protocol_distribution',
+                   'http_request_uris', 'unattributed_dns_addresses']
+    if not isinstance(value, dict) or not all(isinstance(value.get(key), list) for key in inventories):
+        return None
+    capture = {key: item for key, item in value.items() if key not in inventories + ['diagnostics']}
+    capture['retained_counts'] = {key: len(value[key]) for key in inventories}
+    diagnostics = value.get('diagnostics')
+    if 'diagnostics' not in value:
+        capture['diagnostics_code_units'] = None
+    elif isinstance(diagnostics, dict) and all(isinstance(diagnostics.get(key + '_stderr'), str) for key in ['tshark', 'capinfos']):
+        capture['diagnostics_code_units'] = {key: len(diagnostics[key + '_stderr'].encode('utf-16-le', errors='surrogatepass')) // 2
+                                             for key in ['tshark', 'capinfos']}
+    else:
+        return None
+    return capture
+
+
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'flutter-aot', 'sqlite', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True)
+    parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'flutter-aot', 'sqlite', 'pcap', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True)
     parser.add_argument('--trial', type=Path, required=True)
     parser.add_argument('--reference', type=Path)
     parser.add_argument('--resource-log', type=Path, required=True)
@@ -83,25 +121,43 @@ def main():
     reference = read(args.reference / 'answer.json') if args.reference else None
     observed_ids = set()
     producer_results = {}
+    pcap_calls = []
+    pcap_attempts = []
+    workflow_checks = {'pcap_exact_call_sequence': False} if args.domain == 'pcap' else {}
     tool_errors = []
-    for line in (args.trial / 'events.jsonl').open():
-        event = json.loads(line)
-        item = event.get('item', {})
-        if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call':
-            continue
-        result = item.get('result') or {}
-        parsed = decode(result)
-        if item['status'] == 'completed':
-            # Only response text contributes observed IDs, never model arguments.
-            for text in strings(result):
-                observed_ids.update(re.findall(r'ev_[a-f0-9]{64}', text))
-            if parsed is not None:
-                producer_results.setdefault(item['tool'], []).append(parsed)
-        else:
-            tool_errors.append({'tool': item['tool'], 'arguments': item['arguments'], 'response': parsed if parsed is not None else result})
-            if parsed is not None and parsed.get('error', {}).get('code') == 'resource_constraint':
-                reference_value = parsed['error'].get('details', {}).get('reported_limits', {}).get('evidence_reference')
-                observed_ids.update(evidence_ids(reference_value))
+    with (args.trial / 'events.jsonl').open() as event_lines:
+        for line in event_lines:
+            event = json.loads(line)
+            item = event.get('item', {})
+            if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call':
+                continue
+            result = item.get('result') or {}
+            parsed = decode(result)
+            if args.domain == 'pcap':
+                pcap_attempts.append({'tool': item.get('tool'), 'arguments': item.get('arguments'),
+                                      'status': item.get('status'),
+                                      'evidence_id': parsed.get('evidence_id') if isinstance(parsed, dict) else None,
+                                      'readable_evidence': isinstance(parsed, dict)
+                                      and isinstance(parsed.get('normalized_result'), dict)
+                                      and isinstance(parsed.get('evidence_id'), str)
+                                      and re.fullmatch(r'ev_[a-f0-9]{64}', parsed['evidence_id']) is not None})
+            if item['status'] == 'completed':
+                # Only response text contributes observed IDs, never model arguments.
+                for text in strings(result):
+                    observed_ids.update(re.findall(r'ev_[a-f0-9]{64}', text))
+                if parsed is not None:
+                    if args.domain == 'pcap':
+                        pcap_calls.append((item['tool'], item.get('arguments'), parsed))
+                    else:
+                        producer_results.setdefault(item['tool'], []).append(parsed)
+            else:
+                tool_errors.append({'tool': item['tool'], 'arguments': item['arguments'], 'response': parsed if parsed is not None else result})
+                error = parsed.get('error') if isinstance(parsed, dict) else None
+                if isinstance(error, dict) and error.get('code') == 'resource_constraint':
+                    details = error.get('details')
+                    limits = details.get('reported_limits') if isinstance(details, dict) else None
+                    if isinstance(limits, dict):
+                        observed_ids.update(evidence_ids(limits.get('evidence_reference')))
     reported_ids = evidence_ids(answer.get('evidence_ids'))
     checks = {
         'reported_evidence_ids_observed_in_responses': bool(reported_ids) and reported_ids <= observed_ids,
@@ -138,6 +194,187 @@ def main():
             # Missing is not SQL/JSON null; booleans are not numeric counters.
             check(key, (key in answer, json.dumps(answer.get(key), sort_keys=True)),
                   (True, json.dumps(reference[key], sort_keys=True)))
+    elif domain == 'pcap':
+        check('capture', ('capture' in answer, json.dumps(answer.get('capture'), sort_keys=True)),
+              (True, json.dumps(reference['capture'], sort_keys=True)))
+        prefix = answer.get('http_request_uris_first2')
+        records = {}
+        for tool, arguments, value in pcap_calls:
+            if not isinstance(value, dict) or not isinstance(value.get('evidence_id'), str):
+                continue
+            identity = value['evidence_id']
+            normalized = value.get('normalized_result')
+            supported = (isinstance(normalized, dict) and isinstance(value.get('limitations'), list)
+                         and all(isinstance(item, str) for item in value['limitations']))
+            if supported:
+                supported = all(key not in normalized or (isinstance(normalized[key], list)
+                                and all(isinstance(item, str) for item in normalized[key]))
+                                for key in ['limitations', 'unknowns'])
+            if supported and ('summary' in normalized or normalized.get('kind') == 'summary'):
+                supported = isinstance(normalized.get('summary'), dict)
+            if supported and ('page' in normalized or normalized.get('kind') == 'pcap'):
+                supported = isinstance(normalized.get('page'), dict)
+            entry = (tool, arguments, value) if supported else None
+            if identity not in records:
+                records[identity] = entry
+            elif json.dumps(records[identity], sort_keys=True) != json.dumps(entry, sort_keys=True):
+                # An ID with conflicting producers or envelopes is ambiguous.
+                records[identity] = None
+        observations = answer.get('delivery_observations')
+        canonical_observations = [pcap_observation_fields(value) for value in observations] if isinstance(observations, list) else observations
+        if isinstance(prefix, dict):
+            checks['pcap_prefix_fields_supported'] = set(prefix) <= {'items', 'coverage', 'source_coverage'} and 'items' in prefix
+            if 'source_coverage' in prefix:
+                check('pcap_prefix_source_coverage', prefix['source_coverage'], reference['capture']['coverage'])
+            if 'coverage' in prefix:
+                last = canonical_observations[-1] if isinstance(canonical_observations, list) and canonical_observations else None
+                identity = last.get('evidence_id') if isinstance(last, dict) else None
+                entry = records.get(identity) if isinstance(identity, str) else None
+                record = entry[2] if entry is not None else None
+                page = record['normalized_result'] if record is not None else {}
+                checks['pcap_prefix_coverage_observed'] = (
+                    page.get('kind') == 'pcap'
+                    and page.get('page', {}).get('facet') == 'http-uris'
+                    and json.dumps(page['page'].get('items'), sort_keys=True) == json.dumps(prefix.get('items'), sort_keys=True)
+                    and 'coverage' in page
+                    and json.dumps(prefix['coverage'], sort_keys=True) == json.dumps(page['coverage'], sort_keys=True)
+                )
+            prefix = prefix.get('items')
+        check('http_request_uris_first2', ('http_request_uris_first2' in answer, json.dumps(prefix, sort_keys=True)),
+              (True, json.dumps(reference['http_request_uris_first2'], sort_keys=True)))
+        expected_observations = []
+        if isinstance(canonical_observations, list):
+            for observation in canonical_observations:
+                identity = observation.get('evidence_id') if isinstance(observation, dict) else None
+                entry = records.get(identity) if isinstance(identity, str) else None
+                record = entry[2] if entry is not None else None
+                if record is None:
+                    expected_observations.append(None)
+                    continue
+                value = record['normalized_result']
+                expected = {'evidence_id': record['evidence_id'], 'limitations': record['limitations']}
+                for key in ['kind', 'view', 'parent_evidence_id', 'coverage', 'unknowns']:
+                    if key in value:
+                        expected[key] = value[key]
+                if 'limitations' in value:
+                    expected['normalized_limitations'] = value['limitations']
+                if 'source_coverage' in observation and 'source_coverage' in value:
+                    expected['source_coverage'] = value['source_coverage']
+                expected_observations.append(expected)
+        checks['pcap_delivery_observations_nonempty'] = isinstance(observations, list) and bool(observations)
+        check('pcap_original_delivery_observations', json.dumps(canonical_observations, sort_keys=True),
+              json.dumps(expected_observations, sort_keys=True))
+        if isinstance(canonical_observations, list) and canonical_observations and all(isinstance(x, dict) for x in canonical_observations):
+            check('pcap_requested_delivery_chain', [x.get('kind', 'complete') for x in canonical_observations],
+                  [x.get('kind', 'complete') for x in reference['delivery_observations']])
+            check('pcap_requested_views', json.dumps([x.get('view') for x in canonical_observations], sort_keys=True),
+                  json.dumps([x.get('view') for x in reference['delivery_observations']], sort_keys=True))
+            if len(reference['delivery_observations']) == 2:
+                check('pcap_summary_coverage', json.dumps(canonical_observations[0].get('coverage'), sort_keys=True),
+                      json.dumps({'status': 'complete-within-view', 'examined': 1, 'total': 1,
+                                  'next_offset': None, 'exhausted': True}, sort_keys=True))
+            identities = [x.get('evidence_id') for x in canonical_observations]
+            checks['pcap_observations_have_reported_ids'] = (
+                isinstance(answer.get('evidence_ids'), list)
+                and answer['evidence_ids'] == identities
+                and all(isinstance(value, str) and re.fullmatch(r'ev_[a-f0-9]{64}', value) for value in identities)
+                and len(set(identities)) == len(identities)
+            )
+            entries = [records.get(identity) if isinstance(identity, str) else None for identity in identities]
+            cited = [entry[2] if entry is not None else None for entry in entries]
+            expected_tools = ['inspect_pcap_capture'] + (['inspect_analysis_view'] if len(reference['delivery_observations']) == 2 else [])
+            check('pcap_cited_producer_tools', [entry[0] if entry is not None else None for entry in entries], expected_tools)
+            # Summary delivery delegates to a retained view; its envelope operation
+            # is inspect_analysis_view even though the requested tool is inspect_pcap_capture.
+            expected_operations = ['inspect_pcap_capture'] if len(expected_tools) == 1 else ['inspect_analysis_view'] * 2
+            check('pcap_cited_operations', [record.get('operation') if isinstance(record, dict) else None for record in cited], expected_operations)
+            path = reference['capture']['target']['path']
+            expected_arguments = [{'path': path}]
+            expected_parameters = [{'path': path}]
+            if len(expected_tools) == 2:
+                parent = cited[0]['normalized_result'].get('parent_evidence_id') if isinstance(cited[0], dict) else None
+                views = [observation.get('view') for observation in reference['delivery_observations']]
+                expected_arguments = [{'path': path, 'detail': 'summary'},
+                                      {'source': {'kind': 'retained-evidence', 'evidence_id': parent}, 'view': views[-1]}]
+                expected_parameters = [{'parent_evidence_id': parent, 'view': view} for view in views]
+            check('pcap_cited_call_arguments', json.dumps([entry[1] if entry is not None else None for entry in entries], sort_keys=True),
+                  json.dumps(expected_arguments, sort_keys=True))
+            workflow_checks['pcap_exact_call_sequence'] = (
+                len(identities) == len(expected_tools)
+                and json.dumps(pcap_attempts, sort_keys=True) == json.dumps([
+                    {'tool': tool, 'arguments': arguments, 'status': 'completed',
+                     'evidence_id': identity, 'readable_evidence': True}
+                    for tool, arguments, identity in zip(expected_tools, expected_arguments, identities)
+                ], sort_keys=True)
+            )
+            check('pcap_cited_parameters', json.dumps([record.get('parameters') if isinstance(record, dict) else None for record in cited], sort_keys=True),
+                  json.dumps(expected_parameters, sort_keys=True))
+            checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all(
+                isinstance(record, dict)
+                and isinstance(record.get('subject'), dict)
+                and isinstance(record['subject'].get('digest'), dict)
+                and record['subject']['digest'].get('sha256') == reference['artifact_sha256']
+                and record['subject'].get('local_path') == reference['capture']['target']['path']
+                for record in cited
+            )
+            checks['pcap_cited_parents_linked'] = bool(cited) and all(isinstance(record, dict) for record in cited) and (
+                len(cited) == 1 or (len(cited) == 2 and all(
+                    isinstance(record['normalized_result'].get('parent_evidence_id'), str)
+                    and re.fullmatch(r'ev_[a-f0-9]{64}', record['normalized_result']['parent_evidence_id'])
+                    and record['normalized_result'].get('parent_digest') == record['normalized_result']['parent_evidence_id'][3:]
+                    and record['normalized_result']['parent_evidence_id'] != record['evidence_id']
+                    and isinstance(record.get('evidence_links'), list)
+                    and record['normalized_result']['parent_evidence_id'] in record['evidence_links']
+                    for record in cited
+                ))
+            )
+            checks['pcap_cited_source_supports_uri_prefix'] = False
+            checks['pcap_cited_capture'] = False
+            if all(isinstance(record, dict) for record in cited):
+                if len(cited) == 1:
+                    source = cited[0]['normalized_result']
+                    check('pcap_cited_capture', json.dumps(answer.get('capture'), sort_keys=True),
+                          json.dumps(pcap_complete_capture(source), sort_keys=True))
+                    checks['pcap_cited_source_supports_uri_prefix'] = (
+                        source.get('target') == reference['capture']['target']
+                        and isinstance(source.get('http_request_uris'), list)
+                        and json.dumps(source['http_request_uris'][:2], sort_keys=True) == json.dumps(prefix, sort_keys=True)
+                    )
+                elif len(cited) == 2:
+                    source, page = [record['normalized_result'] for record in cited]
+                    capture = source.get('summary', {})
+                    check('pcap_cited_capture', json.dumps(answer.get('capture'), sort_keys=True),
+                          json.dumps(capture, sort_keys=True))
+                    selected_page = page.get('page', {})
+                    checks['pcap_cited_source_supports_uri_prefix'] = (
+                        source.get('kind') == 'summary'
+                        and source.get('parent_operation') == 'inspect_pcap_capture'
+                        and capture == reference['capture']
+                        and source.get('artifact') == {key: capture['target'][key] for key in ['path', 'sha256']}
+                        and isinstance(source.get('parent_evidence_id'), str)
+                        and page.get('parent_evidence_id') == source['parent_evidence_id']
+                        and page.get('parent_operation') == 'inspect_pcap_capture'
+                        and json.dumps(page.get('view'), sort_keys=True) == json.dumps({'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}, sort_keys=True)
+                        and selected_page.get('facet') == 'http-uris'
+                        and json.dumps(selected_page.get('items'), sort_keys=True) == json.dumps(prefix, sort_keys=True)
+                        and page.get('source_coverage') == capture.get('coverage')
+                        and json.dumps(page.get('coverage'), sort_keys=True) == json.dumps(reference['delivery_observations'][-1].get('coverage'), sort_keys=True)
+                        and page.get('artifact') == {key: capture['target'][key] for key in ['path', 'sha256']}
+                    )
+        else:
+            checks['pcap_requested_delivery_chain'] = False
+            if len(reference['delivery_observations']) == 2:
+                checks['pcap_summary_coverage'] = False
+            checks['pcap_observations_have_reported_ids'] = False
+            checks['pcap_requested_views'] = False
+            checks['pcap_cited_source_supports_uri_prefix'] = False
+            checks['pcap_cited_capture'] = False
+            checks['pcap_cited_subjects_match_artifact'] = False
+            checks['pcap_cited_parents_linked'] = False
+            checks['pcap_cited_producer_tools'] = False
+            checks['pcap_cited_operations'] = False
+            checks['pcap_cited_call_arguments'] = False
+            checks['pcap_cited_parameters'] = False
     elif domain == 'flutter-aot':
         for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']:
             check(key, answer.get(key), reference[key])
@@ -323,6 +560,7 @@ def main():
                     statuses[status] = statuses.get(status, 0) + 1
             check('network_response_statuses', network['observed_response_statuses'], statuses)
     factual_checks = dict(checks)
+    checks.update(workflow_checks)
     scopes = [json.loads(line) for line in args.resource_log.read_text().splitlines() if line.startswith('{')]
     scope = next(value for value in reversed(scopes) if 'resource_scope' in value)
     checks['no_resource_abort'] = scope['abort'] is None
@@ -342,6 +580,8 @@ def main():
         'review_source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
         'domain': domain, 'checks': checks, 'all_selected_checks_pass': all(checks.values()),
         'factual_checks': factual_checks, 'all_factual_checks_pass': all(factual_checks.values()),
+        **({'workflow_checks': workflow_checks, 'all_workflow_checks_pass': all(workflow_checks.values())}
+           if domain == 'pcap' else {}),
         'differences': differences, 'tool_errors': tool_errors, 'summary': summary,
         'fingerprints': {key: value for key, value in settings.items() if 'sha256' in key or key in ['runtime_revision', 'enabled_tools', 'authority']},
         'answer_file_sha256': hashlib.sha256((args.trial / 'answer.txt').read_bytes()).hexdigest(),

experiments/analysis-efficiency/test_codex_context.py

@@ -1,6 +1,7 @@
 """Distinct capture/cleanup failures; simulated CLI is not real-model evidence."""
 
 import importlib.util
+import copy
 import io
 import json
 import os
@@ -38,6 +39,546 @@ class ManagedCoverageTests(unittest.TestCase):
         self.assertNotEqual(scope, expected)
 
 
+class PcapCoverageTests(unittest.TestCase):
+    def test_explicit_labels_preserve_original_nulls_and_source_scope(self):
+        canonical = runpy.run_path(str(Path(__file__).with_name('review-model.py')))['pcap_observation_fields']
+        original = {'outer_limitations': [], 'normalized_kind': 'pcap',
+                    'normalized_coverage': {'examined': 2, 'total': 23, 'next_offset': 2, 'exhausted': False},
+                    'normalized_source_coverage': 'partial', 'normalized_parent_evidence_id': None}
+        result = canonical(original)
+        self.assertEqual(result['limitations'], [])
+        self.assertEqual(result['source_coverage'], 'partial')
+        self.assertEqual(result['coverage']['total'], 23)
+        self.assertIsNone(result['parent_evidence_id'])
+        self.assertIn('outer_limitations', original)
+        self.assertNotIn('parent_evidence_id', canonical({}))
+
+    def test_conflicting_duplicates_and_json_type_substitutions_stay_failures(self):
+        canonical = runpy.run_path(str(Path(__file__).with_name('review-model.py')))['pcap_observation_fields']
+        for wrong in ['partial', None]:
+            result = canonical({'normalized_source_coverage': 'complete', 'source_coverage': wrong})
+            self.assertIn('conflicting_fields', result)
+            self.assertEqual(result['normalized_source_coverage'], 'complete')
+            self.assertEqual(result['source_coverage'], wrong)
+        result = canonical({'normalized_coverage': {'examined': 1}, 'coverage': {'examined': True}})
+        self.assertIn('conflicting_fields', result)
+        self.assertEqual(canonical({'outer_limitations': [], 'limitations': []}), {'limitations': []})
+
+    def test_unsupported_claims_are_retained_for_exact_producer_checking(self):
+        canonical = runpy.run_path(str(Path(__file__).with_name('review-model.py')))['pcap_observation_fields']
+        original = {'normalized_kind': 'pcap', 'unrecorded_traffic_absent': True}
+        self.assertIn('unrecorded_traffic_absent', canonical(original))
+        self.assertIsNone(canonical(None))
+
+
+class PcapGraderWorkflowTests(unittest.TestCase):
+    """Synthetic receipts test the grader boundary, not Wireshark or real models."""
+
+    def fixture(self):
+        parent, summary_id, page_id = ['ev_' + value * 64 for value in ['a', 'b', 'c']]
+        capture = {'target': {'path': '/fixture/http.pcapng', 'sha256': 'd' * 64},
+                   'capture_start': None, 'coverage': 'complete',
+                   'retained_counts': {'http_request_uris': 3}}
+        prefix = ['http://example.test/one', 'http://example.test/two']
+        coverage = {'status': 'page', 'examined': 2, 'total': 3, 'next_offset': 2, 'exhausted': False}
+        subject = {'digest': {'sha256': capture['target']['sha256']}, 'local_path': capture['target']['path']}
+        summary = {'evidence_id': summary_id, 'operation': 'inspect_analysis_view',
+                   'subject': copy.deepcopy(subject), 'evidence_links': [parent],
+                   'limitations': ['source', 'view'],
+                   'normalized_result': {'kind': 'summary', 'view': {'kind': 'summary'},
+                       'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent,
+                       'summary': capture, 'artifact': copy.deepcopy(capture['target']),
+                       'coverage': {'status': 'complete-within-view', 'examined': 1, 'total': 1, 'next_offset': None, 'exhausted': True},
+                       'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}}
+        page = {'evidence_id': page_id, 'operation': 'inspect_analysis_view',
+                'subject': copy.deepcopy(subject), 'evidence_links': [parent],
+                'limitations': ['source', 'view'],
+                'normalized_result': {'kind': 'pcap',
+                    'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2},
+                    'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent,
+                    'page': {'facet': 'http-uris', 'items': prefix}, 'coverage': coverage,
+                    'source_coverage': 'complete', 'artifact': capture['target'],
+                    'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}}
+        for record in [summary, page]:
+            record['normalized_result']['parent_digest'] = parent[3:]
+            record['parameters'] = {'parent_evidence_id': parent, 'view': copy.deepcopy(record['normalized_result']['view'])}
+        observations = [self.observation(value) for value in [summary, page]]
+        reference = {'artifact_sha256': 'd' * 64, 'capture': capture,
+                     'http_request_uris_first2': prefix, 'delivery_observations': observations}
+        answer = copy.deepcopy({**reference, 'http_request_uris_first2': {
+            'items': prefix, 'coverage': coverage, 'source_coverage': 'complete'},
+            'evidence_ids': [summary_id, page_id]})
+        return answer, [summary, page], copy.deepcopy(reference)
+
+    def observation(self, record):
+        value = record['normalized_result']
+        return {'evidence_id': record['evidence_id'], 'limitations': record['limitations'],
+                'kind': value['kind'], 'view': value['view'],
+                'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'],
+                'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']}
+
+    def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None, extra_events=None, producer_results=None):
+        review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory);trial = root / 'trial';trial.mkdir()
+            for name, value in [('summary.json', {'completed_mcp_calls': []}),
+                                ('trial-settings.json', {}), ('cleanup.json', {'profile_removed': True})]:
+                (trial / name).write_text(json.dumps(value))
+            (trial / 'answer.txt').write_text(json.dumps(answer))
+            events = []
+            for index, value in enumerate(records):
+                tool = producer_tools[index] if producer_tools is not None else \
+                    ('inspect_analysis_view' if value['normalized_result'].get('kind') == 'pcap' else 'inspect_pcap_capture')
+                arguments = {'path': reference['capture']['target']['path']}
+                if tool == 'inspect_analysis_view':
+                    normalized = value.get('normalized_result')
+                    parent = normalized.get('parent_evidence_id') if isinstance(normalized, dict) else None
+                    arguments = {'source': {'kind': 'retained-evidence', 'evidence_id': parent},
+                                 'view': reference['delivery_observations'][-1].get('view')}
+                elif len(reference['delivery_observations']) == 2:
+                    arguments['detail'] = 'summary'
+                if producer_arguments is not None:
+                    arguments = producer_arguments[index]
+                events.append({'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
+                               'tool': tool, 'arguments': arguments, 'status': 'completed',
+                               'result': {'structuredContent': value} if producer_results is None else producer_results[index]}})
+            events.extend(extra_events or [])
+            (trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events))
+            source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference))
+            resource = root / 'resource.log'
+            resource.write_text(json.dumps({'resource_scope': 'simulated-grader-only', 'abort': None,
+                                           'resource_counters_verified': False, 'resource_events_final': None}) + '\n')
+            output = root / 'review.json'
+            argv = ['review-model.py', '--domain', 'pcap', '--trial', str(trial), '--reference', str(source),
+                    '--resource-log', str(resource), '--output', str(output)]
+            with patch.object(sys, 'argv', argv), patch('sys.stdout', io.StringIO()), self.assertRaises(SystemExit) as stopped:
+                review['main']()
+            self.assertEqual(stopped.exception.code, 1)  # Unknown resource counters stay false.
+            return json.loads(output.read_text())
+
+    def test_complete_grader_accepts_supported_receipts_and_equivalent_labels(self):
+        answer, records, reference = self.fixture()
+        self.assertTrue(self.grade(answer, records, reference)['all_factual_checks_pass'])
+        for observation in answer['delivery_observations']:
+            observation['outer_limitations'] = observation.pop('limitations')
+            observation['normalized_coverage'] = observation.pop('coverage')
+        self.assertTrue(self.grade(answer, records, reference)['all_factual_checks_pass'])
+
+    def test_complete_grader_rejects_missing_nulls_bad_scope_and_malformed_identity(self):
+        mutations = [lambda x: x['capture'].pop('capture_start'),
+                     lambda x: x['http_request_uris_first2']['coverage'].__setitem__('exhausted', True),
+                     lambda x: x['delivery_observations'][-1].__setitem__('evidence_id', {'unexpected': 'value'}),
+                     lambda x: x['delivery_observations'][-1].__setitem__('unrecorded_traffic_absent', True)]
+        for mutate in mutations:
+            with self.subTest(mutation=mutate):
+                answer, records, reference = self.fixture();mutate(answer)
+                self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
+
+    def test_faithful_wrong_receipts_do_not_support_the_reported_uri_prefix(self):
+        for wrong in ['facet', 'range', 'parent']:
+            with self.subTest(wrong=wrong):
+                answer, records, reference = self.fixture()
+                page = copy.deepcopy(records[-1]);page['evidence_id'] = 'ev_' + 'e' * 64
+                value = page['normalized_result']
+                if wrong == 'facet':
+                    value['view']['facet'] = 'endpoints'
+                    value['page'] = {'facet': 'endpoints', 'items': [{'address': '127.0.0.1'}]}
+                elif wrong == 'range':
+                    value['view']['offset'] = 1
+                    value['page']['items'] = ['http://example.test/two', 'http://example.test/three']
+                else:
+                    value['parent_evidence_id'] = 'ev_' + 'f' * 64
+                records.append(page)
+                answer['delivery_observations'][-1] = self.observation(page)
+                answer['evidence_ids'][-1] = page['evidence_id']
+                report = self.grade(answer, records, reference)
+                self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix'])
+                self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_complete_receipt_metadata_counts_and_diagnostics_support_reported_capture(self):
+        answer, records, reference = self.fixture()
+        capture = reference['capture']
+        capture.update({'packet_count': 3, 'client': {'name': 'fixture', 'version': '1'},
+                        'retained_counts': {key: 0 for key in ['endpoints', 'dns_queries', 'tls_sni_values',
+                                            'protocol_distribution', 'http_request_uris', 'unattributed_dns_addresses']},
+                        'diagnostics_code_units': {'tshark': 0, 'capinfos': 2}})
+        capture['retained_counts']['http_request_uris'] = 3
+        source = {key: value for key, value in capture.items() if key not in ['retained_counts', 'diagnostics_code_units']}
+        source.update({key: [] for key in capture['retained_counts']})
+        source['http_request_uris'] = reference['http_request_uris_first2'] + ['http://example.test/three']
+        source['diagnostics'] = {'tshark_stderr': '', 'capinfos_stderr': '\U0001f4a5'}
+        receipt = {'evidence_id': 'ev_' + 'a' * 64, 'operation': 'inspect_pcap_capture',
+                   'parameters': {'path': source['target']['path']},
+                   'subject': copy.deepcopy(records[0]['subject']), 'evidence_links': [],
+                   'limitations': ['source'], 'normalized_result': source}
+        reference['delivery_observations'] = [{'evidence_id': receipt['evidence_id'],
+                                              'limitations': receipt['limitations'], 'coverage': source['coverage']}]
+        answer = copy.deepcopy({**reference, 'evidence_ids': [receipt['evidence_id']]})
+        self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass'])
+        for parameters in [None, {}, {'path': '/fixture/other.pcapng'}]:
+            changed = copy.deepcopy(receipt);changed['parameters'] = parameters
+            report = self.grade(answer, [changed], reference)
+            self.assertFalse(report['factual_checks']['pcap_cited_parameters'])
+            self.assertFalse(report['all_factual_checks_pass'])
+        self.assertFalse(self.grade(answer, [receipt], reference,
+                                   producer_arguments=[{'path': '/fixture/other.pcapng'}])['all_factual_checks_pass'])
+        self.assertFalse(self.grade(answer, [receipt], reference, ['inspect_analysis_view'])['all_factual_checks_pass'])
+        wrong_operation = copy.deepcopy(receipt);wrong_operation['operation'] = 'inspect_analysis_view'
+        self.assertFalse(self.grade(answer, [wrong_operation], reference)['all_factual_checks_pass'])
+        for subject in [None, {'digest': {'sha256': 'e' * 64}, 'local_path': source['target']['path']}]:
+            changed = copy.deepcopy(receipt);changed['subject'] = subject
+            report = self.grade(answer, [changed], reference)
+            self.assertTrue(report['factual_checks']['pcap_cited_capture'])
+            self.assertFalse(report['factual_checks']['pcap_cited_subjects_match_artifact'])
+            self.assertFalse(report['all_factual_checks_pass'])
+        mutations = [lambda x: x.__setitem__('packet_count', 4),
+                     lambda x: x['client'].__setitem__('version', '2'),
+                     lambda x: x.__setitem__('capture_start', 'changed'),
+                     lambda x: x.__setitem__('coverage', 'partial'),
+                     lambda x: x['endpoints'].append({'address': '127.0.0.1'}),
+                     lambda x: x['diagnostics'].__setitem__('capinfos_stderr', '')]
+        for mutate in mutations:
+            changed = copy.deepcopy(receipt);mutate(changed['normalized_result'])
+            report = self.grade(answer, [changed], reference)
+            self.assertTrue(report['factual_checks']['capture'])
+            self.assertFalse(report['factual_checks']['pcap_cited_capture'])
+            self.assertFalse(report['all_factual_checks_pass'])
+        receipt['normalized_result'].pop('diagnostics')
+        reference['capture']['diagnostics_code_units'] = None
+        answer['capture']['diagnostics_code_units'] = None
+        self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass'])
+
+    def test_cited_receipt_subjects_must_identify_the_reference_capture(self):
+        for index in [0, 1]:
+            for subject in [None, [], 'artifact', {}, {'digest': None},
+                            {'digest': {'sha256': 'e' * 64}, 'local_path': '/fixture/http.pcapng'},
+                            {'digest': {'sha256': 'd' * 64}, 'local_path': '/fixture/other.pcapng'}]:
+                with self.subTest(index=index, subject=subject):
+                    answer, records, reference = self.fixture()
+                    records[index]['subject'] = subject
+                    report = self.grade(answer, records, reference)
+                    self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                    self.assertFalse(report['factual_checks']['pcap_cited_subjects_match_artifact'])
+                    self.assertFalse(report['all_factual_checks_pass'])
+            answer, records, reference = self.fixture();records[index].pop('subject')
+            self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
+
+    def test_faithful_inconsistent_page_coverage_does_not_support_the_prefix(self):
+        for field, wrong in [('status', 'complete-within-view'), ('next_offset', None), ('exhausted', True)]:
+            with self.subTest(field=field):
+                answer, records, reference = self.fixture()
+                records[-1]['normalized_result']['coverage'][field] = wrong
+                answer['delivery_observations'][-1] = self.observation(records[-1])
+                answer['http_request_uris_first2']['coverage'][field] = wrong
+                report = self.grade(answer, records, reference)
+                self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                self.assertTrue(report['factual_checks']['pcap_prefix_coverage_observed'])
+                self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix'])
+                self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_cited_coverage_preserves_exact_json_scalar_types(self):
+        for field, wrong in [('exhausted', 0), ('examined', 2.0), ('total', 3.0), ('next_offset', 2.0)]:
+            with self.subTest(field=field):
+                answer, records, reference = self.fixture()
+                records[-1]['normalized_result']['coverage'][field] = wrong
+                answer['delivery_observations'][-1] = self.observation(records[-1])
+                answer['http_request_uris_first2']['coverage'][field] = wrong
+                report = self.grade(answer, records, reference)
+                self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                self.assertTrue(report['factual_checks']['pcap_prefix_coverage_observed'])
+                self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_cited_views_preserve_exact_json_scalar_types(self):
+        for field, wrong in [('offset', False), ('offset', 0.0), ('limit', 2.0), ('limit', True)]:
+            with self.subTest(field=field, wrong=wrong):
+                answer, records, reference = self.fixture()
+                records[-1]['normalized_result']['view'][field] = wrong
+                answer['delivery_observations'][-1] = self.observation(records[-1])
+                report = self.grade(answer, records, reference)
+                self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                self.assertFalse(report['factual_checks']['pcap_requested_views'])
+                self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix'])
+                self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_empty_delivery_observations_write_failed_facts(self):
+        answer, records, reference = self.fixture()
+        answer['delivery_observations'] = []
+        answer['evidence_ids'] = []
+        report = self.grade(answer, records, reference)
+        self.assertFalse(report['factual_checks']['pcap_delivery_observations_nonempty'])
+        self.assertFalse(report['factual_checks']['pcap_cited_call_arguments'])
+        self.assertFalse(report['factual_checks']['pcap_cited_parameters'])
+        self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_cited_view_parameters_and_call_arguments_bind_actual_requests(self):
+        for index in [0, 1]:
+            for parameters in [None, {}, {'parent_evidence_id': 'ev_' + 'f' * 64,
+                                          'view': {'kind': 'summary'}}]:
+                with self.subTest(index=index, parameters=parameters):
+                    answer, records, reference = self.fixture()
+                    records[index]['parameters'] = parameters
+                    report = self.grade(answer, records, reference)
+                    self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                    self.assertFalse(report['factual_checks']['pcap_cited_parameters'])
+                    self.assertFalse(report['all_factual_checks_pass'])
+        for wrong in [None, {}, {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'f' * 64},
+                                'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}},
+                      {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'a' * 64},
+                       'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': False, 'limit': 2}}]:
+            with self.subTest(arguments=wrong):
+                answer, records, reference = self.fixture()
+                report = self.grade(answer, records, reference,
+                                    producer_arguments=[{'path': reference['capture']['target']['path'], 'detail': 'summary'}, wrong])
+                self.assertFalse(report['factual_checks']['pcap_cited_call_arguments'])
+                self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_malformed_receipt_envelopes_write_failed_facts_instead_of_crashing(self):
+        tools = ['inspect_pcap_capture', 'inspect_analysis_view']
+        for index in [0, 1]:
+            for field, wrong in [('normalized_result', None), ('normalized_result', []),
+                                 ('normalized_result', 'capture'), ('limitations', None),
+                                 ('limitations', {}), ('limitations', [None]),
+                                 ('limitations', ['source', 1])]:
+                with self.subTest(index=index, field=field, wrong=wrong):
+                    answer, records, reference = self.fixture();records[index][field] = wrong
+                    if field == 'limitations':
+                        answer['delivery_observations'][index]['limitations'] = copy.deepcopy(wrong)
+                    report = self.grade(answer, records, reference, tools)
+                    self.assertTrue(report['factual_checks']['capture'])
+                    self.assertFalse(report['all_factual_checks_pass'])
+            for field in ['normalized_result', 'limitations']:
+                with self.subTest(index=index, missing=field):
+                    answer, records, reference = self.fixture();records[index].pop(field)
+                    self.assertFalse(self.grade(answer, records, reference, tools)['all_factual_checks_pass'])
+        for index, field in [(0, 'summary'), (1, 'page')]:
+            for wrong in [None, [], 'view']:
+                with self.subTest(index=index, nested=field, wrong=wrong):
+                    answer, records, reference = self.fixture();records[index]['normalized_result'][field] = wrong
+                    self.assertFalse(self.grade(answer, records, reference, tools)['all_factual_checks_pass'])
+
+    def test_retained_parent_digest_must_match_its_declared_identity(self):
+        for index in [0, 1]:
+            for digest in [None, {}, [], True, 'digest', 'e' * 64]:
+                with self.subTest(index=index, digest=digest):
+                    answer, records, reference = self.fixture()
+                    records[index]['normalized_result']['parent_digest'] = digest
+                    report = self.grade(answer, records, reference)
+                    self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                    self.assertFalse(report['all_factual_checks_pass'])
+            answer, records, reference = self.fixture()
+            records[index]['normalized_result'].pop('parent_digest')
+            self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
+
+    def test_summary_outer_artifact_must_identify_the_selected_capture(self):
+        for wrong in [None, {}, [], 'artifact',
+                      {'path': '/fixture/other.pcapng', 'sha256': 'd' * 64},
+                      {'path': '/fixture/http.pcapng', 'sha256': 'e' * 64}]:
+            with self.subTest(artifact=wrong):
+                answer, records, reference = self.fixture()
+                records[0]['normalized_result']['artifact'] = wrong
+                report = self.grade(answer, records, reference)
+                self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                self.assertFalse(report['all_factual_checks_pass'])
+        answer, records, reference = self.fixture()
+        records[0]['normalized_result'].pop('artifact')
+        self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
+
+    def test_faithful_summary_coverage_must_describe_the_single_complete_view(self):
+        for field, wrong in [('status', 'page'), ('examined', 0), ('total', 2),
+                             ('next_offset', 0), ('exhausted', False), ('examined', 1.0)]:
+            with self.subTest(field=field, wrong=wrong):
+                answer, records, reference = self.fixture()
+                records[0]['normalized_result']['coverage'][field] = wrong
+                answer['delivery_observations'][0] = self.observation(records[0])
+                report = self.grade(answer, records, reference)
+                self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_present_normalized_provenance_requires_string_arrays(self):
+        for index in [0, 1]:
+            for field in ['limitations', 'unknowns']:
+                for wrong in [None, {}, 'scope', [None], [1]]:
+                    with self.subTest(index=index, field=field, wrong=wrong):
+                        answer, records, reference = self.fixture()
+                        records[index]['normalized_result'][field] = wrong
+                        answer['delivery_observations'][index] = self.observation(records[index])
+                        self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
+                for empty in [True, False]:
+                    answer, records, reference = self.fixture()
+                    key = 'normalized_limitations' if field == 'limitations' else field
+                    if empty:
+                        records[index]['normalized_result'][field] = []
+                        answer['delivery_observations'][index][key] = []
+                    else:
+                        records[index]['normalized_result'].pop(field)
+                        answer['delivery_observations'][index].pop(key)
+                    self.assertTrue(self.grade(answer, records, reference)['all_factual_checks_pass'])
+
+    def test_missing_or_wrong_kind_does_not_hide_malformed_view_containers(self):
+        for index, container in [(0, 'summary'), (1, 'page')]:
+            for kind in [None, 'unexpected']:
+                for wrong in [None, [], 'view']:
+                    with self.subTest(index=index, kind=kind, container=container):
+                        answer, records, reference = self.fixture()
+                        if kind is None:
+                            records[index]['normalized_result'].pop('kind')
+                        else:
+                            records[index]['normalized_result']['kind'] = kind
+                        records[index]['normalized_result'][container] = wrong
+                        report = self.grade(answer, records, reference,
+                                            producer_tools=['inspect_pcap_capture', 'inspect_analysis_view'])
+                        self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_retained_parents_require_declared_distinct_evidence_links(self):
+        for index in [0, 1]:
+            for links in [None, [], ['ev_' + 'f' * 64], {'parent': 'ev_' + 'a' * 64}]:
+                with self.subTest(index=index, links=links):
+                    answer, records, reference = self.fixture();records[index]['evidence_links'] = links
+                    report = self.grade(answer, records, reference)
+                    self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                    self.assertFalse(report['factual_checks']['pcap_cited_parents_linked'])
+                    self.assertFalse(report['all_factual_checks_pass'])
+            answer, records, reference = self.fixture();records[index].pop('evidence_links')
+            self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
+            answer, records, reference = self.fixture()
+            parent = records[index]['evidence_id']
+            for record in records:
+                record['normalized_result']['parent_evidence_id'] = parent;record['evidence_links'] = [parent]
+            answer['delivery_observations'] = [self.observation(record) for record in records]
+            report = self.grade(answer, records, reference)
+            self.assertFalse(report['factual_checks']['pcap_cited_parents_linked'])
+            self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_cited_receipts_preserve_tool_and_delegated_operation_provenance(self):
+        for tools in [['inspect_analysis_view', 'inspect_analysis_view'],
+                      ['inspect_pcap_capture', 'inspect_pcap_capture'],
+                      ['unrelated_tool', 'inspect_analysis_view']]:
+            with self.subTest(tools=tools):
+                answer, records, reference = self.fixture()
+                report = self.grade(answer, records, reference, tools)
+                self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                self.assertFalse(report['all_factual_checks_pass'])
+        for index in [0, 1]:
+            answer, records, reference = self.fixture();records[index]['operation'] = 'inspect_pcap_capture'
+            self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
+
+    def test_failed_reply_shapes_preserve_a_written_report_and_partial_facts(self):
+        payloads = ['malformed', [1], None, False, 7, {'error': 'malformed'},
+                    {'error': []}, {'error': None}]
+        payloads += [{'error': {'code': 'resource_constraint', 'details': value}}
+                     for value in [None, [], 'details']]
+        payloads += [{'error': {'code': 'resource_constraint', 'details': {'reported_limits': value}}}
+                     for value in [None, [], 'limits']]
+        for payload in payloads:
+            with self.subTest(payload=payload):
+                answer, records, reference = self.fixture()
+                result = {'structuredContent': payload}
+                event = {'type': 'item.completed', 'item': {
+                    'type': 'mcp_tool_call', 'tool': 'inspect_analysis_view', 'status': 'failed',
+                    'arguments': {'view': {'kind': 'pcap', 'facet': 'endpoints'}}, 'result': result}}
+                report = self.grade(answer, records, reference, extra_events=[event])
+                self.assertTrue(report['all_factual_checks_pass'])
+                self.assertFalse(report['all_workflow_checks_pass'])
+                self.assertEqual(report['tool_errors'][-1]['response'], payload if payload is not None else result)
+        answer, records, reference = self.fixture()
+        partial_id = 'ev_' + 'e' * 64
+        payload = {'error': {'code': 'resource_constraint', 'details': {
+            'reported_limits': {'evidence_reference': {'evidence_id': partial_id}}}}}
+        event = {'type': 'item.completed', 'item': {
+            'type': 'mcp_tool_call', 'tool': 'inspect_analysis_view', 'status': 'failed',
+            'arguments': {}, 'result': {'structuredContent': payload}}}
+        answer['evidence_ids'].append(partial_id)
+        report = self.grade(answer, records, reference, extra_events=[event])
+        self.assertTrue(report['factual_checks']['reported_evidence_ids_observed_in_responses'])
+        self.assertFalse(report['all_factual_checks_pass'])
+        self.assertFalse(report['all_workflow_checks_pass'])
+
+    def test_workflow_citation_count_must_match_the_planned_calls(self):
+        for count, made_page_call in [(1, False), (1, True), (3, True)]:
+            with self.subTest(count=count, made_page_call=made_page_call):
+                answer, records, reference = self.fixture()
+                if count == 1:
+                    answer['delivery_observations'] = answer['delivery_observations'][:1]
+                    answer['evidence_ids'] = answer['evidence_ids'][:1]
+                else:
+                    extra = copy.deepcopy(answer['delivery_observations'][-1])
+                    extra['evidence_id'] = 'ev_' + 'e' * 64
+                    answer['delivery_observations'].append(extra)
+                    answer['evidence_ids'].append(extra['evidence_id'])
+                report = self.grade(answer, records if made_page_call else records[:1], reference)
+                self.assertFalse(report['all_factual_checks_pass'])
+                self.assertFalse(report['all_workflow_checks_pass'])
+
+    def test_unreadable_planned_reply_does_not_pass_workflow_acceptance(self):
+        answer, records, reference = self.fixture()
+        report = self.grade(answer, records, reference,
+                            producer_results=[{'structuredContent': records[0]}, {}])
+        self.assertFalse(report['all_factual_checks_pass'])
+        self.assertFalse(report['all_workflow_checks_pass'])
+
+    def test_extra_calls_are_not_an_equivalent_workflow_even_with_correct_facts(self):
+        for status in ['completed', 'failed']:
+            for with_receipt in [True, False]:
+                with self.subTest(status=status, with_receipt=with_receipt):
+                    answer, records, reference = self.fixture()
+                    extra = copy.deepcopy(records[-1]);extra['evidence_id'] = 'ev_' + 'e' * 64
+                    extra['normalized_result']['view']['facet'] = 'endpoints'
+                    event = {'type': 'item.completed', 'item': {
+                        'type': 'mcp_tool_call', 'tool': 'inspect_analysis_view', 'status': status,
+                        'arguments': {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'a' * 64},
+                                      'view': extra['normalized_result']['view']},
+                        'result': {'structuredContent': extra} if with_receipt else {}}}
+                    report = self.grade(answer, records, reference, extra_events=[event])
+                    self.assertTrue(report['all_factual_checks_pass'])
+                    self.assertFalse(report.get('all_workflow_checks_pass', True))
+        answer, records, reference = self.fixture()
+        report = self.grade(answer, records[::-1], reference)
+        self.assertTrue(report['all_factual_checks_pass'])
+        self.assertFalse(report.get('all_workflow_checks_pass', True))
+
+    def test_repeated_identical_receipts_pass_but_conflicting_id_ownership_fails(self):
+        answer, records, reference = self.fixture();duplicate = copy.deepcopy(records[0])
+        report = self.grade(answer, [duplicate, *records], reference)
+        self.assertTrue(report['all_factual_checks_pass'])
+        self.assertFalse(report['all_workflow_checks_pass'])
+        for duplicate_first in [True, False]:
+            with self.subTest(duplicate_first=duplicate_first):
+                items = [duplicate, *records] if duplicate_first else [*records, duplicate]
+                tools = ['unrelated_tool', 'inspect_pcap_capture', 'inspect_analysis_view'] if duplicate_first else \
+                        ['inspect_pcap_capture', 'inspect_analysis_view', 'unrelated_tool']
+                self.assertFalse(self.grade(answer, items, reference, tools)['all_factual_checks_pass'])
+        duplicate = copy.deepcopy(records[-1]);duplicate['normalized_result']['coverage']['exhausted'] = 0
+        for items in [[*records, duplicate], [records[0], duplicate, records[-1]]]:
+            self.assertFalse(self.grade(answer, items, reference)['all_factual_checks_pass'])
+
+    def test_reported_ids_are_only_the_ordered_unique_response_receipts(self):
+        answer, records, reference = self.fixture()
+        original = answer['evidence_ids']
+        parent = records[0]['normalized_result']['parent_evidence_id']
+        for ids in [original + [parent], original + [original[-1]], original[::-1],
+                    [parent, *original], [original], {'receipts': original},
+                    [*original, {'extra': parent}], original + ['not-an-id']]:
+            with self.subTest(ids=ids):
+                changed = copy.deepcopy(answer);changed['evidence_ids'] = ids
+                report = self.grade(changed, records, reference)
+                self.assertFalse(report['factual_checks']['pcap_observations_have_reported_ids'])
+                self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_prefix_coverage_must_come_from_the_cited_uri_receipt(self):
+        answer, records, reference = self.fixture()
+        extra = copy.deepcopy(records[-1]);extra['evidence_id'] = 'ev_' + 'e' * 64
+        extra['normalized_result']['coverage']['next_offset'] = None
+        extra['normalized_result']['coverage']['exhausted'] = True
+        records.append(extra)
+        answer['http_request_uris_first2']['coverage'] = extra['normalized_result']['coverage']
+        report = self.grade(answer, records, reference)
+        self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+        self.assertTrue(report['factual_checks']['pcap_cited_source_supports_uri_prefix'])
+        self.assertFalse(report['factual_checks']['pcap_prefix_coverage_observed'])
+        self.assertFalse(report['all_factual_checks_pass'])
+
+
 class ContextCaptureTests(unittest.TestCase):
     def test_filters_ambient_account_and_nested_usage_state(self):
         with tempfile.TemporaryDirectory() as directory:

scripts/verify-windows-native-inspection.mjs

@@ -4,8 +4,8 @@ import {
   copyFile,
   mkdir,
   mkdtemp,
+  readFile,
   readdir,
-  rmdir,
   rm,
   stat,
   symlink,
@@ -54,7 +54,7 @@ try {
     join(workspace, "dist", "windows", "WindowsNativeLoader.js"),
   ).href;
   const source = `import assert from 'node:assert/strict';
-import {existsSync,mkdirSync,readdirSync,renameSync,statSync} from 'node:fs';
+import {existsSync,mkdirSync,readFileSync,readdirSync,renameSync,statSync,writeFileSync} from 'node:fs';
 import {join} from 'node:path';
 import {systemWindowsNativeAuthority} from ${JSON.stringify(loader)};
 const temp=process.env.TEMP;
@@ -64,17 +64,18 @@ assert.equal(first.retryable,true);
 assert.match(first.reason,/Delete owned object by handle failed.*Win32 5/s);
 const [name,...extra]=readdirSync(temp);assert.ok(name);assert.equal(extra.length,0);
 const path=join(temp,name);const identity=statSync(path,{bigint:true});
-// Ownership follows the retained NTFS handle even when the directory moves.
-// A replacement at its old pathname must remain caller-owned.
-const moved=path+'-renamed';renameSync(path,moved);
-assert.equal(statSync(moved,{bigint:true}).ino,identity.ino);
+const retainedPath=path+'-retained';renameSync(path,retainedPath);
+assert.equal(statSync(retainedPath,{bigint:true}).ino,identity.ino);
+// FILE_SHARE_DELETE permits rename; ownership follows the retained handle.
 mkdirSync(path);const replacement=statSync(path,{bigint:true});
 assert.notEqual(replacement.ino,identity.ino);
+const marker=join(path,'caller-evidence.txt');writeFileSync(marker,'caller-owned replacement');
 if(process.argv[2]!=='teardown'){
  const second=systemWindowsNativeAuthority();assert.equal(second.available,true);
- assert.equal(existsSync(moved),false);
+ assert.equal(existsSync(retainedPath),false);
  assert.deepEqual(readdirSync(temp),[name]);
  assert.equal(statSync(path,{bigint:true}).ino,replacement.ino);
+ assert.equal(readFileSync(marker,'utf8'),'caller-owned replacement');
  assert.equal(systemWindowsNativeAuthority(),second);
 }
 console.log(JSON.stringify({mode:process.argv[2],firstFailed:true,retainedOwner:true,inode:identity.ino.toString(),replacementName:name,replacementInode:replacement.ino.toString()}));
@@ -99,11 +100,13 @@ console.log(JSON.stringify({mode:process.argv[2],firstFailed:true,retainedOwner:
       (await stat(replacement, { bigint: true })).ino.toString(),
       observed.replacementInode,
     );
-    await rmdir(replacement);
-    assert.deepEqual(await readdir(temp), []);
+    assert.equal(
+      await readFile(join(replacement, "caller-evidence.txt"), "utf8"),
+      "caller-owned replacement",
+    );
   }
   process.stdout.write(
-    `${JSON.stringify({ ok: true, controlledDispositionFailure: true, retainedOwner: true, renamedOwnerCleanup: true, replacementPreserved: true, retryRecovers: true, successfulAdmissionCached: true, environmentTeardownCleansProbe: true, issue1701HostCauseReproduced: false })}\n`,
+    `${JSON.stringify({ ok: true, controlledDispositionFailure: true, retainedOwner: true, renamePreservesIdentity: true, foreignReplacementPreserved: true, retryRecovers: true, successfulAdmissionCached: true, environmentTeardownCleansProbe: true, issue1701HostCauseReproduced: false })}\n`,
   );
 } finally {
   await rm(workspace, { recursive: true, force: true });

tests/evaluation/analysisEfficiencyComparison.test.ts

@@ -0,0 +1,199 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "vitest";
+import { z } from "zod";
+import { fileURLToPath } from "node:url";
+
+type Delivery = "complete" | "summary";
+type Observation = {
+  evidence_id: string;
+  parent_evidence_id?: string;
+  limitations: string[];
+  kind?: string;
+  view?: { kind: string; facet?: string; offset?: number; limit?: number };
+  coverage?:
+    | string
+    | {
+        examined: number;
+        total: number;
+        next_offset: number | null;
+        exhausted: boolean;
+      };
+  unknowns?: string[];
+};
+type Fixture = {
+  summary: {
+    artifact: string;
+    delivery: Delivery;
+    provider_search_path_sha256: string;
+    calls: unknown[];
+  };
+  answer: {
+    capture: { coverage: string; capture_start: null };
+    http_request_uris_first2: string[];
+    delivery_observations: Observation[];
+  };
+};
+const reportSchema = z.object({
+  facts_match: z.boolean(),
+  settings_match: z.boolean(),
+  fact_differences: z.array(z.string()),
+  preserved_run_specific_values: z.array(z.object({ value: z.unknown() })),
+});
+
+// Synthetic measurement directories exercise the comparator CLI, not providers.
+const observation = (kind: string): Observation => ({
+  evidence_id: `ev_${kind}`,
+  parent_evidence_id: "ev_parent",
+  limitations: ["saved bytes only"],
+  kind,
+  view:
+    kind === "summary"
+      ? { kind }
+      : { kind, facet: "http-uris", offset: 0, limit: 2 },
+  coverage: { examined: 2, total: 3, next_offset: 2, exhausted: false },
+  unknowns: ["unrecorded traffic"],
+});
+const fixture = (delivery: Delivery, artifact: string): Fixture => ({
+  summary: {
+    artifact,
+    delivery,
+    provider_search_path_sha256: "a".repeat(64),
+    calls: [],
+  },
+  answer: {
+    capture: { coverage: "complete", capture_start: null },
+    http_request_uris_first2: [
+      "http://example.test/one",
+      "http://example.test/two",
+    ],
+    delivery_observations:
+      delivery === "summary"
+        ? [observation("summary"), observation("pcap")]
+        : [
+            {
+              evidence_id: "ev_complete",
+              limitations: ["saved bytes only"],
+              coverage: "complete",
+            },
+          ],
+  },
+});
+function compare(left: Fixture, right: Fixture) {
+  const root = mkdtempSync(join(tmpdir(), "rea-compare-test-"));
+  try {
+    const directories = [join(root, "left"), join(root, "right")] as const;
+    for (const [index, value] of [left, right].entries()) {
+      const directory = directories[index];
+      assert.ok(directory);
+      mkdirSync(directory);
+      for (const name of ["summary", "answer"] as const)
+        writeFileSync(
+          join(directory, `${name}.json`),
+          JSON.stringify(value[name]),
+        );
+    }
+    const result = spawnSync(
+      process.execPath,
+      [
+        fileURLToPath(
+          new URL(
+            "../../experiments/analysis-efficiency/compare.mjs",
+            import.meta.url,
+          ),
+        ),
+        "--left",
+        directories[0],
+        "--right",
+        directories[1],
+      ],
+      { encoding: "utf8" },
+    );
+    assert.ifError(result.error);
+    assert.equal(result.stderr, "");
+    return {
+      status: result.status,
+      report: reportSchema.parse(JSON.parse(result.stdout)),
+    };
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+
+for (const artifact of [
+  "wireshark-http.cap",
+  "wireshark-http_redirects.pcapng",
+]) {
+  test(`${artifact}: repeated modes ignore only receipt IDs`, () => {
+    for (const delivery of ["complete", "summary"] as const) {
+      const left = fixture(delivery, artifact);
+      const right = structuredClone(left);
+      for (const value of right.answer.delivery_observations) {
+        value.evidence_id += "_second";
+        if (typeof value.parent_evidence_id === "string")
+          value.parent_evidence_id += "_second";
+      }
+      const { status, report } = compare(left, right);
+      assert.equal(status, 0);
+      assert.equal(report.facts_match, true);
+      assert.deepEqual(
+        report.preserved_run_specific_values.map((value) => value.value),
+        [left.answer.delivery_observations, right.answer.delivery_observations],
+      );
+    }
+  });
+
+  test(`${artifact}: repeated modes reject stable observation changes`, () => {
+    for (const delivery of ["complete", "summary"] as const) {
+      const left = fixture(delivery, artifact);
+      const mutations: ((values: Observation[]) => void)[] = [
+        (values) => values[0]!.limitations.push("new limitation"),
+        (values) => {
+          values[0]!.coverage = "partial";
+        },
+        (values) => {
+          delete values[0]!.coverage;
+        },
+      ];
+      if (delivery === "summary")
+        mutations.push(
+          (values) => {
+            values[1]!.view!.offset = 1;
+          },
+          (values) => {
+            const coverage = values[1]!.coverage;
+            assert.ok(typeof coverage === "object" && coverage !== null);
+            coverage.next_offset = null;
+          },
+          (values) => values[1]!.unknowns!.push("extra unknown"),
+          (values) => values.reverse(),
+        );
+      for (const mutate of mutations) {
+        const right = structuredClone(left);
+        mutate(right.answer.delivery_observations);
+        const { status, report } = compare(left, right);
+        assert.equal(status, 1);
+        assert.equal(report.settings_match, true);
+        assert.equal(report.facts_match, false);
+        assert.deepEqual(report.fact_differences, ["delivery_observations"]);
+      }
+    }
+  });
+
+  test(`${artifact}: different modes preserve their different receipt scopes`, () => {
+    const left = fixture("complete", artifact);
+    const right = fixture("summary", artifact);
+    const { status, report } = compare(left, right);
+    assert.equal(status, 0);
+    assert.equal(report.facts_match, true);
+    assert.deepEqual(
+      report.preserved_run_specific_values.map((value) => value.value),
+      [left.answer.delivery_observations, right.answer.delivery_observations],
+    );
+    right.answer.capture.coverage = "partial";
+    assert.equal(compare(left, right).report.facts_match, false);
+  });
+}