Merge branch 'main' into codex/review-2049-27f1b367

* fix(comparison): preserve incomplete function call evidence

Keep missing, unknown, partial and unresolved call relationships unknown in function comparisons and reconstruction claims. Aggregate IDA caller and callee coverage without overwriting uncertainty.

Validate with 144 focused tests, typecheck/lint, docs checks and test discovery. Real providers were not verified; the full headless IDA session suite requires unavailable Windows native controls.

* fix(comparison): recheck historical calls-only changes

---------

Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>

bridge/ghidra/ReaGhidraBridge.java

@@ -3632,13 +3632,17 @@ public final class ReaGhidraBridge extends HeadlessScript {
      * The canonical empty string is a small literal with a zero count.
      */
     private SwiftStringLiteralDecode swiftStringLiterals(Function function) throws Exception {
+        return swiftStringLiterals(function, new SwiftLiteralBudget());
+    }
+
+    // The budget is owned by one decode run; an explicit budget also permits deterministic producer regressions.
+    private SwiftStringLiteralDecode swiftStringLiterals(Function function, SwiftLiteralBudget budget) throws Exception {
         Register[] registers = new Register[31];
         for (int index = 0; index < registers.length; index++) {
             registers[index] = currentProgram.getRegister("x" + index);
         }
         TreeMap<String, JsonObject> observed = new TreeMap<>();
         TreeMap<String, JsonObject> omitted = new TreeMap<>();
-        SwiftLiteralBudget budget = new SwiftLiteralBudget();
         Map<String, Boolean> selectedValues = new HashMap<>();
         SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false);
         propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() {
@@ -3883,6 +3887,11 @@ public final class ReaGhidraBridge extends HeadlessScript {
             remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2);
         }
 
+        SwiftLiteralBudget(long remaining) {
+            if (remaining < 0) throw new IllegalArgumentException("Swift literal budget must be nonnegative");
+            this.remaining = remaining;
+        }
+
         boolean takeState() {
             if (remaining < BYTES_PER_RECORD) return false;
             remaining -= BYTES_PER_RECORD;

scripts/verify-real-ghidra-swift.mjs

@@ -1,7 +1,14 @@
 #!/usr/bin/env node
 import assert from "node:assert/strict";
 import { execFile } from "node:child_process";
-import { copyFile, mkdir, mkdtemp, rm } from "node:fs/promises";
+import {
+  copyFile,
+  mkdir,
+  mkdtemp,
+  readFile,
+  readdir,
+  rm,
+} from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -9,6 +16,11 @@ import { promisify } from "node:util";
 import { Client } from "@modelcontextprotocol/client";
 import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
 import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js";
+import { cleanupOwnedProcessGroup } from "../dist/process/ProcessOwnership.js";
+import {
+  ProviderProcessSupervisor,
+  spawnOwnedProviderProcess,
+} from "../dist/process/ProviderProcess.js";
 import { toolContract } from "../dist/contracts/toolContracts.js";
 import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
 import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
@@ -50,8 +62,13 @@ const literalSource = fileURLToPath(
 );
 const binary = join(root, "swift-inventory");
 const literalBinary = join(root, "swift-literals");
+const literalAssembly = fileURLToPath(
+  new URL("../tests/conformance/swift/decoder-safeguards.S", import.meta.url),
+);
+const literalObject = join(root, "decoder-safeguards.o");
 const reports = [];
 let literalReport;
+let decoderCleaned = true;
 try {
   await execute("swiftc", ["-O", "-g", source, "-o", binary], {
     timeout: 60_000,
@@ -59,6 +76,19 @@ try {
   await mkdir(join(root, "copied"));
   const copied = join(root, "copied", "swift-inventory");
   await copyFile(binary, copied);
+  await execute(
+    "xcrun",
+    [
+      "clang",
+      "-target",
+      "arm64-apple-macosx13.0",
+      "-c",
+      literalAssembly,
+      "-o",
+      literalObject,
+    ],
+    { timeout: 60_000 },
+  );
   // Decoding covers AArch64 register pairs, so build that slice on any Mac host.
   await execute(
     "swiftc",
@@ -67,11 +97,13 @@ try {
       "-target",
       "arm64-apple-macosx13.0",
       literalSource,
+      literalObject,
       "-o",
       literalBinary,
     ],
     { timeout: 60_000 },
   );
+  await verifyDecoderSafeguards();
   await client.connect(transport);
   for (const [variant, path] of [
     ["debug", binary],
@@ -154,7 +186,7 @@ try {
 } finally {
   await client.close();
   await transport.close();
-  await rm(root, { recursive: true, force: true });
+  if (decoderCleaned) await rm(root, { recursive: true, force: true });
 }
 console.log(
   JSON.stringify(
@@ -227,3 +259,113 @@ async function verifyStringLiterals() {
     await call("close_binary", {});
   }
 }
+
+// Exercise the actual Java producer inside a disposable Ghidra database. The
+// explicit private budget boundary makes exhaustion deterministic, without a
+// production CLI/MCP option or dependence on the verification host's free heap.
+async function verifyDecoderSafeguards() {
+  const scripts = join(root, "decoder-scripts");
+  await mkdir(scripts);
+  const bridgeSources = fileURLToPath(
+    new URL("../bridge/ghidra/", import.meta.url),
+  );
+  for (const name of await readdir(bridgeSources)) {
+    if (name.endsWith(".java"))
+      await copyFile(join(bridgeSources, name), join(scripts, name));
+  }
+  await copyFile(
+    fileURLToPath(
+      new URL(
+        "../tests/conformance/swift/ReaSwiftDecoderRegressions.java",
+        import.meta.url,
+      ),
+    ),
+    join(scripts, "ReaSwiftDecoderRegressions.java"),
+  );
+  const report = join(root, "decoder-safeguards.txt");
+  await runDecoderProbe(scripts, report);
+  assert.equal(
+    await readFile(report, "utf8"),
+    "REA_SWIFT_DECODER_SAFEGUARDS_OK\n",
+  );
+}
+
+async function runDecoderProbe(scripts, report) {
+  const controller = new AbortController();
+  let interrupt;
+  const interrupted = new Promise((resolve) => {
+    interrupt = () => {
+      controller.abort(new Error("Swift decoder probe interrupted"));
+      resolve(false);
+    };
+  });
+  process.on("SIGINT", interrupt);
+  process.on("SIGTERM", interrupt);
+  let supervisor;
+  try {
+    const launch = await spawnOwnedProviderProcess({
+      command: installation.analyzeHeadlessPath,
+      arguments: [
+        root,
+        "decoder-regressions",
+        "-import",
+        literalBinary,
+        "-scriptPath",
+        scripts,
+        "-postScript",
+        "ReaSwiftDecoderRegressions.java",
+        report,
+        "-deleteProject",
+        "-max-cpu",
+        "2",
+      ],
+      runId: run.run_id,
+      expectedCommand: null,
+      env: { JAVA_HOME: installation.javaHome },
+      signal: controller.signal,
+    });
+    decoderCleaned = false;
+    supervisor = new ProviderProcessSupervisor(
+      {
+        ...launch,
+        ownsProcessLifetime: true,
+        cleanup: () => cleanupOwnedProcessGroup(launch.ownership),
+      },
+      { maxDiagnosticBytes: 8 * 1024 * 1024 },
+    );
+    assert.ok(
+      await Promise.race([supervisor.waitForExit(120_000), interrupted]),
+      controller.signal.aborted
+        ? "Swift decoder probe interrupted"
+        : "Swift decoder probe timed out",
+    );
+    assert.ok(
+      await supervisor.waitForOutputClose(1_000),
+      "Swift decoder probe output did not close",
+    );
+    const snapshot = supervisor.snapshot();
+    assert.equal(
+      snapshot.exitCode,
+      0,
+      `${snapshot.stdout.text}\n${snapshot.stderr.text}`,
+    );
+    assert.ok(
+      !snapshot.diagnosticTruncated,
+      "Swift decoder probe exceeded its diagnostic capacity",
+    );
+  } finally {
+    try {
+      if (supervisor !== undefined) {
+        const stopped = await supervisor.stop();
+        decoderCleaned = stopped.status === "verified-cleanup";
+        assert.ok(
+          decoderCleaned,
+          `Owned Ghidra cleanup incomplete; retained ${root}: ${JSON.stringify(stopped)}`,
+        );
+      }
+    } finally {
+      process.off("SIGINT", interrupt);
+      process.off("SIGTERM", interrupt);
+    }
+  }
+}

tests/conformance/swift/ReaSwiftDecoderRegressions.java

@@ -0,0 +1,152 @@
+// Producer regressions run inside a real, ephemeral Ghidra analysis of the
+// source-owned AArch64 fixture. No target code is executed and no decoded JSON
+// fixture substitutes for the Java decoder under test.
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import ghidra.app.util.headless.HeadlessScript;
+import ghidra.program.model.address.Address;
+import ghidra.program.model.listing.Function;
+import ghidra.program.model.listing.FunctionIterator;
+import ghidra.program.model.listing.Instruction;
+import ghidra.program.model.listing.InstructionIterator;
+import ghidra.program.model.lang.Register;
+import ghidra.program.model.symbol.Symbol;
+import ghidra.program.model.symbol.SymbolIterator;
+
+public class ReaSwiftDecoderRegressions extends HeadlessScript {
+    private ReaGhidraBridge decoder;
+    private Class<?> budgetClass;
+
+    @Override
+    public void run() throws Exception {
+        decoder = new ReaGhidraBridge();
+        setInherited(decoder, "currentProgram", currentProgram);
+        setInherited(decoder, "monitor", monitor);
+        setInherited(decoder, "sessionDefaultAddressSpace", currentProgram.getAddressFactory().getDefaultAddressSpace());
+        budgetClass = Class.forName("ReaGhidraBridge$SwiftLiteralBudget", true, decoder.getClass().getClassLoader());
+        Function copied = function("rea_swift_decoder_copied");
+        JsonArray copiedValues = literals(decode(copied, 65536));
+        require(copiedValues.size() > 0, "copied constant pair was not decoded");
+        for (var element : copiedValues) {
+            require(element.getAsJsonObject().get("value").getAsString().equals("ok"), "copied words invented text");
+        }
+        for (String name : new String[] {"select", "selected_copy", "padding", "utf8"}) {
+            Object result = decode(function("rea_swift_decoder_" + name), 65536);
+            require(literals(result).isEmpty(), name + " produced an invented Swift literal");
+            require(omissions(result).isEmpty(), name + " was confused with capacity exhaustion");
+        }
+        for (String name : new String[] {"select", "selected_copy"}) {
+            Function selected = function("rea_swift_decoder_" + name);
+            require((Boolean) invoke("selectedValue", new Class<?>[] {Function.class, Instruction.class, Register.class}, selected, returnInstruction(selected), currentProgram.getRegister("x0")),
+                name + " did not retain conditional-select provenance");
+        }
+        require(invoke("swiftLiteralLayout", new Class<?>[] {long.class, long.class}, 0x6261L, 0xe100000000000000L) == null,
+            "nonzero small-string padding was accepted");
+        require(invoke("swiftLiteralLayout", new Class<?>[] {long.class, long.class}, 0xc0L, 0xa100000000000000L) == null,
+            "malformed small-string UTF-8 was accepted");
+        rejectLarge("rea_swift_decoder_bad_terminator", 0xd000000000000003L);
+        rejectLarge("rea_swift_decoder_bad_large_utf8", 0x1000000000000001L);
+        // Zero covers provenance-state exhaustion; 8192 admits both register
+        // cache entries and then rejects the retained literal/text charge.
+        assertCapacityOmission(copied, 0);
+        assertCapacityOmission(copied, 8192);
+        require(getScriptArgs().length == 1, "Expected owned result path");
+        Files.writeString(Path.of(getScriptArgs()[0]), "REA_SWIFT_DECODER_SAFEGUARDS_OK\n");
+    }
+
+    private void assertCapacityOmission(Function function, long remaining) throws Exception {
+        Object exhausted = decode(function, remaining);
+        require(literals(exhausted).isEmpty(), "capacity failure published decoded small text at budget " + remaining);
+        JsonArray omitted = omissions(exhausted);
+        require(omitted.size() == 1, "capacity failure did not stop subsequent use sites at budget " + remaining);
+        JsonObject omission = omitted.get(0).getAsJsonObject();
+        require(omission.get("reason").getAsString().equals("capacity"), "capacity omission lost its reason");
+        require(!omission.has("value"), "omission claimed a retained literal value");
+    }
+
+    private Object decode(Function function, long remaining) throws Exception {
+        Constructor<?> constructor = budgetClass.getDeclaredConstructor(long.class);
+        constructor.setAccessible(true);
+        return invoke("swiftStringLiterals", new Class<?>[] {Function.class, budgetClass}, function, constructor.newInstance(remaining));
+    }
+
+    private void rejectLarge(String name, long word) throws Exception {
+        Address address = symbol(name).getAddress();
+        long object = 0x8000000000000000L | ((address.getOffset() - 0x20) & 0x00ffffffffffffffL);
+        Object layout = invoke("swiftLiteralLayout", new Class<?>[] {long.class, long.class}, word, object);
+        require(layout != null, "large negative fixture did not reach the memory-text decoder");
+        require(invoke("swiftLargeLiteralText", new Class<?>[] {layout.getClass()}, layout) == null,
+            name + " was accepted as literal text");
+    }
+
+    private Instruction returnInstruction(Function function) {
+        InstructionIterator instructions = currentProgram.getListing().getInstructions(function.getBody(), true);
+        while (instructions.hasNext()) {
+            Instruction instruction = instructions.next();
+            if (instruction.getMnemonicString().equalsIgnoreCase("ret")) return instruction;
+        }
+        throw new AssertionError("Fixture has no return instruction");
+    }
+
+    private Function function(String name) {
+        FunctionIterator functions = currentProgram.getFunctionManager().getFunctions(true);
+        Function found = null;
+        while (functions.hasNext()) {
+            Function candidate = functions.next();
+            if (!candidate.getName().equals(name) && !candidate.getName().equals("_" + name)) continue;
+            require(found == null, "ambiguous fixture function " + name);
+            found = candidate;
+        }
+        require(found != null, "missing fixture function " + name);
+        return found;
+    }
+
+    private Symbol symbol(String name) {
+        SymbolIterator symbols = currentProgram.getSymbolTable().getAllSymbols(true);
+        Symbol found = null;
+        while (symbols.hasNext()) {
+            Symbol candidate = symbols.next();
+            if (!candidate.getName().equals(name) && !candidate.getName().equals("_" + name)) continue;
+            require(found == null, "ambiguous fixture symbol " + name);
+            found = candidate;
+        }
+        require(found != null, "missing fixture symbol " + name);
+        return found;
+    }
+
+    private Object invoke(String name, Class<?>[] types, Object... arguments) throws Exception {
+        Method method = ReaGhidraBridge.class.getDeclaredMethod(name, types);
+        method.setAccessible(true);
+        return method.invoke(decoder, arguments);
+    }
+
+    private JsonArray literals(Object result) throws Exception { return array(result, "literals"); }
+    private JsonArray omissions(Object result) throws Exception { return array(result, "omissions"); }
+    private JsonArray array(Object result, String name) throws Exception {
+        Method method = result.getClass().getDeclaredMethod(name);
+        method.setAccessible(true);
+        return (JsonArray) method.invoke(result);
+    }
+
+    private void setInherited(Object target, String name, Object value) throws Exception {
+        for (Class<?> type = target.getClass(); type != null; type = type.getSuperclass()) {
+            try {
+                Field field = type.getDeclaredField(name);
+                field.setAccessible(true);
+                field.set(target, value);
+                return;
+            } catch (NoSuchFieldException nextOwner) { }
+        }
+        throw new AssertionError("Missing Ghidra state field " + name);
+    }
+
+    private static void require(boolean condition, String message) {
+        if (!condition) throw new AssertionError(message);
+    }
+}

tests/conformance/swift/decoder-safeguards.S

@@ -0,0 +1,58 @@
+/* Static-analysis fixture only. Never executed. Deliberate malformed ABI words
+ * cannot be produced by a well-formed Swift source literal. */
+.text
+.p2align 2
+.globl _rea_swift_decoder_copied
+_rea_swift_decoder_copied:
+    mov x5, #0x6b6f
+    movz x6, #0xe200, lsl #48
+    mov x0, x5
+    mov x1, x6
+    stp x0, x1, [sp, #-16]!
+    ldp x0, x1, [sp], #16
+    ret
+
+.p2align 2
+.globl _rea_swift_decoder_select
+_rea_swift_decoder_select:
+    mov x2, #0x61
+    mov x3, #0x6262
+    movz x6, #0xe100, lsl #48
+    movz x7, #0xe200, lsl #48
+    cmp w4, #0
+    csel x0, x2, x3, eq
+    csel x1, x6, x7, eq
+    ret
+
+.p2align 2
+.globl _rea_swift_decoder_selected_copy
+_rea_swift_decoder_selected_copy:
+    mov x2, #0x61
+    mov x3, #0x62
+    cmp w4, #0
+    csel x5, x2, x3, eq
+    mov x0, x5
+    movz x1, #0xe100, lsl #48
+    ret
+
+.p2align 2
+.globl _rea_swift_decoder_padding
+_rea_swift_decoder_padding:
+    mov x0, #0x6261
+    movz x1, #0xe100, lsl #48
+    ret
+
+.p2align 2
+.globl _rea_swift_decoder_utf8
+_rea_swift_decoder_utf8:
+    mov x0, #0xc0
+    movz x1, #0xa100, lsl #48
+    ret
+
+.section __DATA,__const
+.globl _rea_swift_decoder_bad_terminator
+_rea_swift_decoder_bad_terminator:
+    .byte 0x61, 0x62, 0x63, 0x78
+.globl _rea_swift_decoder_bad_large_utf8
+_rea_swift_decoder_bad_large_utf8:
+    .byte 0xc0, 0