From 560257b00e92188a7fabded623fefd8655bc761a Mon Sep 17 00:00:00 2001 From: Omisakin Joshua <117924167+thefolahan@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:40:02 +0100 Subject: [PATCH] fix(ghidra): bound literal records and trace selects across branches Every Swift literal record now pays a fixed cost against the dossier budget in addition to its text, so many empty or small literals are bounded as well. The first record that does not fit is kept as an omission and decoding stops there, which keeps the omission record itself bounded; omissions now describe small or large literals. The conditional select check searches every predecessor path for the reaching definitions, following register copies, so selects made before a branch into the use block are recognized. A dossier records Swift literals and their omissions together. Coverage reads omissions first, and strings_names compares every use site that neither side omitted: a decoded literal is known where it was recorded, and an absence is known only in a dossier whose decoding examined every use site. --- bridge/ghidra/ReaGhidraBridge.java | 92 +++++++++++++------ src/domain/functionComparison.test.ts | 86 ++++++++++++----- src/domain/functionComparisonDimensions.ts | 8 +- src/domain/functionComparisonNormalization.ts | 34 +++++-- src/domain/hopperValues.ts | 28 ++++-- src/ghidra/GhidraFunctionValues.test.ts | 7 ++ src/ghidra/GhidraFunctionValues.ts | 12 ++- 7 files changed, 193 insertions(+), 74 deletions(-) diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index f780f89d6..17de12f50 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -24,6 +24,7 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermissions; import java.security.MessageDigest; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -3356,22 +3357,21 @@ public final class ReaGhidraBridge extends HeadlessScript { SwiftLiteralLayout layout = swiftLiteralLayout(word, object); // Ghidra forks each select without correlating its condition, so a pair whose // registers are both selected may combine words from different cases. - if (layout == null || (selectedValue(instruction, pair[0]) && selectedValue(instruction, pair[1]))) continue; + if (layout == null || + (selectedValue(function, instruction, pair[0]) && selectedValue(function, instruction, pair[1]))) continue; // Forks can reach one site with different constants, so the words are part of the key. String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() + "\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) + "\u0000" + Long.toUnsignedString(object, 16); if (observed.containsKey(key) || omitted.containsKey(key)) continue; - // Small text is at most 15 bytes and already decoded; only large reads draw on the budget. + // Every record draws on the budget; the first one that does not fit is kept as an + // omission and decoding stops, so later sites are never examined. String value = layout.smallValue(); - boolean overCapacity = false; - if (value == null && !budget.take(layout.byteLength())) { - overCapacity = true; - } else if (value == null) { + boolean overCapacity = !budget.take(layout.byteLength()); + if (!overCapacity && value == null) { try { value = swiftLargeLiteralText(layout); } catch (OutOfMemoryError exhausted) { - budget.exhaust(); overCapacity = true; } } @@ -3385,6 +3385,7 @@ public final class ReaGhidraBridge extends HeadlessScript { pairRegisters.add(pair[1].getName()); item.add("registers", pairRegisters); (value == null ? omitted : observed).put(key, item); + if (overCapacity) return true; } return false; } @@ -3435,32 +3436,59 @@ public final class ReaGhidraBridge extends HeadlessScript { return value != null && context.isConstant(value) ? value.getOffset() : null; } - // Follow register copies back to the defining instruction, since a select copied into an - // argument register forks just as independently as one written there directly. - private boolean selectedValue(Instruction use, Register register) { - Register target = register.getBaseRegister(); - Address from = use.getFallFrom(); - Instruction instruction = from == null ? null : currentProgram.getListing().getInstructionAt(from); - while (instruction != null) { + // Search every predecessor path for the reaching definitions, following register copies, + // since a select copied into an argument register or defined before a branch forks just as + // independently as one written there directly. + private boolean selectedValue(Function function, Instruction use, Register register) { + ArrayDeque instructions = new ArrayDeque<>(); + ArrayDeque targets = new ArrayDeque<>(); + Set visited = new HashSet<>(); + for (Instruction predecessor : predecessors(function, use)) { + instructions.push(predecessor); + targets.push(register.getBaseRegister()); + } + while (!instructions.isEmpty()) { + Instruction instruction = instructions.pop(); + Register target = targets.pop(); + if (!visited.add(canonicalAddress(instruction.getAddress()) + "\u0000" + target.getName())) continue; boolean defines = false; for (Object result : instruction.getResultObjects()) { if (result instanceof Register written && written.getBaseRegister().equals(target)) defines = true; } + Register next = target; if (defines) { PcodeOp[] operations = instruction.getPcode(); for (PcodeOp operation : operations) { if (operation.getOpcode() == PcodeOp.CBRANCH && operation.getInput(0).isConstant()) return true; } Register source = copiedRegister(operations); - if (source == null) return false; - target = source.getBaseRegister(); + if (source == null) continue; + next = source.getBaseRegister(); + } + for (Instruction predecessor : predecessors(function, instruction)) { + instructions.push(predecessor); + targets.push(next); } - from = instruction.getFallFrom(); - instruction = from == null ? null : currentProgram.getListing().getInstructionAt(from); } return false; } + private List predecessors(Function function, Instruction instruction) { + List result = new ArrayList<>(); + Address from = instruction.getFallFrom(); + if (from != null) { + Instruction previous = currentProgram.getListing().getInstructionAt(from); + if (previous != null) result.add(previous); + } + for (Reference reference : currentProgram.getReferenceManager().getReferencesTo(instruction.getAddress())) { + if (!reference.getReferenceType().isFlow() || reference.getReferenceType().isCall()) continue; + if (!function.getBody().contains(reference.getFromAddress())) continue; + Instruction source = currentProgram.getListing().getInstructionAt(reference.getFromAddress()); + if (source != null) result.add(source); + } + return result; + } + private Register copiedRegister(PcodeOp[] operations) { if (operations.length != 1) return null; int opcode = operations[0].getOpcode(); @@ -3518,13 +3546,16 @@ public final class ReaGhidraBridge extends HeadlessScript { } /** - * Every large literal of one dossier draws on half of the heap that was free when decoding - * began, charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up to six - * escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer (up to - * twice its length), again in the returned String, and once more as UTF-8 on the socket. + * Every literal record of one dossier draws on half of the heap that was free when decoding + * began. Text is charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up + * to six escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer + * (up to twice its length), again in the returned String, and once more as UTF-8 on the + * socket. Each record also pays a fixed cost for its map key, JSON members, addresses, words + * and their serialized form, so many empty or small literals are bounded too. */ private static final class SwiftLiteralBudget { private static final long BYTES_PER_LITERAL_BYTE = 2 + 6 * 2 * 2 + 6 * 2 + 6; + private static final long BYTES_PER_RECORD = 4096; private long remaining; SwiftLiteralBudget() { @@ -3533,14 +3564,13 @@ public final class ReaGhidraBridge extends HeadlessScript { } boolean take(long bytes) { - if (bytes > remaining / BYTES_PER_LITERAL_BYTE) return false; - remaining -= bytes * BYTES_PER_LITERAL_BYTE; + if (remaining < BYTES_PER_RECORD || + bytes > (remaining - BYTES_PER_RECORD) / BYTES_PER_LITERAL_BYTE) { + return false; + } + remaining -= BYTES_PER_RECORD + bytes * BYTES_PER_LITERAL_BYTE; return true; } - - void exhaust() { - remaining = 0; - } } // Swift String values may contain any scalar, including controls; JSON escapes them. @@ -3559,7 +3589,11 @@ public final class ReaGhidraBridge extends HeadlessScript { private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) { JsonObject item = new JsonObject(); item.addProperty("representation", layout.representation()); - item.addProperty("address", canonicalAddress(layout.address())); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } item.addProperty("byte_length", layout.byteLength()); item.addProperty("reason", "capacity"); item.add("words", swiftStringWords(word, object)); diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index 3a258bbc4..629a2524d 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -572,30 +572,30 @@ describe("function comparison", () => { }); }); -describe("function comparison Swift string literals", () => { - const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) => - functionDossierSchema.parse({ - ...dossier("return 0;", base), - swift_string_literals: values.map((value) => ({ - source_address: base, - value, - representation: "small" as const, - address: null, - registers: ["x0", "x1"] as const, - words: ["0x0", "0xe"] as const, - })), - }); - const stringsDimension = ( - leftDigit: string, - left: ReturnType | ReturnType, - rightDigit: string, - right: ReturnType | ReturnType, - ) => - compareFunctions( - observe(leftDigit, left), - observe(rightDigit, right), - ).dimensions.find(({ dimension }) => dimension === "strings_names"); +const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) => + functionDossierSchema.parse({ + ...dossier("return 0;", base), + swift_string_literals: values.map((value) => ({ + source_address: base, + value, + representation: "small" as const, + address: null, + registers: ["x0", "x1"] as const, + words: ["0x0", "0xe"] as const, + })), + }); +const stringsDimension = ( + leftDigit: string, + left: ReturnType | ReturnType, + rightDigit: string, + right: ReturnType | ReturnType, +) => + compareFunctions( + observe(leftDigit, left), + observe(rightDigit, right), + ).dimensions.find(({ dimension }) => dimension === "strings_names"); +describe("function comparison Swift string literals", () => { it("treats differing decoded literals as a strings and names change", () => { expect( stringsDimension( @@ -671,6 +671,46 @@ describe("function comparison Swift string literals", () => { ).toMatchObject({ status: "unknown" }); }); + it("reports a literal that a complete dossier shows absent at a site the partial one decoded", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("d", withLiterals("0x1000", []), "e", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("treats an omission record without decoded literals as partial coverage", () => { + const omissionOnly = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("f", dossier("return 0;", "0x1000"), "0", omissionOnly), + ).toMatchObject({ status: "unknown" }); + }); + it("reports a conflicting literal at a shared use site despite partial coverage", () => { const partial = (base: "0x1000" | "0x2000", value: string) => functionDossierSchema.parse({ diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index 2dcafa0a7..db1cdfe1b 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -184,10 +184,10 @@ const compareStringsAndNames = ( const swiftLiteralCoverage = ( snapshot: FunctionSnapshot, ): "absent" | "complete" | "partial" => - snapshot.dossier.swift_string_literals === undefined - ? "absent" - : (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0 - ? "partial" + (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0 + ? "partial" + : snapshot.dossier.swift_string_literals === undefined + ? "absent" : "complete"; const compareReferences = ( diff --git a/src/domain/functionComparisonNormalization.ts b/src/domain/functionComparisonNormalization.ts index 03412c3ee..3b14b4d2d 100644 --- a/src/domain/functionComparisonNormalization.ts +++ b/src/domain/functionComparisonNormalization.ts @@ -194,37 +194,48 @@ export const stringAndNameProjection = ( : sorted(values); }; -/** Whether both dossiers decoded different literals at one use site that neither left partial. */ +/** + * Whether one use site is known to hold different literals in the two dossiers. A decoded + * literal is known wherever it was recorded; an absence is known only in a dossier whose + * decoding examined every use site, and the site where decoding stopped is never known. + */ export const swiftLiteralSitesConflict = ( left: FunctionSnapshot, right: FunctionSnapshot, ): boolean => { const leftSites = swiftLiteralSites(left); const rightSites = swiftLiteralSites(right); - for (const [site, values] of leftSites.values) { - const other = rightSites.values.get(site); - if (other === undefined || rightSites.omitted.has(site)) continue; - if (canonicalJson(values) !== canonicalJson(other)) return true; + for (const site of new Set([ + ...leftSites.values.keys(), + ...rightSites.values.keys(), + ])) { + if (leftSites.omitted.has(site) || rightSites.omitted.has(site)) continue; + const leftValues = + leftSites.values.get(site) ?? (leftSites.complete ? [] : undefined); + const rightValues = + rightSites.values.get(site) ?? (rightSites.complete ? [] : undefined); + if (leftValues === undefined || rightValues === undefined) continue; + if (canonicalJson(leftValues) !== canonicalJson(rightValues)) return true; } return false; }; const swiftLiteralSites = (snapshot: FunctionSnapshot) => { const base = snapshot.dossier.procedure.address; + const omissions = snapshot.dossier.swift_string_literal_omissions ?? []; const site = (sourceAddress: string, registers: readonly string[]) => { const offset = relativeAddress(sourceAddress, base); return offset === null ? null : canonicalJson([offset, registers]); }; const omitted = new Set( - (snapshot.dossier.swift_string_literal_omissions ?? []).map( - ({ source_address: sourceAddress, registers }) => - site(sourceAddress, registers), + omissions.map(({ source_address: sourceAddress, registers }) => + site(sourceAddress, registers), ), ); const values = new Map(); for (const literal of snapshot.dossier.swift_string_literals ?? []) { const key = site(literal.source_address, literal.registers); - if (key === null || omitted.has(key)) continue; + if (key === null) continue; values.set( key, [...(values.get(key) ?? []), literal.value].sort( @@ -232,7 +243,10 @@ const swiftLiteralSites = (snapshot: FunctionSnapshot) => { ), ); } - return { values, omitted }; + const complete = + snapshot.dossier.swift_string_literals !== undefined && + omissions.length === 0; + return { values, omitted, complete }; }; export const sorted = (values: readonly unknown[]): readonly unknown[] => diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 41527c7f0..ef1a12ebf 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -321,20 +321,34 @@ export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [ }), ]); -/** A large Swift literal whose layout matched but whose text was not decoded. */ -export const swiftStringLiteralOmissionSchema = z.strictObject({ +const swiftStringLiteralOmissionFields = { source_address: swiftStringLiteralFields.source_address, - representation: z.literal("large"), - address: z.string().describe("Address of the literal's bytes."), byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), reason: z .literal("capacity") .describe( - "The text did not fit the provider's memory budget for this dossier.", + "The record did not fit the provider's memory budget for this dossier; decoding stopped here, so later use sites were not examined.", ), registers: swiftStringLiteralFields.registers, words: swiftStringLiteralFields.words, -}); +}; + +/** A Swift literal whose layout matched but whose record did not fit; decoding stopped there. */ +export const swiftStringLiteralOmissionSchema = z.discriminatedUnion( + "representation", + [ + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("small"), + address: z.null(), + }), + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("large"), + address: z.string().describe("Address of the literal's bytes."), + }), + ], +); /** Observed endpoints with complete or partial reference classification evidence. */ export const referenceEdgeSchema = z @@ -397,7 +411,7 @@ export const functionDossierSchema = z .array(swiftStringLiteralOmissionSchema) .exactOptional() .describe( - "Literals that matched the Swift layout but were not decoded. Empty when swift_string_literals is complete for the examined use sites.", + "The literal at which decoding stopped for capacity. Present exactly when swift_string_literals is present; empty when decoding examined every use site.", ), basic_blocks: z.array( z diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index 68eaada71..4abbd7434 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -210,6 +210,7 @@ describe("Ghidra Swift String literals", () => { parseFunctionValueFixture("analyze_function", { ...dossier, swift_string_literals: literals, + swift_string_literal_omissions: [], }), ).toMatchObject({ ok: true, value: { swift_string_literals: literals } }); }); @@ -457,6 +458,7 @@ const malformedSwiftStringLiterals = ( "analyze_function", { ...dossier, + swift_string_literal_omissions: [], swift_string_literals: [ { source_address: "0x401004", @@ -484,6 +486,11 @@ const malformedOutputs = (): Array< throw new TypeError("Ghidra native API fixture is invalid"); return [ ["non-canonical address", "procedure_callers", ["401000"]], + [ + "Swift literals without their omission record", + "analyze_function", + { ...dossier, swift_string_literals: [] }, + ], [ "missing classification", "procedure_info", diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index bc35114bf..f6e76b691 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -292,6 +292,15 @@ const ghidraFunctionDossier = functionDossierSchema }) .strict() .superRefine((value, context) => { + if ( + (value.swift_string_literals === undefined) !== + (value.swift_string_literal_omissions === undefined) + ) + context.addIssue({ + code: "custom", + message: + "Ghidra dossier must record Swift literals and their omissions together", + }); const addresses = [ value.procedure.address, ...value.comments.map(({ address }) => address), @@ -312,7 +321,8 @@ const ghidraFunctionDossier = functionDossierSchema address === null ? [source_address] : [address, source_address], ), ...(value.swift_string_literal_omissions ?? []).flatMap( - ({ address, source_address }) => [address, source_address], + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], ), ...value.basic_blocks.flatMap(({ start, end, successors }) => [ start,