diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index f780f89d6..17de12f50 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -24,6 +24,7 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermissions; import java.security.MessageDigest; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -3356,22 +3357,21 @@ public final class ReaGhidraBridge extends HeadlessScript { SwiftLiteralLayout layout = swiftLiteralLayout(word, object); // Ghidra forks each select without correlating its condition, so a pair whose // registers are both selected may combine words from different cases. - if (layout == null || (selectedValue(instruction, pair[0]) && selectedValue(instruction, pair[1]))) continue; + if (layout == null || + (selectedValue(function, instruction, pair[0]) && selectedValue(function, instruction, pair[1]))) continue; // Forks can reach one site with different constants, so the words are part of the key. String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() + "\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) + "\u0000" + Long.toUnsignedString(object, 16); if (observed.containsKey(key) || omitted.containsKey(key)) continue; - // Small text is at most 15 bytes and already decoded; only large reads draw on the budget. + // Every record draws on the budget; the first one that does not fit is kept as an + // omission and decoding stops, so later sites are never examined. String value = layout.smallValue(); - boolean overCapacity = false; - if (value == null && !budget.take(layout.byteLength())) { - overCapacity = true; - } else if (value == null) { + boolean overCapacity = !budget.take(layout.byteLength()); + if (!overCapacity && value == null) { try { value = swiftLargeLiteralText(layout); } catch (OutOfMemoryError exhausted) { - budget.exhaust(); overCapacity = true; } } @@ -3385,6 +3385,7 @@ public final class ReaGhidraBridge extends HeadlessScript { pairRegisters.add(pair[1].getName()); item.add("registers", pairRegisters); (value == null ? omitted : observed).put(key, item); + if (overCapacity) return true; } return false; } @@ -3435,32 +3436,59 @@ public final class ReaGhidraBridge extends HeadlessScript { return value != null && context.isConstant(value) ? value.getOffset() : null; } - // Follow register copies back to the defining instruction, since a select copied into an - // argument register forks just as independently as one written there directly. - private boolean selectedValue(Instruction use, Register register) { - Register target = register.getBaseRegister(); - Address from = use.getFallFrom(); - Instruction instruction = from == null ? null : currentProgram.getListing().getInstructionAt(from); - while (instruction != null) { + // Search every predecessor path for the reaching definitions, following register copies, + // since a select copied into an argument register or defined before a branch forks just as + // independently as one written there directly. + private boolean selectedValue(Function function, Instruction use, Register register) { + ArrayDeque instructions = new ArrayDeque<>(); + ArrayDeque targets = new ArrayDeque<>(); + Set visited = new HashSet<>(); + for (Instruction predecessor : predecessors(function, use)) { + instructions.push(predecessor); + targets.push(register.getBaseRegister()); + } + while (!instructions.isEmpty()) { + Instruction instruction = instructions.pop(); + Register target = targets.pop(); + if (!visited.add(canonicalAddress(instruction.getAddress()) + "\u0000" + target.getName())) continue; boolean defines = false; for (Object result : instruction.getResultObjects()) { if (result instanceof Register written && written.getBaseRegister().equals(target)) defines = true; } + Register next = target; if (defines) { PcodeOp[] operations = instruction.getPcode(); for (PcodeOp operation : operations) { if (operation.getOpcode() == PcodeOp.CBRANCH && operation.getInput(0).isConstant()) return true; } Register source = copiedRegister(operations); - if (source == null) return false; - target = source.getBaseRegister(); + if (source == null) continue; + next = source.getBaseRegister(); + } + for (Instruction predecessor : predecessors(function, instruction)) { + instructions.push(predecessor); + targets.push(next); } - from = instruction.getFallFrom(); - instruction = from == null ? null : currentProgram.getListing().getInstructionAt(from); } return false; } + private List predecessors(Function function, Instruction instruction) { + List result = new ArrayList<>(); + Address from = instruction.getFallFrom(); + if (from != null) { + Instruction previous = currentProgram.getListing().getInstructionAt(from); + if (previous != null) result.add(previous); + } + for (Reference reference : currentProgram.getReferenceManager().getReferencesTo(instruction.getAddress())) { + if (!reference.getReferenceType().isFlow() || reference.getReferenceType().isCall()) continue; + if (!function.getBody().contains(reference.getFromAddress())) continue; + Instruction source = currentProgram.getListing().getInstructionAt(reference.getFromAddress()); + if (source != null) result.add(source); + } + return result; + } + private Register copiedRegister(PcodeOp[] operations) { if (operations.length != 1) return null; int opcode = operations[0].getOpcode(); @@ -3518,13 +3546,16 @@ public final class ReaGhidraBridge extends HeadlessScript { } /** - * Every large literal of one dossier draws on half of the heap that was free when decoding - * began, charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up to six - * escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer (up to - * twice its length), again in the returned String, and once more as UTF-8 on the socket. + * Every literal record of one dossier draws on half of the heap that was free when decoding + * began. Text is charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up + * to six escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer + * (up to twice its length), again in the returned String, and once more as UTF-8 on the + * socket. Each record also pays a fixed cost for its map key, JSON members, addresses, words + * and their serialized form, so many empty or small literals are bounded too. */ private static final class SwiftLiteralBudget { private static final long BYTES_PER_LITERAL_BYTE = 2 + 6 * 2 * 2 + 6 * 2 + 6; + private static final long BYTES_PER_RECORD = 4096; private long remaining; SwiftLiteralBudget() { @@ -3533,14 +3564,13 @@ public final class ReaGhidraBridge extends HeadlessScript { } boolean take(long bytes) { - if (bytes > remaining / BYTES_PER_LITERAL_BYTE) return false; - remaining -= bytes * BYTES_PER_LITERAL_BYTE; + if (remaining < BYTES_PER_RECORD || + bytes > (remaining - BYTES_PER_RECORD) / BYTES_PER_LITERAL_BYTE) { + return false; + } + remaining -= BYTES_PER_RECORD + bytes * BYTES_PER_LITERAL_BYTE; return true; } - - void exhaust() { - remaining = 0; - } } // Swift String values may contain any scalar, including controls; JSON escapes them. @@ -3559,7 +3589,11 @@ public final class ReaGhidraBridge extends HeadlessScript { private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) { JsonObject item = new JsonObject(); item.addProperty("representation", layout.representation()); - item.addProperty("address", canonicalAddress(layout.address())); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } item.addProperty("byte_length", layout.byteLength()); item.addProperty("reason", "capacity"); item.add("words", swiftStringWords(word, object)); diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index 3a258bbc4..629a2524d 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -572,30 +572,30 @@ describe("function comparison", () => { }); }); -describe("function comparison Swift string literals", () => { - const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) => - functionDossierSchema.parse({ - ...dossier("return 0;", base), - swift_string_literals: values.map((value) => ({ - source_address: base, - value, - representation: "small" as const, - address: null, - registers: ["x0", "x1"] as const, - words: ["0x0", "0xe"] as const, - })), - }); - const stringsDimension = ( - leftDigit: string, - left: ReturnType | ReturnType, - rightDigit: string, - right: ReturnType | ReturnType, - ) => - compareFunctions( - observe(leftDigit, left), - observe(rightDigit, right), - ).dimensions.find(({ dimension }) => dimension === "strings_names"); +const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) => + functionDossierSchema.parse({ + ...dossier("return 0;", base), + swift_string_literals: values.map((value) => ({ + source_address: base, + value, + representation: "small" as const, + address: null, + registers: ["x0", "x1"] as const, + words: ["0x0", "0xe"] as const, + })), + }); +const stringsDimension = ( + leftDigit: string, + left: ReturnType | ReturnType, + rightDigit: string, + right: ReturnType | ReturnType, +) => + compareFunctions( + observe(leftDigit, left), + observe(rightDigit, right), + ).dimensions.find(({ dimension }) => dimension === "strings_names"); +describe("function comparison Swift string literals", () => { it("treats differing decoded literals as a strings and names change", () => { expect( stringsDimension( @@ -671,6 +671,46 @@ describe("function comparison Swift string literals", () => { ).toMatchObject({ status: "unknown" }); }); + it("reports a literal that a complete dossier shows absent at a site the partial one decoded", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("d", withLiterals("0x1000", []), "e", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("treats an omission record without decoded literals as partial coverage", () => { + const omissionOnly = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("f", dossier("return 0;", "0x1000"), "0", omissionOnly), + ).toMatchObject({ status: "unknown" }); + }); + it("reports a conflicting literal at a shared use site despite partial coverage", () => { const partial = (base: "0x1000" | "0x2000", value: string) => functionDossierSchema.parse({ diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index 2dcafa0a7..db1cdfe1b 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -184,10 +184,10 @@ const compareStringsAndNames = ( const swiftLiteralCoverage = ( snapshot: FunctionSnapshot, ): "absent" | "complete" | "partial" => - snapshot.dossier.swift_string_literals === undefined - ? "absent" - : (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0 - ? "partial" + (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0 + ? "partial" + : snapshot.dossier.swift_string_literals === undefined + ? "absent" : "complete"; const compareReferences = ( diff --git a/src/domain/functionComparisonNormalization.ts b/src/domain/functionComparisonNormalization.ts index 03412c3ee..3b14b4d2d 100644 --- a/src/domain/functionComparisonNormalization.ts +++ b/src/domain/functionComparisonNormalization.ts @@ -194,37 +194,48 @@ export const stringAndNameProjection = ( : sorted(values); }; -/** Whether both dossiers decoded different literals at one use site that neither left partial. */ +/** + * Whether one use site is known to hold different literals in the two dossiers. A decoded + * literal is known wherever it was recorded; an absence is known only in a dossier whose + * decoding examined every use site, and the site where decoding stopped is never known. + */ export const swiftLiteralSitesConflict = ( left: FunctionSnapshot, right: FunctionSnapshot, ): boolean => { const leftSites = swiftLiteralSites(left); const rightSites = swiftLiteralSites(right); - for (const [site, values] of leftSites.values) { - const other = rightSites.values.get(site); - if (other === undefined || rightSites.omitted.has(site)) continue; - if (canonicalJson(values) !== canonicalJson(other)) return true; + for (const site of new Set([ + ...leftSites.values.keys(), + ...rightSites.values.keys(), + ])) { + if (leftSites.omitted.has(site) || rightSites.omitted.has(site)) continue; + const leftValues = + leftSites.values.get(site) ?? (leftSites.complete ? [] : undefined); + const rightValues = + rightSites.values.get(site) ?? (rightSites.complete ? [] : undefined); + if (leftValues === undefined || rightValues === undefined) continue; + if (canonicalJson(leftValues) !== canonicalJson(rightValues)) return true; } return false; }; const swiftLiteralSites = (snapshot: FunctionSnapshot) => { const base = snapshot.dossier.procedure.address; + const omissions = snapshot.dossier.swift_string_literal_omissions ?? []; const site = (sourceAddress: string, registers: readonly string[]) => { const offset = relativeAddress(sourceAddress, base); return offset === null ? null : canonicalJson([offset, registers]); }; const omitted = new Set( - (snapshot.dossier.swift_string_literal_omissions ?? []).map( - ({ source_address: sourceAddress, registers }) => - site(sourceAddress, registers), + omissions.map(({ source_address: sourceAddress, registers }) => + site(sourceAddress, registers), ), ); const values = new Map(); for (const literal of snapshot.dossier.swift_string_literals ?? []) { const key = site(literal.source_address, literal.registers); - if (key === null || omitted.has(key)) continue; + if (key === null) continue; values.set( key, [...(values.get(key) ?? []), literal.value].sort( @@ -232,7 +243,10 @@ const swiftLiteralSites = (snapshot: FunctionSnapshot) => { ), ); } - return { values, omitted }; + const complete = + snapshot.dossier.swift_string_literals !== undefined && + omissions.length === 0; + return { values, omitted, complete }; }; export const sorted = (values: readonly unknown[]): readonly unknown[] => diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 41527c7f0..ef1a12ebf 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -321,20 +321,34 @@ export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [ }), ]); -/** A large Swift literal whose layout matched but whose text was not decoded. */ -export const swiftStringLiteralOmissionSchema = z.strictObject({ +const swiftStringLiteralOmissionFields = { source_address: swiftStringLiteralFields.source_address, - representation: z.literal("large"), - address: z.string().describe("Address of the literal's bytes."), byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), reason: z .literal("capacity") .describe( - "The text did not fit the provider's memory budget for this dossier.", + "The record did not fit the provider's memory budget for this dossier; decoding stopped here, so later use sites were not examined.", ), registers: swiftStringLiteralFields.registers, words: swiftStringLiteralFields.words, -}); +}; + +/** A Swift literal whose layout matched but whose record did not fit; decoding stopped there. */ +export const swiftStringLiteralOmissionSchema = z.discriminatedUnion( + "representation", + [ + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("small"), + address: z.null(), + }), + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("large"), + address: z.string().describe("Address of the literal's bytes."), + }), + ], +); /** Observed endpoints with complete or partial reference classification evidence. */ export const referenceEdgeSchema = z @@ -397,7 +411,7 @@ export const functionDossierSchema = z .array(swiftStringLiteralOmissionSchema) .exactOptional() .describe( - "Literals that matched the Swift layout but were not decoded. Empty when swift_string_literals is complete for the examined use sites.", + "The literal at which decoding stopped for capacity. Present exactly when swift_string_literals is present; empty when decoding examined every use site.", ), basic_blocks: z.array( z diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index 68eaada71..4abbd7434 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -210,6 +210,7 @@ describe("Ghidra Swift String literals", () => { parseFunctionValueFixture("analyze_function", { ...dossier, swift_string_literals: literals, + swift_string_literal_omissions: [], }), ).toMatchObject({ ok: true, value: { swift_string_literals: literals } }); }); @@ -457,6 +458,7 @@ const malformedSwiftStringLiterals = ( "analyze_function", { ...dossier, + swift_string_literal_omissions: [], swift_string_literals: [ { source_address: "0x401004", @@ -484,6 +486,11 @@ const malformedOutputs = (): Array< throw new TypeError("Ghidra native API fixture is invalid"); return [ ["non-canonical address", "procedure_callers", ["401000"]], + [ + "Swift literals without their omission record", + "analyze_function", + { ...dossier, swift_string_literals: [] }, + ], [ "missing classification", "procedure_info", diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index bc35114bf..f6e76b691 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -292,6 +292,15 @@ const ghidraFunctionDossier = functionDossierSchema }) .strict() .superRefine((value, context) => { + if ( + (value.swift_string_literals === undefined) !== + (value.swift_string_literal_omissions === undefined) + ) + context.addIssue({ + code: "custom", + message: + "Ghidra dossier must record Swift literals and their omissions together", + }); const addresses = [ value.procedure.address, ...value.comments.map(({ address }) => address), @@ -312,7 +321,8 @@ const ghidraFunctionDossier = functionDossierSchema address === null ? [source_address] : [address, source_address], ), ...(value.swift_string_literal_omissions ?? []).flatMap( - ({ address, source_address }) => [address, source_address], + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], ), ...value.basic_blocks.flatMap(({ start, end, successors }) => [ start,