diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index c0889022e..2e225152b 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -1242,11 +1242,10 @@ public final class ReaGhidraBridge extends HeadlessScript { result.add("referenced_names", referencedNames); boolean swift = swiftProgram(); boolean aarch64 = currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:"); - boolean omittedSwiftLiteralForCapacity = false; - if (swift && aarch64) { - SwiftStringLiteralDecode decoded = swiftStringLiterals(function); - result.add("swift_string_literals", decoded.literals()); - omittedSwiftLiteralForCapacity = decoded.omittedForCapacity(); + SwiftStringLiteralDecode swiftLiterals = swift && aarch64 ? swiftStringLiterals(function) : null; + if (swiftLiterals != null) { + result.add("swift_string_literals", swiftLiterals.literals()); + result.add("swift_string_literal_omissions", swiftLiterals.omissions()); } result.add("basic_blocks", blocks); result.add( @@ -1271,9 +1270,9 @@ public final class ReaGhidraBridge extends HeadlessScript { limitations.add( "Swift String literals are decoded from constant register pairs at calls, returns and paired stores; literals stored one word at a time, chosen by two conditional selects, or built at runtime are not reported." ); - if (omittedSwiftLiteralForCapacity) { + if (swiftLiterals.omissions().size() > 0) { limitations.add( - "A Swift String literal matched the immortal layout but was omitted because its byte length does not fit in the remaining Java heap before UTF-16 and JSON expansion." + "Swift String literals listed in swift_string_literal_omissions matched the immortal layout but were not decoded because their text does not fit the heap budget for this dossier." ); } } else if (swift) { @@ -3344,7 +3343,8 @@ public final class ReaGhidraBridge extends HeadlessScript { registers[index] = currentProgram.getRegister("x" + index); } TreeMap observed = new TreeMap<>(); - boolean[] omittedForCapacity = { false }; + TreeMap omitted = new TreeMap<>(); + SwiftLiteralBudget budget = new SwiftLiteralBudget(); SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false); propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() { @Override @@ -3353,27 +3353,47 @@ public final class ReaGhidraBridge extends HeadlessScript { Long word = registerConstant(context, pair[0]); Long object = registerConstant(context, pair[1]); if (word == null || object == null) continue; - JsonObject item = swiftStringLiteral(word, object, omittedForCapacity); + SwiftLiteralLayout layout = swiftLiteralLayout(word, object); // Ghidra forks each select without correlating its condition, so a pair whose // registers are both selected may combine words from different cases. - if (item == null || (selectedValue(instruction, pair[0]) && selectedValue(instruction, pair[1]))) continue; + if (layout == null || (selectedValue(instruction, pair[0]) && selectedValue(instruction, pair[1]))) continue; + // Forks can reach one site with different constants, so the words are part of the key. + String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() + + "\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) + + "\u0000" + Long.toUnsignedString(object, 16); + if (observed.containsKey(key) || omitted.containsKey(key)) continue; + // Small text is at most 15 bytes and already decoded; only large reads draw on the budget. + String value = layout.smallValue(); + boolean overCapacity = false; + if (value == null && !budget.take(layout.byteLength())) { + overCapacity = true; + } else if (value == null) { + try { + value = swiftLargeLiteralText(layout); + } catch (OutOfMemoryError exhausted) { + budget.exhaust(); + overCapacity = true; + } + } + if (value == null && !overCapacity) continue; + JsonObject item = value == null + ? swiftStringOmission(layout, word, object) + : swiftStringItem(value, layout, word, object); item.addProperty("source_address", canonicalAddress(instruction.getAddress())); JsonArray pairRegisters = new JsonArray(); pairRegisters.add(pair[0].getName()); pairRegisters.add(pair[1].getName()); item.add("registers", pairRegisters); - observed.put( - canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() + - "\u0000" + pair[1].getName() + "\u0000" + item.get("value").getAsString(), - item - ); + (value == null ? omitted : observed).put(key, item); } return false; } }, false, monitor); - JsonArray result = new JsonArray(); - observed.values().forEach(result::add); - return new SwiftStringLiteralDecode(result, omittedForCapacity[0]); + JsonArray literals = new JsonArray(); + observed.values().forEach(literals::add); + JsonArray omissions = new JsonArray(); + omitted.values().forEach(omissions::add); + return new SwiftStringLiteralDecode(literals, omissions); } private static List swiftStringPairs( @@ -3391,11 +3411,17 @@ public final class ReaGhidraBridge extends HeadlessScript { return pairs; } FlowType flow = instruction.getFlowType(); - boolean returns = flow.isTerminal() && !flow.isJump() && !flow.isCall(); - boolean leaves = flow.isCall() || returns || (flow.isJump() && flow.isComputed()) || - (flow.isJump() && !flow.isConditional() && - Arrays.stream(instruction.getFlows()).anyMatch(target -> !function.getBody().contains(target))); - if (!leaves) return pairs; + boolean returns = false; + for (PcodeOp operation : instruction.getPcode()) { + if (operation.getOpcode() == PcodeOp.RETURN) returns = true; + } + // A jump leaves the function only toward a target outside its body; a computed jump + // without resolved targets is an indirect tail call, while a resolved jump table is not. + Address[] targets = instruction.getFlows(); + boolean tailCall = flow.isJump() && !flow.isConditional() && (targets.length == 0 + ? flow.isComputed() + : Arrays.stream(targets).anyMatch(target -> !function.getBody().contains(target))); + if (!flow.isCall() && !returns && !tailCall) return pairs; // A String occupies two consecutive registers among x0-x7, or x0 and x1 when returned. int lastFirstRegister = returns ? 0 : 6; for (int index = 0; index <= lastFirstRegister; index++) { @@ -3428,7 +3454,8 @@ public final class ReaGhidraBridge extends HeadlessScript { return false; } - private JsonObject swiftStringLiteral(long word, long object, boolean[] omittedForCapacity) { + /** Validate the ABI layout of a pair; a small literal is decoded here, a large one only located. */ + private SwiftLiteralLayout swiftLiteralLayout(long word, long object) { int discriminator = (int) (object >>> 60); if (discriminator == 0xE || discriminator == 0xA) { int count = (int) ((object >>> 56) & 0xF); @@ -3440,7 +3467,7 @@ public final class ReaGhidraBridge extends HeadlessScript { } // Count zero is the canonical empty string (object 0xE000000000000000, word 0). String value = swiftStringText(bytes, count, discriminator == 0xE); - return value == null ? null : swiftStringItem(value, "small", null, word, object); + return value == null ? null : new SwiftLiteralLayout("small", null, count, true, value); } int flags = (int) (word >>> 60); long count = word & 0xFFFFFFFFFFFFL; @@ -3448,59 +3475,61 @@ public final class ReaGhidraBridge extends HeadlessScript { if ((object >>> 56) != 0x80 || (flags != 0xD && flags != 0x1) || ((word >>> 48) & 0xFFF) != 0) { return null; } - Address address; - byte[] bytes; try { - address = currentProgram.getAddressFactory().getDefaultAddressSpace() + Address address = currentProgram.getAddressFactory().getDefaultAddressSpace() .getAddress((object & 0x00FFFFFFFFFFFFFFL) + 0x20); MemoryBlock block = currentProgram.getMemory().getBlock(address); - // The bytes and their terminator must lie in one initialized, non-executable block. + // The bytes and their terminator must lie in one initialized, non-executable block, + // and Java arrays bound one read. if (block == null || !block.isInitialized() || block.isExecute() || - count >= block.getEnd().subtract(address) + 1) { + count >= block.getEnd().subtract(address) + 1 || count >= Integer.MAX_VALUE) { return null; } - if (!swiftLiteralBytesFit(count)) { - omittedForCapacity[0] = true; - return null; - } - bytes = new byte[(int) count + 1]; - if (currentProgram.getMemory().getBytes(address, bytes) != bytes.length) return null; - } catch (OutOfMemoryError exhausted) { - omittedForCapacity[0] = true; + return new SwiftLiteralLayout("large", address, count, flags == 0xD, null); + } catch (Exception unaddressable) { return null; + } + } + + private String swiftLargeLiteralText(SwiftLiteralLayout layout) { + int count = (int) layout.byteLength(); + byte[] bytes = new byte[count + 1]; + try { + if (currentProgram.getMemory().getBytes(layout.address(), bytes) != bytes.length) return null; } catch (Exception unreadable) { return null; } - if (bytes[(int) count] != 0) return null; - String value = swiftStringText(bytes, (int) count, flags == 0xD); - return value == null ? null : swiftStringItem(value, "large", address, word, object); + return bytes[count] == 0 ? swiftStringText(bytes, count, layout.ascii()) : null; } /** - * A large literal is retained as a byte array, a UTF-16 String, and JSON text. - * ASCII expands to two UTF-16 bytes and JSON serialization retains another copy, - * so require five bytes of remaining heap per content byte and leave half the heap - * for the rest of the dossier. Measure that before allocating. + * Decoded text is retained as UTF-16 and again as JSON text, so every large literal of one + * dossier draws five bytes per UTF-8 byte from half of the heap that was free when decoding began. */ - private static boolean swiftLiteralBytesFit(long count) { - if (count < 0 || count >= Integer.MAX_VALUE) return false; - Runtime runtime = Runtime.getRuntime(); - long available = runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory(); - long expanded; - try { - expanded = Math.addExact(Math.multiplyExact(count, 5L), 1L); - } catch (ArithmeticException overflow) { - return false; + private static final class SwiftLiteralBudget { + private long remaining; + + SwiftLiteralBudget() { + Runtime runtime = Runtime.getRuntime(); + remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2); + } + + boolean take(long bytes) { + if (bytes > remaining / 5) return false; + remaining -= bytes * 5; + return true; + } + + void exhaust() { + remaining = 0; } - return available > 0 && expanded <= available / 2; } + // Swift String values may contain any scalar, including controls; JSON escapes them. private static String swiftStringText(byte[] bytes, int length, boolean ascii) { if (length < 0 || length > bytes.length) return null; for (int index = 0; index < length; index++) { - byte value = bytes[index]; - if ((value >= 0 && value < 0x20 && value != '\t' && value != '\n' && value != '\r') || value == 0x7f) return null; - if (ascii && value < 0) return null; + if (ascii && bytes[index] < 0) return null; } try { return StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes, 0, length)).toString(); @@ -3509,26 +3538,39 @@ public final class ReaGhidraBridge extends HeadlessScript { } } + private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) { + JsonObject item = new JsonObject(); + item.addProperty("representation", layout.representation()); + item.addProperty("address", canonicalAddress(layout.address())); + item.addProperty("byte_length", layout.byteLength()); + item.addProperty("reason", "capacity"); + item.add("words", swiftStringWords(word, object)); + return item; + } + private static JsonObject swiftStringItem( String value, - String representation, - Address address, + SwiftLiteralLayout layout, long word, long object ) { JsonObject item = new JsonObject(); item.addProperty("value", value); - item.addProperty("representation", representation); - if (address == null) { + item.addProperty("representation", layout.representation()); + if (layout.address() == null) { item.add("address", JsonNull.INSTANCE); } else { - item.addProperty("address", canonicalAddress(address)); + item.addProperty("address", canonicalAddress(layout.address())); } + item.add("words", swiftStringWords(word, object)); + return item; + } + + private static JsonArray swiftStringWords(long word, long object) { JsonArray words = new JsonArray(); words.add("0x" + Long.toUnsignedString(word, 16)); words.add("0x" + Long.toUnsignedString(object, 16)); - item.add("words", words); - return item; + return words; } private static String exclusiveAddress(Address inclusive) { @@ -4070,7 +4112,14 @@ public final class ReaGhidraBridge extends HeadlessScript { private record AddressNameRequest(String inputAddress, Address address, String name) {} private record FunctionEntry(Function function, InventoryItem item) {} private record FunctionAnalysisObservation(JsonObject dossier, List limitations) {} - private record SwiftStringLiteralDecode(JsonArray literals, boolean omittedForCapacity) {} + private record SwiftStringLiteralDecode(JsonArray literals, JsonArray omissions) {} + private record SwiftLiteralLayout( + String representation, + Address address, + long byteLength, + boolean ascii, + String smallValue + ) {} private record InstructionScan(List instructions, boolean truncated) {} private record SessionDescriptor( String transport, diff --git a/scripts/verify-real-ghidra-swift.mjs b/scripts/verify-real-ghidra-swift.mjs index 7f78edc16..0aa58e2f0 100644 --- a/scripts/verify-real-ghidra-swift.mjs +++ b/scripts/verify-real-ghidra-swift.mjs @@ -59,9 +59,19 @@ try { await mkdir(join(root, "copied")); const copied = join(root, "copied", "swift-inventory"); await copyFile(binary, copied); - await execute("swiftc", ["-O", literalSource, "-o", literalBinary], { - timeout: 60_000, - }); + // Decoding covers AArch64 register pairs, so build that slice on any Mac host. + await execute( + "swiftc", + [ + "-O", + "-target", + "arm64-apple-macosx13.0", + literalSource, + "-o", + literalBinary, + ], + { timeout: 60_000 }, + ); await client.connect(transport); for (const [variant, path] of [ ["debug", binary], @@ -182,6 +192,10 @@ async function verifyStringLiterals() { ["café", "small"], ["Résumé of the naïve décor, written twice", "large"], ], + emptyAndControlLiterals: [ + ["", "small"], + ["tab\there", "small"], + ], returnedLiteral: [["Holding awake", "small"]], }; const decoded = {}; @@ -193,6 +207,7 @@ async function verifyStringLiterals() { }); const observed = dossier.swift_string_literals; assert(Array.isArray(observed), `${name} recorded no Swift literals`); + assert.deepEqual(dossier.swift_string_literal_omissions, []); assert.deepEqual( observed.map(({ value, representation }) => [value, representation]), literals, diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index c767c2bf8..bee905d8b 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -97,6 +97,7 @@ const nativeViewSchema = z.strictObject({ "referenced_strings", "referenced_names", "swift_string_literals", + "swift_string_literal_omissions", "native_api", "value_flow_summary", "value_flow_operations", diff --git a/src/domain/analysisView/nativeFunctionView.ts b/src/domain/analysisView/nativeFunctionView.ts index 2bac9f6c9..a16d1bc3f 100644 --- a/src/domain/analysisView/nativeFunctionView.ts +++ b/src/domain/analysisView/nativeFunctionView.ts @@ -121,6 +121,7 @@ const nativeRows = ( referenced_strings: dossier.referenced_strings, referenced_names: dossier.referenced_names, swift_string_literals: dossier.swift_string_literals, + swift_string_literal_omissions: dossier.swift_string_literal_omissions, value_flow_operations: flow?.operations, value_flow_def_use: flow?.def_use, value_flow_effects: flow?.effects, diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index e937dd953..272eccf22 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -628,13 +628,48 @@ describe("function comparison Swift string literals", () => { ), ).toMatchObject({ status: "unknown", - left_count: 0, - right_count: null, limitations: [ - "Swift String literal coverage differs; one dossier did not record decoded literals.", + "Swift String literal coverage differs or is partial; referenced strings and names match, but decoded literals were not compared.", ], }); }); + + it("reports a referenced name change even when Swift literal coverage differs", () => { + const named = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + referenced_names: [ + { address: "0x3000", value: "helper", source_address: "0x2000" }, + ], + }); + expect( + stringsDimension("7", withLiterals("0x1000", []), "8", named), + ).toMatchObject({ status: "changed" }); + }); + + it("does not equate a partial literal record with a complete one", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2000", + representation: "large", + address: "0x3000", + byte_length: 64, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000040", "0x8000000000002fe0"], + }, + ], + }); + expect( + stringsDimension( + "9", + withLiterals("0x1000", ["SleepDisabled"]), + "a", + partial, + ), + ).toMatchObject({ status: "unknown" }); + }); }); const expectInlineComparisonEvidence = ( diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index c390dd8b9..63cc69698 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -139,20 +139,12 @@ const compareStringsAndNames = ( right: FunctionSnapshot, context: ComparisonContext, ): FunctionDimension => { - const leftRecorded = left.dossier.swift_string_literals !== undefined; - const rightRecorded = right.dossier.swift_string_literals !== undefined; - if (leftRecorded !== rightRecorded) - return unresolvedDimension({ - dimension: "strings_names", - links: context.links, - leftCount: recordedStringFactCount(left), - rightCount: recordedStringFactCount(right), - limitations: [ - "Swift String literal coverage differs; one dossier did not record decoded literals.", - ], - }); - const leftValues = stringAndNameProjection(left); - const rightValues = stringAndNameProjection(right); + const complete = + swiftLiteralCoverage(left) === swiftLiteralCoverage(right) && + swiftLiteralCoverage(left) !== "partial"; + // Facts both dossiers record decide a change before Swift literal coverage can. + const leftValues = stringAndNameProjection(left, complete); + const rightValues = stringAndNameProjection(right, complete); if (leftValues === null || rightValues === null) return unresolvedDimension({ dimension: "strings_names", @@ -163,15 +155,32 @@ const compareStringsAndNames = ( "Reference source locations could not be normalized to function offsets.", ], }); - return compareValues("strings_names", leftValues, rightValues, context); + const compared = compareValues( + "strings_names", + leftValues, + rightValues, + context, + ); + if (complete || compared.status === "changed") return compared; + return unresolvedDimension({ + dimension: "strings_names", + links: context.links, + leftCount: null, + rightCount: null, + limitations: [ + "Swift String literal coverage differs or is partial; referenced strings and names match, but decoded literals were not compared.", + ], + }); }; -const recordedStringFactCount = (snapshot: FunctionSnapshot): number | null => +const swiftLiteralCoverage = ( + snapshot: FunctionSnapshot, +): "absent" | "complete" | "partial" => snapshot.dossier.swift_string_literals === undefined - ? null - : snapshot.dossier.referenced_strings.length + - snapshot.dossier.referenced_names.length + - snapshot.dossier.swift_string_literals.length; + ? "absent" + : (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0 + ? "partial" + : "complete"; const compareReferences = ( left: FunctionSnapshot, diff --git a/src/domain/functionComparisonNormalization.ts b/src/domain/functionComparisonNormalization.ts index f00dce1a7..0e27b7515 100644 --- a/src/domain/functionComparisonNormalization.ts +++ b/src/domain/functionComparisonNormalization.ts @@ -145,6 +145,7 @@ export const commentProjection = ( export const stringAndNameProjection = ( snapshot: FunctionSnapshot, + includeSwiftLiterals = true, ): readonly unknown[] | null => { const values = [ ...snapshot.dossier.referenced_strings.map( @@ -167,7 +168,10 @@ export const stringAndNameProjection = ( value, }), ), - ...(snapshot.dossier.swift_string_literals ?? []).map( + ...(includeSwiftLiterals + ? (snapshot.dossier.swift_string_literals ?? []) + : [] + ).map( ({ source_address: sourceAddress, value, diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index b706cd22e..41527c7f0 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -321,6 +321,21 @@ export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [ }), ]); +/** A large Swift literal whose layout matched but whose text was not decoded. */ +export const swiftStringLiteralOmissionSchema = z.strictObject({ + source_address: swiftStringLiteralFields.source_address, + representation: z.literal("large"), + address: z.string().describe("Address of the literal's bytes."), + byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + reason: z + .literal("capacity") + .describe( + "The text did not fit the provider's memory budget for this dossier.", + ), + registers: swiftStringLiteralFields.registers, + words: swiftStringLiteralFields.words, +}); + /** Observed endpoints with complete or partial reference classification evidence. */ export const referenceEdgeSchema = z .object({ @@ -378,6 +393,12 @@ export const functionDossierSchema = z .describe( "Swift String literals decoded from constant register values. Absent when the provider does not decode them for this target.", ), + swift_string_literal_omissions: z + .array(swiftStringLiteralOmissionSchema) + .exactOptional() + .describe( + "Literals that matched the Swift layout but were not decoded. Empty when swift_string_literals is complete for the examined use sites.", + ), basic_blocks: z.array( z .object({ diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index b8f6ab4d4..68eaada71 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -515,6 +515,25 @@ const malformedOutputs = (): Array< ), ], ...malformedSwiftStringLiterals(dossier), + [ + "non-canonical Swift literal omission address", + "analyze_function", + { + ...dossier, + swift_string_literals: [], + swift_string_literal_omissions: [ + { + source_address: "0x401004", + representation: "large", + address: "402240", + byte_length: 39, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000027", "0x8000000000402220"], + }, + ], + }, + ], [ "non-canonical native API target", "analyze_function", diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index 8cab00cb6..bc35114bf 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -311,6 +311,9 @@ const ghidraFunctionDossier = functionDossierSchema ({ address, source_address }) => address === null ? [source_address] : [address, source_address], ), + ...(value.swift_string_literal_omissions ?? []).flatMap( + ({ address, source_address }) => [address, source_address], + ), ...value.basic_blocks.flatMap(({ start, end, successors }) => [ start, end, diff --git a/tests/conformance/swift/string-literals.swift b/tests/conformance/swift/string-literals.swift index 8f06a7550..345c4cf74 100644 --- a/tests/conformance/swift/string-literals.swift +++ b/tests/conformance/swift/string-literals.swift @@ -10,8 +10,14 @@ func unicodeLiterals() { consume("Résumé of the naïve décor, written twice") } @inline(never) +func emptyAndControlLiterals() { + consume("") + consume("tab\there") +} +@inline(never) func returnedLiteral() -> String { "Holding awake" } smallLiteral() largeLiteral() unicodeLiterals() +emptyAndControlLiterals() print(returnedLiteral(), CommandLine.arguments.count)