fix(ghidra): budget Swift literal decoding and tighten use sites

Large literals now draw on one heap budget per dossier, measured once
before decoding, instead of a separate check per literal. A literal that
does not fit is listed in a new swift_string_literal_omissions field
with its address, byte length and reason, so consumers read omissions
from structured data rather than from limitation text.

strings_names comparison checks referenced strings and names first and
reports a change there even when Swift literal coverage differs; it
returns unknown only when those facts match and literal coverage is
absent on one side or partial.

A computed jump counts as a tail call only when it has no resolved
target inside the function, so jump tables no longer qualify, and a
return requires a RETURN p-code operation, so traps no longer qualify.

Decoded literals keep control characters, which Swift strings may
contain and JSON escapes. The Swift verifier builds the literal fixture
for arm64 on any Mac host and now also covers the empty string and a
literal containing a tab.
This commit is contained in:
Omisakin Joshua
2026-10-12 10:42:03 +01:00
parent 702dc9f932
commit facd58973a
11 changed files with 256 additions and 93 deletions
+115 -66
View File
@@ -1242,11 +1242,10 @@ public final class ReaGhidraBridge extends HeadlessScript {
result.add("referenced_names", referencedNames);
boolean swift = swiftProgram();
boolean aarch64 = currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:");
boolean omittedSwiftLiteralForCapacity = false;
if (swift && aarch64) {
SwiftStringLiteralDecode decoded = swiftStringLiterals(function);
result.add("swift_string_literals", decoded.literals());
omittedSwiftLiteralForCapacity = decoded.omittedForCapacity();
SwiftStringLiteralDecode swiftLiterals = swift && aarch64 ? swiftStringLiterals(function) : null;
if (swiftLiterals != null) {
result.add("swift_string_literals", swiftLiterals.literals());
result.add("swift_string_literal_omissions", swiftLiterals.omissions());
}
result.add("basic_blocks", blocks);
result.add(
@@ -1271,9 +1270,9 @@ public final class ReaGhidraBridge extends HeadlessScript {
limitations.add(
"Swift String literals are decoded from constant register pairs at calls, returns and paired stores; literals stored one word at a time, chosen by two conditional selects, or built at runtime are not reported."
);
if (omittedSwiftLiteralForCapacity) {
if (swiftLiterals.omissions().size() > 0) {
limitations.add(
"A Swift String literal matched the immortal layout but was omitted because its byte length does not fit in the remaining Java heap before UTF-16 and JSON expansion."
"Swift String literals listed in swift_string_literal_omissions matched the immortal layout but were not decoded because their text does not fit the heap budget for this dossier."
);
}
} else if (swift) {
@@ -3344,7 +3343,8 @@ public final class ReaGhidraBridge extends HeadlessScript {
registers[index] = currentProgram.getRegister("x" + index);
}
TreeMap<String, JsonObject> observed = new TreeMap<>();
boolean[] omittedForCapacity = { false };
TreeMap<String, JsonObject> omitted = new TreeMap<>();
SwiftLiteralBudget budget = new SwiftLiteralBudget();
SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false);
propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() {
@Override
@@ -3353,27 +3353,47 @@ public final class ReaGhidraBridge extends HeadlessScript {
Long word = registerConstant(context, pair[0]);
Long object = registerConstant(context, pair[1]);
if (word == null || object == null) continue;
JsonObject item = swiftStringLiteral(word, object, omittedForCapacity);
SwiftLiteralLayout layout = swiftLiteralLayout(word, object);
// Ghidra forks each select without correlating its condition, so a pair whose
// registers are both selected may combine words from different cases.
if (item == null || (selectedValue(instruction, pair[0]) && selectedValue(instruction, pair[1]))) continue;
if (layout == null || (selectedValue(instruction, pair[0]) && selectedValue(instruction, pair[1]))) continue;
// Forks can reach one site with different constants, so the words are part of the key.
String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() +
"\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) +
"\u0000" + Long.toUnsignedString(object, 16);
if (observed.containsKey(key) || omitted.containsKey(key)) continue;
// Small text is at most 15 bytes and already decoded; only large reads draw on the budget.
String value = layout.smallValue();
boolean overCapacity = false;
if (value == null && !budget.take(layout.byteLength())) {
overCapacity = true;
} else if (value == null) {
try {
value = swiftLargeLiteralText(layout);
} catch (OutOfMemoryError exhausted) {
budget.exhaust();
overCapacity = true;
}
}
if (value == null && !overCapacity) continue;
JsonObject item = value == null
? swiftStringOmission(layout, word, object)
: swiftStringItem(value, layout, word, object);
item.addProperty("source_address", canonicalAddress(instruction.getAddress()));
JsonArray pairRegisters = new JsonArray();
pairRegisters.add(pair[0].getName());
pairRegisters.add(pair[1].getName());
item.add("registers", pairRegisters);
observed.put(
canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() +
"\u0000" + pair[1].getName() + "\u0000" + item.get("value").getAsString(),
item
);
(value == null ? omitted : observed).put(key, item);
}
return false;
}
}, false, monitor);
JsonArray result = new JsonArray();
observed.values().forEach(result::add);
return new SwiftStringLiteralDecode(result, omittedForCapacity[0]);
JsonArray literals = new JsonArray();
observed.values().forEach(literals::add);
JsonArray omissions = new JsonArray();
omitted.values().forEach(omissions::add);
return new SwiftStringLiteralDecode(literals, omissions);
}
private static List<Register[]> swiftStringPairs(
@@ -3391,11 +3411,17 @@ public final class ReaGhidraBridge extends HeadlessScript {
return pairs;
}
FlowType flow = instruction.getFlowType();
boolean returns = flow.isTerminal() && !flow.isJump() && !flow.isCall();
boolean leaves = flow.isCall() || returns || (flow.isJump() && flow.isComputed()) ||
(flow.isJump() && !flow.isConditional() &&
Arrays.stream(instruction.getFlows()).anyMatch(target -> !function.getBody().contains(target)));
if (!leaves) return pairs;
boolean returns = false;
for (PcodeOp operation : instruction.getPcode()) {
if (operation.getOpcode() == PcodeOp.RETURN) returns = true;
}
// A jump leaves the function only toward a target outside its body; a computed jump
// without resolved targets is an indirect tail call, while a resolved jump table is not.
Address[] targets = instruction.getFlows();
boolean tailCall = flow.isJump() && !flow.isConditional() && (targets.length == 0
? flow.isComputed()
: Arrays.stream(targets).anyMatch(target -> !function.getBody().contains(target)));
if (!flow.isCall() && !returns && !tailCall) return pairs;
// A String occupies two consecutive registers among x0-x7, or x0 and x1 when returned.
int lastFirstRegister = returns ? 0 : 6;
for (int index = 0; index <= lastFirstRegister; index++) {
@@ -3428,7 +3454,8 @@ public final class ReaGhidraBridge extends HeadlessScript {
return false;
}
private JsonObject swiftStringLiteral(long word, long object, boolean[] omittedForCapacity) {
/** Validate the ABI layout of a pair; a small literal is decoded here, a large one only located. */
private SwiftLiteralLayout swiftLiteralLayout(long word, long object) {
int discriminator = (int) (object >>> 60);
if (discriminator == 0xE || discriminator == 0xA) {
int count = (int) ((object >>> 56) & 0xF);
@@ -3440,7 +3467,7 @@ public final class ReaGhidraBridge extends HeadlessScript {
}
// Count zero is the canonical empty string (object 0xE000000000000000, word 0).
String value = swiftStringText(bytes, count, discriminator == 0xE);
return value == null ? null : swiftStringItem(value, "small", null, word, object);
return value == null ? null : new SwiftLiteralLayout("small", null, count, true, value);
}
int flags = (int) (word >>> 60);
long count = word & 0xFFFFFFFFFFFFL;
@@ -3448,59 +3475,61 @@ public final class ReaGhidraBridge extends HeadlessScript {
if ((object >>> 56) != 0x80 || (flags != 0xD && flags != 0x1) || ((word >>> 48) & 0xFFF) != 0) {
return null;
}
Address address;
byte[] bytes;
try {
address = currentProgram.getAddressFactory().getDefaultAddressSpace()
Address address = currentProgram.getAddressFactory().getDefaultAddressSpace()
.getAddress((object & 0x00FFFFFFFFFFFFFFL) + 0x20);
MemoryBlock block = currentProgram.getMemory().getBlock(address);
// The bytes and their terminator must lie in one initialized, non-executable block.
// The bytes and their terminator must lie in one initialized, non-executable block,
// and Java arrays bound one read.
if (block == null || !block.isInitialized() || block.isExecute() ||
count >= block.getEnd().subtract(address) + 1) {
count >= block.getEnd().subtract(address) + 1 || count >= Integer.MAX_VALUE) {
return null;
}
if (!swiftLiteralBytesFit(count)) {
omittedForCapacity[0] = true;
return null;
}
bytes = new byte[(int) count + 1];
if (currentProgram.getMemory().getBytes(address, bytes) != bytes.length) return null;
} catch (OutOfMemoryError exhausted) {
omittedForCapacity[0] = true;
return new SwiftLiteralLayout("large", address, count, flags == 0xD, null);
} catch (Exception unaddressable) {
return null;
}
}
private String swiftLargeLiteralText(SwiftLiteralLayout layout) {
int count = (int) layout.byteLength();
byte[] bytes = new byte[count + 1];
try {
if (currentProgram.getMemory().getBytes(layout.address(), bytes) != bytes.length) return null;
} catch (Exception unreadable) {
return null;
}
if (bytes[(int) count] != 0) return null;
String value = swiftStringText(bytes, (int) count, flags == 0xD);
return value == null ? null : swiftStringItem(value, "large", address, word, object);
return bytes[count] == 0 ? swiftStringText(bytes, count, layout.ascii()) : null;
}
/**
* A large literal is retained as a byte array, a UTF-16 String, and JSON text.
* ASCII expands to two UTF-16 bytes and JSON serialization retains another copy,
* so require five bytes of remaining heap per content byte and leave half the heap
* for the rest of the dossier. Measure that before allocating.
* Decoded text is retained as UTF-16 and again as JSON text, so every large literal of one
* dossier draws five bytes per UTF-8 byte from half of the heap that was free when decoding began.
*/
private static boolean swiftLiteralBytesFit(long count) {
if (count < 0 || count >= Integer.MAX_VALUE) return false;
Runtime runtime = Runtime.getRuntime();
long available = runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory();
long expanded;
try {
expanded = Math.addExact(Math.multiplyExact(count, 5L), 1L);
} catch (ArithmeticException overflow) {
return false;
private static final class SwiftLiteralBudget {
private long remaining;
SwiftLiteralBudget() {
Runtime runtime = Runtime.getRuntime();
remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2);
}
boolean take(long bytes) {
if (bytes > remaining / 5) return false;
remaining -= bytes * 5;
return true;
}
void exhaust() {
remaining = 0;
}
return available > 0 && expanded <= available / 2;
}
// Swift String values may contain any scalar, including controls; JSON escapes them.
private static String swiftStringText(byte[] bytes, int length, boolean ascii) {
if (length < 0 || length > bytes.length) return null;
for (int index = 0; index < length; index++) {
byte value = bytes[index];
if ((value >= 0 && value < 0x20 && value != '\t' && value != '\n' && value != '\r') || value == 0x7f) return null;
if (ascii && value < 0) return null;
if (ascii && bytes[index] < 0) return null;
}
try {
return StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes, 0, length)).toString();
@@ -3509,26 +3538,39 @@ public final class ReaGhidraBridge extends HeadlessScript {
}
}
private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) {
JsonObject item = new JsonObject();
item.addProperty("representation", layout.representation());
item.addProperty("address", canonicalAddress(layout.address()));
item.addProperty("byte_length", layout.byteLength());
item.addProperty("reason", "capacity");
item.add("words", swiftStringWords(word, object));
return item;
}
private static JsonObject swiftStringItem(
String value,
String representation,
Address address,
SwiftLiteralLayout layout,
long word,
long object
) {
JsonObject item = new JsonObject();
item.addProperty("value", value);
item.addProperty("representation", representation);
if (address == null) {
item.addProperty("representation", layout.representation());
if (layout.address() == null) {
item.add("address", JsonNull.INSTANCE);
} else {
item.addProperty("address", canonicalAddress(address));
item.addProperty("address", canonicalAddress(layout.address()));
}
item.add("words", swiftStringWords(word, object));
return item;
}
private static JsonArray swiftStringWords(long word, long object) {
JsonArray words = new JsonArray();
words.add("0x" + Long.toUnsignedString(word, 16));
words.add("0x" + Long.toUnsignedString(object, 16));
item.add("words", words);
return item;
return words;
}
private static String exclusiveAddress(Address inclusive) {
@@ -4070,7 +4112,14 @@ public final class ReaGhidraBridge extends HeadlessScript {
private record AddressNameRequest(String inputAddress, Address address, String name) {}
private record FunctionEntry(Function function, InventoryItem item) {}
private record FunctionAnalysisObservation(JsonObject dossier, List<String> limitations) {}
private record SwiftStringLiteralDecode(JsonArray literals, boolean omittedForCapacity) {}
private record SwiftStringLiteralDecode(JsonArray literals, JsonArray omissions) {}
private record SwiftLiteralLayout(
String representation,
Address address,
long byteLength,
boolean ascii,
String smallValue
) {}
private record InstructionScan(List<Instruction> instructions, boolean truncated) {}
private record SessionDescriptor(
String transport,
+18 -3
View File
@@ -59,9 +59,19 @@ try {
await mkdir(join(root, "copied"));
const copied = join(root, "copied", "swift-inventory");
await copyFile(binary, copied);
await execute("swiftc", ["-O", literalSource, "-o", literalBinary], {
timeout: 60_000,
});
// Decoding covers AArch64 register pairs, so build that slice on any Mac host.
await execute(
"swiftc",
[
"-O",
"-target",
"arm64-apple-macosx13.0",
literalSource,
"-o",
literalBinary,
],
{ timeout: 60_000 },
);
await client.connect(transport);
for (const [variant, path] of [
["debug", binary],
@@ -182,6 +192,10 @@ async function verifyStringLiterals() {
["café", "small"],
["Résumé of the naïve décor, written twice", "large"],
],
emptyAndControlLiterals: [
["", "small"],
["tab\there", "small"],
],
returnedLiteral: [["Holding awake", "small"]],
};
const decoded = {};
@@ -193,6 +207,7 @@ async function verifyStringLiterals() {
});
const observed = dossier.swift_string_literals;
assert(Array.isArray(observed), `${name} recorded no Swift literals`);
assert.deepEqual(dossier.swift_string_literal_omissions, []);
assert.deepEqual(
observed.map(({ value, representation }) => [value, representation]),
literals,
+1
View File
@@ -97,6 +97,7 @@ const nativeViewSchema = z.strictObject({
"referenced_strings",
"referenced_names",
"swift_string_literals",
"swift_string_literal_omissions",
"native_api",
"value_flow_summary",
"value_flow_operations",
@@ -121,6 +121,7 @@ const nativeRows = (
referenced_strings: dossier.referenced_strings,
referenced_names: dossier.referenced_names,
swift_string_literals: dossier.swift_string_literals,
swift_string_literal_omissions: dossier.swift_string_literal_omissions,
value_flow_operations: flow?.operations,
value_flow_def_use: flow?.def_use,
value_flow_effects: flow?.effects,
+38 -3
View File
@@ -628,13 +628,48 @@ describe("function comparison Swift string literals", () => {
),
).toMatchObject({
status: "unknown",
left_count: 0,
right_count: null,
limitations: [
"Swift String literal coverage differs; one dossier did not record decoded literals.",
"Swift String literal coverage differs or is partial; referenced strings and names match, but decoded literals were not compared.",
],
});
});
it("reports a referenced name change even when Swift literal coverage differs", () => {
const named = functionDossierSchema.parse({
...dossier("return 0;", "0x2000"),
referenced_names: [
{ address: "0x3000", value: "helper", source_address: "0x2000" },
],
});
expect(
stringsDimension("7", withLiterals("0x1000", []), "8", named),
).toMatchObject({ status: "changed" });
});
it("does not equate a partial literal record with a complete one", () => {
const partial = functionDossierSchema.parse({
...withLiterals("0x2000", ["SleepDisabled"]),
swift_string_literal_omissions: [
{
source_address: "0x2000",
representation: "large",
address: "0x3000",
byte_length: 64,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0xd000000000000040", "0x8000000000002fe0"],
},
],
});
expect(
stringsDimension(
"9",
withLiterals("0x1000", ["SleepDisabled"]),
"a",
partial,
),
).toMatchObject({ status: "unknown" });
});
});
const expectInlineComparisonEvidence = (
+29 -20
View File
@@ -139,20 +139,12 @@ const compareStringsAndNames = (
right: FunctionSnapshot,
context: ComparisonContext,
): FunctionDimension => {
const leftRecorded = left.dossier.swift_string_literals !== undefined;
const rightRecorded = right.dossier.swift_string_literals !== undefined;
if (leftRecorded !== rightRecorded)
return unresolvedDimension({
dimension: "strings_names",
links: context.links,
leftCount: recordedStringFactCount(left),
rightCount: recordedStringFactCount(right),
limitations: [
"Swift String literal coverage differs; one dossier did not record decoded literals.",
],
});
const leftValues = stringAndNameProjection(left);
const rightValues = stringAndNameProjection(right);
const complete =
swiftLiteralCoverage(left) === swiftLiteralCoverage(right) &&
swiftLiteralCoverage(left) !== "partial";
// Facts both dossiers record decide a change before Swift literal coverage can.
const leftValues = stringAndNameProjection(left, complete);
const rightValues = stringAndNameProjection(right, complete);
if (leftValues === null || rightValues === null)
return unresolvedDimension({
dimension: "strings_names",
@@ -163,15 +155,32 @@ const compareStringsAndNames = (
"Reference source locations could not be normalized to function offsets.",
],
});
return compareValues("strings_names", leftValues, rightValues, context);
const compared = compareValues(
"strings_names",
leftValues,
rightValues,
context,
);
if (complete || compared.status === "changed") return compared;
return unresolvedDimension({
dimension: "strings_names",
links: context.links,
leftCount: null,
rightCount: null,
limitations: [
"Swift String literal coverage differs or is partial; referenced strings and names match, but decoded literals were not compared.",
],
});
};
const recordedStringFactCount = (snapshot: FunctionSnapshot): number | null =>
const swiftLiteralCoverage = (
snapshot: FunctionSnapshot,
): "absent" | "complete" | "partial" =>
snapshot.dossier.swift_string_literals === undefined
? null
: snapshot.dossier.referenced_strings.length +
snapshot.dossier.referenced_names.length +
snapshot.dossier.swift_string_literals.length;
? "absent"
: (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0
? "partial"
: "complete";
const compareReferences = (
left: FunctionSnapshot,
@@ -145,6 +145,7 @@ export const commentProjection = (
export const stringAndNameProjection = (
snapshot: FunctionSnapshot,
includeSwiftLiterals = true,
): readonly unknown[] | null => {
const values = [
...snapshot.dossier.referenced_strings.map(
@@ -167,7 +168,10 @@ export const stringAndNameProjection = (
value,
}),
),
...(snapshot.dossier.swift_string_literals ?? []).map(
...(includeSwiftLiterals
? (snapshot.dossier.swift_string_literals ?? [])
: []
).map(
({
source_address: sourceAddress,
value,
+21
View File
@@ -321,6 +321,21 @@ export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [
}),
]);
/** A large Swift literal whose layout matched but whose text was not decoded. */
export const swiftStringLiteralOmissionSchema = z.strictObject({
source_address: swiftStringLiteralFields.source_address,
representation: z.literal("large"),
address: z.string().describe("Address of the literal's bytes."),
byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
reason: z
.literal("capacity")
.describe(
"The text did not fit the provider's memory budget for this dossier.",
),
registers: swiftStringLiteralFields.registers,
words: swiftStringLiteralFields.words,
});
/** Observed endpoints with complete or partial reference classification evidence. */
export const referenceEdgeSchema = z
.object({
@@ -378,6 +393,12 @@ export const functionDossierSchema = z
.describe(
"Swift String literals decoded from constant register values. Absent when the provider does not decode them for this target.",
),
swift_string_literal_omissions: z
.array(swiftStringLiteralOmissionSchema)
.exactOptional()
.describe(
"Literals that matched the Swift layout but were not decoded. Empty when swift_string_literals is complete for the examined use sites.",
),
basic_blocks: z.array(
z
.object({
+19
View File
@@ -515,6 +515,25 @@ const malformedOutputs = (): Array<
),
],
...malformedSwiftStringLiterals(dossier),
[
"non-canonical Swift literal omission address",
"analyze_function",
{
...dossier,
swift_string_literals: [],
swift_string_literal_omissions: [
{
source_address: "0x401004",
representation: "large",
address: "402240",
byte_length: 39,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0xd000000000000027", "0x8000000000402220"],
},
],
},
],
[
"non-canonical native API target",
"analyze_function",
+3
View File
@@ -311,6 +311,9 @@ const ghidraFunctionDossier = functionDossierSchema
({ address, source_address }) =>
address === null ? [source_address] : [address, source_address],
),
...(value.swift_string_literal_omissions ?? []).flatMap(
({ address, source_address }) => [address, source_address],
),
...value.basic_blocks.flatMap(({ start, end, successors }) => [
start,
end,
@@ -10,8 +10,14 @@ func unicodeLiterals() {
consume("Résumé of the naïve décor, written twice")
}
@inline(never)
func emptyAndControlLiterals() {
consume("")
consume("tab\there")
}
@inline(never)
func returnedLiteral() -> String { "Holding awake" }
smallLiteral()
largeLiteral()
unicodeLiterals()
emptyAndControlLiterals()
print(returnedLiteral(), CommandLine.arguments.count)