leefer/rea
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(ghidra): bound literal records and trace selects across branches

Browse Source 
Every Swift literal record now pays a fixed cost against the dossier
budget in addition to its text, so many empty or small literals are
bounded as well. The first record that does not fit is kept as an
omission and decoding stops there, which keeps the omission record
itself bounded; omissions now describe small or large literals.

The conditional select check searches every predecessor path for the
reaching definitions, following register copies, so selects made
before a branch into the use block are recognized.

A dossier records Swift literals and their omissions together. Coverage
reads omissions first, and strings_names compares every use site that
neither side omitted: a decoded literal is known where it was recorded,
and an absence is known only in a dossier whose decoding examined every
use site.
This commit is contained in:
Omisakin Joshua
2026-10-12 11:40:02 +01:00
parent d5b2edc217
commit 560257b00e
7 changed files with 193 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bridge/ghidra/ReaGhidraBridge.java
+63 -29
View File
@@ -24,6 +24,7 @@ import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.PosixFilePermissions;
import java.security.MessageDigest;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -3356,22 +3357,21 @@ public final class ReaGhidraBridge extends HeadlessScript {
SwiftLiteralLayout layout = swiftLiteralLayout(word, object);
// Ghidra forks each select without correlating its condition, so a pair whose
// registers are both selected may combine words from different cases.
if (layout == null || (selectedValue(instruction, pair[0]) && selectedValue(instruction, pair[1]))) continue;
if (layout == null ||
(selectedValue(function, instruction, pair[0]) && selectedValue(function, instruction, pair[1]))) continue;
// Forks can reach one site with different constants, so the words are part of the key.
String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() +
"\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) +
"\u0000" + Long.toUnsignedString(object, 16);
if (observed.containsKey(key) || omitted.containsKey(key)) continue;
// Small text is at most 15 bytes and already decoded; only large reads draw on the budget.
// Every record draws on the budget; the first one that does not fit is kept as an
// omission and decoding stops, so later sites are never examined.
String value = layout.smallValue();
boolean overCapacity = false;
if (value == null && !budget.take(layout.byteLength())) {
overCapacity = true;
} else if (value == null) {
boolean overCapacity = !budget.take(layout.byteLength());
if (!overCapacity && value == null) {
try {
value = swiftLargeLiteralText(layout);
} catch (OutOfMemoryError exhausted) {
budget.exhaust();
overCapacity = true;
}
}
@@ -3385,6 +3385,7 @@ public final class ReaGhidraBridge extends HeadlessScript {
pairRegisters.add(pair[1].getName());
item.add("registers", pairRegisters);
(value == null ? omitted : observed).put(key, item);
if (overCapacity) return true;
}
return false;
}
@@ -3435,32 +3436,59 @@ public final class ReaGhidraBridge extends HeadlessScript {
return value != null && context.isConstant(value) ? value.getOffset() : null;
}
// Follow register copies back to the defining instruction, since a select copied into an
// argument register forks just as independently as one written there directly.
private boolean selectedValue(Instruction use, Register register) {
Register target = register.getBaseRegister();
Address from = use.getFallFrom();
Instruction instruction = from == null ? null : currentProgram.getListing().getInstructionAt(from);
while (instruction != null) {
// Search every predecessor path for the reaching definitions, following register copies,
// since a select copied into an argument register or defined before a branch forks just as
// independently as one written there directly.
private boolean selectedValue(Function function, Instruction use, Register register) {
ArrayDeque<Instruction> instructions = new ArrayDeque<>();
ArrayDeque<Register> targets = new ArrayDeque<>();
Set<String> visited = new HashSet<>();
for (Instruction predecessor : predecessors(function, use)) {
instructions.push(predecessor);
targets.push(register.getBaseRegister());
}
while (!instructions.isEmpty()) {
Instruction instruction = instructions.pop();
Register target = targets.pop();
if (!visited.add(canonicalAddress(instruction.getAddress()) + "\u0000" + target.getName())) continue;
boolean defines = false;
for (Object result : instruction.getResultObjects()) {
if (result instanceof Register written && written.getBaseRegister().equals(target)) defines = true;
}
Register next = target;
if (defines) {
PcodeOp[] operations = instruction.getPcode();
for (PcodeOp operation : operations) {
if (operation.getOpcode() == PcodeOp.CBRANCH && operation.getInput(0).isConstant()) return true;
}
Register source = copiedRegister(operations);
if (source == null) return false;
target = source.getBaseRegister();
if (source == null) continue;
next = source.getBaseRegister();
}
for (Instruction predecessor : predecessors(function, instruction)) {
instructions.push(predecessor);
targets.push(next);
}
from = instruction.getFallFrom();
instruction = from == null ? null : currentProgram.getListing().getInstructionAt(from);
}
return false;
}
private List<Instruction> predecessors(Function function, Instruction instruction) {
List<Instruction> result = new ArrayList<>();
Address from = instruction.getFallFrom();
if (from != null) {
Instruction previous = currentProgram.getListing().getInstructionAt(from);
if (previous != null) result.add(previous);
}
for (Reference reference : currentProgram.getReferenceManager().getReferencesTo(instruction.getAddress())) {
if (!reference.getReferenceType().isFlow() || reference.getReferenceType().isCall()) continue;
if (!function.getBody().contains(reference.getFromAddress())) continue;
Instruction source = currentProgram.getListing().getInstructionAt(reference.getFromAddress());
if (source != null) result.add(source);
}
return result;
}
private Register copiedRegister(PcodeOp[] operations) {
if (operations.length != 1) return null;
int opcode = operations[0].getOpcode();
@@ -3518,13 +3546,16 @@ public final class ReaGhidraBridge extends HeadlessScript {
}
/**
* Every large literal of one dossier draws on half of the heap that was free when decoding
* began, charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up to six
* escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer (up to
* twice its length), again in the returned String, and once more as UTF-8 on the socket.
* Every literal record of one dossier draws on half of the heap that was free when decoding
* began. Text is charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up
* to six escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer
* (up to twice its length), again in the returned String, and once more as UTF-8 on the
* socket. Each record also pays a fixed cost for its map key, JSON members, addresses, words
* and their serialized form, so many empty or small literals are bounded too.
*/
private static final class SwiftLiteralBudget {
private static final long BYTES_PER_LITERAL_BYTE = 2 + 6 * 2 * 2 + 6 * 2 + 6;
private static final long BYTES_PER_RECORD = 4096;
private long remaining;
SwiftLiteralBudget() {
@@ -3533,14 +3564,13 @@ public final class ReaGhidraBridge extends HeadlessScript {
}
boolean take(long bytes) {
if (bytes > remaining / BYTES_PER_LITERAL_BYTE) return false;
remaining -= bytes * BYTES_PER_LITERAL_BYTE;
if (remaining < BYTES_PER_RECORD ||
bytes > (remaining - BYTES_PER_RECORD) / BYTES_PER_LITERAL_BYTE) {
return false;
}
remaining -= BYTES_PER_RECORD + bytes * BYTES_PER_LITERAL_BYTE;
return true;
}
void exhaust() {
remaining = 0;
}
}
// Swift String values may contain any scalar, including controls; JSON escapes them.
@@ -3559,7 +3589,11 @@ public final class ReaGhidraBridge extends HeadlessScript {
private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) {
JsonObject item = new JsonObject();
item.addProperty("representation", layout.representation());
item.addProperty("address", canonicalAddress(layout.address()));
if (layout.address() == null) {
item.add("address", JsonNull.INSTANCE);
} else {
item.addProperty("address", canonicalAddress(layout.address()));
}
item.addProperty("byte_length", layout.byteLength());
item.addProperty("reason", "capacity");
item.add("words", swiftStringWords(word, object));
src/domain/functionComparison.test.ts
+63 -23
View File
@@ -572,30 +572,30 @@ describe("function comparison", () => {
});
});
describe("function comparison Swift string literals", () => {
const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) =>
functionDossierSchema.parse({
...dossier("return 0;", base),
swift_string_literals: values.map((value) => ({
source_address: base,
value,
representation: "small" as const,
address: null,
registers: ["x0", "x1"] as const,
words: ["0x0", "0xe"] as const,
})),
});
const stringsDimension = (
leftDigit: string,
left: ReturnType<typeof withLiterals> | ReturnType<typeof dossier>,
rightDigit: string,
right: ReturnType<typeof withLiterals> | ReturnType<typeof dossier>,
) =>
compareFunctions(
observe(leftDigit, left),
observe(rightDigit, right),
).dimensions.find(({ dimension }) => dimension === "strings_names");
const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) =>
functionDossierSchema.parse({
...dossier("return 0;", base),
swift_string_literals: values.map((value) => ({
source_address: base,
value,
representation: "small" as const,
address: null,
registers: ["x0", "x1"] as const,
words: ["0x0", "0xe"] as const,
})),
});
const stringsDimension = (
leftDigit: string,
left: ReturnType<typeof withLiterals> | ReturnType<typeof dossier>,
rightDigit: string,
right: ReturnType<typeof withLiterals> | ReturnType<typeof dossier>,
) =>
compareFunctions(
observe(leftDigit, left),
observe(rightDigit, right),
).dimensions.find(({ dimension }) => dimension === "strings_names");
describe("function comparison Swift string literals", () => {
it("treats differing decoded literals as a strings and names change", () => {
expect(
stringsDimension(
@@ -671,6 +671,46 @@ describe("function comparison Swift string literals", () => {
).toMatchObject({ status: "unknown" });
});
it("reports a literal that a complete dossier shows absent at a site the partial one decoded", () => {
const partial = functionDossierSchema.parse({
...withLiterals("0x2000", ["SleepDisabled"]),
swift_string_literal_omissions: [
{
source_address: "0x2004",
representation: "small",
address: null,
byte_length: 4,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0x732f4220", "0xe400000000000000"],
},
],
});
expect(
stringsDimension("d", withLiterals("0x1000", []), "e", partial),
).toMatchObject({ status: "changed" });
});
it("treats an omission record without decoded literals as partial coverage", () => {
const omissionOnly = functionDossierSchema.parse({
...dossier("return 0;", "0x2000"),
swift_string_literal_omissions: [
{
source_address: "0x2004",
representation: "small",
address: null,
byte_length: 4,
reason: "capacity",
registers: ["x0", "x1"],
words: ["0x732f4220", "0xe400000000000000"],
},
],
});
expect(
stringsDimension("f", dossier("return 0;", "0x1000"), "0", omissionOnly),
).toMatchObject({ status: "unknown" });
});
it("reports a conflicting literal at a shared use site despite partial coverage", () => {
const partial = (base: "0x1000" | "0x2000", value: string) =>
functionDossierSchema.parse({
src/domain/functionComparisonDimensions.ts
+4 -4
View File
@@ -184,10 +184,10 @@ const compareStringsAndNames = (
const swiftLiteralCoverage = (
snapshot: FunctionSnapshot,
): "absent" | "complete" | "partial" =>
snapshot.dossier.swift_string_literals === undefined
? "absent"
: (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0
? "partial"
(snapshot.dossier.swift_string_literal_omissions ?? []).length > 0
? "partial"
: snapshot.dossier.swift_string_literals === undefined
? "absent"
: "complete";
const compareReferences = (
src/domain/functionComparisonNormalization.ts
+24 -10
View File
@@ -194,37 +194,48 @@ export const stringAndNameProjection = (
: sorted(values);
};
/** Whether both dossiers decoded different literals at one use site that neither left partial. */
/**
* Whether one use site is known to hold different literals in the two dossiers. A decoded
* literal is known wherever it was recorded; an absence is known only in a dossier whose
* decoding examined every use site, and the site where decoding stopped is never known.
*/
export const swiftLiteralSitesConflict = (
left: FunctionSnapshot,
right: FunctionSnapshot,
): boolean => {
const leftSites = swiftLiteralSites(left);
const rightSites = swiftLiteralSites(right);
for (const [site, values] of leftSites.values) {
const other = rightSites.values.get(site);
if (other === undefined || rightSites.omitted.has(site)) continue;
if (canonicalJson(values) !== canonicalJson(other)) return true;
for (const site of new Set([
...leftSites.values.keys(),
...rightSites.values.keys(),
])) {
if (leftSites.omitted.has(site) || rightSites.omitted.has(site)) continue;
const leftValues =
leftSites.values.get(site) ?? (leftSites.complete ? [] : undefined);
const rightValues =
rightSites.values.get(site) ?? (rightSites.complete ? [] : undefined);
if (leftValues === undefined || rightValues === undefined) continue;
if (canonicalJson(leftValues) !== canonicalJson(rightValues)) return true;
}
return false;
};
const swiftLiteralSites = (snapshot: FunctionSnapshot) => {
const base = snapshot.dossier.procedure.address;
const omissions = snapshot.dossier.swift_string_literal_omissions ?? [];
const site = (sourceAddress: string, registers: readonly string[]) => {
const offset = relativeAddress(sourceAddress, base);
return offset === null ? null : canonicalJson([offset, registers]);
};
const omitted = new Set(
(snapshot.dossier.swift_string_literal_omissions ?? []).map(
({ source_address: sourceAddress, registers }) =>
site(sourceAddress, registers),
omissions.map(({ source_address: sourceAddress, registers }) =>
site(sourceAddress, registers),
),
);
const values = new Map<string, string[]>();
for (const literal of snapshot.dossier.swift_string_literals ?? []) {
const key = site(literal.source_address, literal.registers);
if (key === null || omitted.has(key)) continue;
if (key === null) continue;
values.set(
key,
[...(values.get(key) ?? []), literal.value].sort(
@@ -232,7 +243,10 @@ const swiftLiteralSites = (snapshot: FunctionSnapshot) => {
),
);
}
return { values, omitted };
const complete =
snapshot.dossier.swift_string_literals !== undefined &&
omissions.length === 0;
return { values, omitted, complete };
};
export const sorted = (values: readonly unknown[]): readonly unknown[] =>
src/domain/hopperValues.ts
+21 -7
View File
@@ -321,20 +321,34 @@ export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [
}),
]);
/** A large Swift literal whose layout matched but whose text was not decoded. */
export const swiftStringLiteralOmissionSchema = z.strictObject({
const swiftStringLiteralOmissionFields = {
source_address: swiftStringLiteralFields.source_address,
representation: z.literal("large"),
address: z.string().describe("Address of the literal's bytes."),
byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
reason: z
.literal("capacity")
.describe(
"The text did not fit the provider's memory budget for this dossier.",
"The record did not fit the provider's memory budget for this dossier; decoding stopped here, so later use sites were not examined.",
),
registers: swiftStringLiteralFields.registers,
words: swiftStringLiteralFields.words,
});
};
/** A Swift literal whose layout matched but whose record did not fit; decoding stopped there. */
export const swiftStringLiteralOmissionSchema = z.discriminatedUnion(
"representation",
[
z.strictObject({
...swiftStringLiteralOmissionFields,
representation: z.literal("small"),
address: z.null(),
}),
z.strictObject({
...swiftStringLiteralOmissionFields,
representation: z.literal("large"),
address: z.string().describe("Address of the literal's bytes."),
}),
],
);
/** Observed endpoints with complete or partial reference classification evidence. */
export const referenceEdgeSchema = z
@@ -397,7 +411,7 @@ export const functionDossierSchema = z
.array(swiftStringLiteralOmissionSchema)
.exactOptional()
.describe(
"Literals that matched the Swift layout but were not decoded. Empty when swift_string_literals is complete for the examined use sites.",
"The literal at which decoding stopped for capacity. Present exactly when swift_string_literals is present; empty when decoding examined every use site.",
),
basic_blocks: z.array(
z
src/ghidra/GhidraFunctionValues.test.ts
+7
View File
@@ -210,6 +210,7 @@ describe("Ghidra Swift String literals", () => {
parseFunctionValueFixture("analyze_function", {
...dossier,
swift_string_literals: literals,
swift_string_literal_omissions: [],
}),
).toMatchObject({ ok: true, value: { swift_string_literals: literals } });
});
@@ -457,6 +458,7 @@ const malformedSwiftStringLiterals = (
"analyze_function",
{
...dossier,
swift_string_literal_omissions: [],
swift_string_literals: [
{
source_address: "0x401004",
@@ -484,6 +486,11 @@ const malformedOutputs = (): Array<
throw new TypeError("Ghidra native API fixture is invalid");
return [
["non-canonical address", "procedure_callers", ["401000"]],
[
"Swift literals without their omission record",
"analyze_function",
{ ...dossier, swift_string_literals: [] },
],
[
"missing classification",
"procedure_info",
src/ghidra/GhidraFunctionValues.ts
+11 -1
View File
@@ -292,6 +292,15 @@ const ghidraFunctionDossier = functionDossierSchema
})
.strict()
.superRefine((value, context) => {
if (
(value.swift_string_literals === undefined) !==
(value.swift_string_literal_omissions === undefined)
)
context.addIssue({
code: "custom",
message:
"Ghidra dossier must record Swift literals and their omissions together",
});
const addresses = [
value.procedure.address,
...value.comments.map(({ address }) => address),
@@ -312,7 +321,8 @@ const ghidraFunctionDossier = functionDossierSchema
address === null ? [source_address] : [address, source_address],
),
...(value.swift_string_literal_omissions ?? []).flatMap(
({ address, source_address }) => [address, source_address],
({ address, source_address }) =>
address === null ? [source_address] : [address, source_address],
),
...value.basic_blocks.flatMap(({ start, end, successors }) => [
start,