fix(ghidra): supervise Swift probes and cover retained text capacity

This commit is contained in:
morluto
2026-10-13 22:57:42 +08:00
236 changed files with 12283 additions and 2007 deletions
@@ -60,6 +60,78 @@ public class ReaJadxMetadataRegression {
return apk;
}
private static void verifyDirectInvokes(SessionHolder holder, SessionConfig config,
Constructor<?> constructor, Method handle, Path sources, Path output) throws Exception {
SmaliOptions options = new SmaliOptions();
options.apiLevel = 28;
options.jobs = 1;
options.outputDexFile = output.resolve("direct-invokes.dex").toString();
require(Smali.assemble(options, List.of(
sources.resolve("Caller.smali").toString(), sources.resolve("Target.smali").toString())),
"Direct-invoke fixture assembly failed");
Path apk = output.resolve("direct-invokes.apk");
try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(apk))) {
zip.putNextEntry(new ZipEntry("classes.dex"));
Files.copy(Path.of(options.outputDexFile), zip);
zip.closeEntry();
}
BuildersKt.runBlocking(EmptyCoroutineContext.INSTANCE,
(scope, continuation) -> holder.unload(continuation));
BuildersKt.runBlocking(EmptyCoroutineContext.INSTANCE,
(scope, continuation) -> holder.load(apk.toString(), config, continuation));
JsonObject refs = call(handle, constructor.newInstance(holder), "get_xrefs_to_method",
methodArguments("fixture.Target", "mark"));
checkDirectInvokes(refs, "classes.dex");
SmaliOptions targetOptions = new SmaliOptions();
targetOptions.apiLevel = 28;
targetOptions.jobs = 1;
targetOptions.outputDexFile = output.resolve("direct-target.dex").toString();
SmaliOptions callerOptions = new SmaliOptions();
callerOptions.apiLevel = 28;
callerOptions.jobs = 1;
callerOptions.outputDexFile = output.resolve("direct-caller.dex").toString();
require(Smali.assemble(targetOptions, List.of(sources.resolve("Target.smali").toString())),
"Target DEX assembly failed");
require(Smali.assemble(callerOptions, List.of(sources.resolve("Caller.smali").toString())),
"Caller DEX assembly failed");
Path splitApk = output.resolve("direct-invokes-multidex.apk");
try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(splitApk))) {
zip.putNextEntry(new ZipEntry("classes.dex"));
Files.copy(Path.of(targetOptions.outputDexFile), zip);
zip.closeEntry();
zip.putNextEntry(new ZipEntry("classes2.dex"));
Files.copy(Path.of(callerOptions.outputDexFile), zip);
zip.closeEntry();
}
BuildersKt.runBlocking(EmptyCoroutineContext.INSTANCE,
(scope, continuation) -> holder.unload(continuation));
BuildersKt.runBlocking(EmptyCoroutineContext.INSTANCE,
(scope, continuation) -> holder.load(splitApk.toString(), config, continuation));
JsonObject crossDex = call(handle, constructor.newInstance(holder), "get_xrefs_to_method",
methodArguments("fixture.Target", "mark"));
checkDirectInvokes(crossDex, "classes2.dex");
System.out.println("DIRECT_INVOKE_REGRESSION_PASS nodes=1 offsets=0,3 cross_dex=true");
}
private static void checkDirectInvokes(JsonObject refs, String dexEntry) {
require(refs.get("total").getAsInt() == 1, "JADX using-node count changed");
JsonArray sites = refs.getAsJsonArray("items").get(0).getAsJsonObject()
.getAsJsonArray("direct_invokes");
require(sites.size() == 2, "Raw DEX invoke count differs from the fixture: " + sites);
for (int i = 0; i < sites.size(); i++) {
JsonObject site = sites.get(i).getAsJsonObject();
require(site.get("dex_entry").getAsString().equals(dexEntry), "Wrong DEX entry");
require(site.get("source_raw_method_id").getAsString().equals("fixture.Caller.twice()V"),
"Wrong raw source method");
require(site.get("dex_code_unit_offset").getAsInt() == i * 3, "Wrong DEX offset");
require(site.get("opcode").getAsString().equals("INVOKE_STATIC"), "Wrong opcode");
}
require(sites.get(0).getAsJsonObject().get("dex_method_index").getAsInt()
== sites.get(1).getAsJsonObject().get("dex_method_index").getAsInt(),
"Two invokes did not reference the same DEX method ID");
}
private static JsonObject methodArguments(String className, String methodName) {
JsonObject value = arguments(className);
value.addProperty("method_name", methodName);
@@ -152,13 +224,20 @@ public class ReaJadxMetadataRegression {
JsonObject componentArgs = arguments("androidx.activity.ComponentActivity");
JsonObject componentSummary = call(handle, bridge, "get_class_summary", componentArgs);
List<String> signatures = new ArrayList<>();
List<String> descriptors = new ArrayList<>();
for (JsonElement entry : componentSummary.getAsJsonArray("methods")) {
JsonObject method = entry.getAsJsonObject();
if (method.get("name").getAsString().equals("setContentView"))
if (method.get("name").getAsString().equals("setContentView")) {
signatures.add(method.get("signature").getAsString());
descriptors.add(method.get("dex_descriptor").getAsString());
}
}
require(signatures.size() == 3, "Fixture must contain three setContentView overloads");
require(descriptors.equals(List.of("(I)V", "(Landroid/view/View;)V",
"(Landroid/view/View;Landroid/view/ViewGroup$LayoutParams;)V")),
"Frozen DEX prototypes did not identify the three overloads: " + descriptors);
String[] declarations = { "setContentView(int ", "setContentView(View ", "setContentView(View view, ViewGroup.LayoutParams " };
List<Integer> referenceCounts = new ArrayList<>();
for (int index = 0; index < signatures.size(); index++) {
JsonObject methodArgs = arguments("androidx.activity.ComponentActivity");
methodArgs.addProperty("method_name", "setContentView");
@@ -169,12 +248,35 @@ public class ReaJadxMetadataRegression {
"Overload envelope mismatch");
require(first.get("mode").getAsString().equals("java"), "Fixture did not produce Java");
require(first.get("body").getAsString().contains(declarations[index]), "Wrong overload: " + signatures.get(index));
JsonObject references = call(handle, bridge, "get_xrefs_to_method", methodArgs);
require(references.get("overload_index").getAsInt() == index,
"Reference result did not preserve selected overload index");
require(references.get("reported_signature").getAsString().equals(signatures.get(index)),
"Reference result did not preserve selected overload signature");
if (index == 0) {
JsonArray invokes = references.getAsJsonArray("items").get(0).getAsJsonObject()
.getAsJsonArray("direct_invokes");
require(invokes.size() == 1, "Selected overload must have one raw direct invoke");
JsonObject invoke = invokes.get(0).getAsJsonObject();
require(invoke.get("source_raw_method_id").getAsString()
.equals("androidx.activity.ComponentActivity.onCreate(Landroid/os/Bundle;)V"),
"Wrong source method for overloaded target");
require(invoke.get("dex_code_unit_offset").getAsInt() == 37
&& invoke.get("opcode").getAsString().equals("INVOKE_VIRTUAL"),
"Wrong direct invoke for selected overloaded target");
}
referenceCounts.add(references.get("count").getAsInt());
}
require(referenceCounts.equals(List.of(1, 0, 0)),
"Incoming references were not isolated to the selected overload: " + referenceCounts);
require(componentSummary.equals(call(handle, bridge, "get_class_summary", componentArgs)),
"Frozen summary changed after code generation");
require(apiSummary.equals(call(handle, bridge, "get_class_summary", apiArgs)), "Unrelated frozen summary changed");
System.out.println("METADATA_REGRESSION_PASS classes=" + states.size() + " overloads=" + signatures.size());
System.out.println("METADATA_REGRESSION_PASS classes=" + states.size() + " overloads=" + signatures.size()
+ " reference_counts=" + referenceCounts);
verifyInitializers(holder, config, constructor, handle, initializerFixture(args[1], args[2]));
verifyDirectInvokes(holder, config, constructor, handle,
Path.of(args[3]), Path.of(args[2]));
} finally {
BuildersKt.runBlocking(EmptyCoroutineContext.INSTANCE,
(scope, continuation) -> holder.unload(continuation));
+83 -1
View File
@@ -160,7 +160,89 @@ export async function inspectMipsReadelf(path, byteOrder, floatAbi = "hard") {
};
}
/** Check effective Ghidra observations, not the requested analysis profile. */
/** Independent GNU readelf interpretation for the recordless legacy fixture. */
export async function inspectLegacyMipsReadelf(path, byteOrder) {
const command = process.env.REA_MIPS_READELF ?? "readelf";
const options = {
env: { ...process.env, LC_ALL: "C" },
timeout: 10000,
maxBuffer: 1024 * 1024,
};
const version = await execute(command, ["--version"], options);
assert.match(
version.stdout,
/^GNU readelf\b/u,
"This lane requires GNU readelf",
);
const header = await execute(command, ["-h", "-W", path], options);
assert.equal(
header.stderr.trim(),
"",
`readelf diagnostics: ${header.stderr}`,
);
const field = (expression, label) => {
const matches = [...header.stdout.matchAll(expression)];
assert.equal(matches.length, 1, `Missing or ambiguous readelf ${label}`);
return matches[0][1].trim();
};
assert.equal(field(/^\s*Class:\s*(.+)$/gmu, "class"), "ELF32");
assert.equal(
field(/^\s*Data:.*?,\s*(little|big) endian\s*$/gmu, "byte order"),
byteOrder,
);
assert.match(field(/^\s*Type:\s*(.+)$/gmu, "type"), /^EXEC\b/u);
assert.match(field(/^\s*Machine:\s*(.+)$/gmu, "machine"), /^MIPS\b/u);
const flags = field(/^\s*Flags:\s*(0x[\da-f]+)\b.*$/gmu, "flags");
assert.equal(
Number(flags),
0x50000001,
"Legacy fixture must carry EF_MIPS_ARCH_32 with an unspecified ABI field",
);
const abi = await execute(command, ["-A", path], options);
assert.equal(abi.stderr.trim(), "", `readelf diagnostics: ${abi.stderr}`);
// -A also prints unrelated MIPS facts such as the static GOT; only the ABI
// flags declaration itself must be gone from the recordless fixture.
assert.doesNotMatch(
abi.stdout,
/MIPS ABI Flags Version/u,
"GNU readelf still observes an ABI flags declaration in the legacy fixture",
);
const symbols = await execute(command, ["-s", "-W", path], options);
assert.equal(
symbols.stderr.trim(),
"",
`readelf diagnostics: ${symbols.stderr}`,
);
const parsed = {};
for (const name of ["rea_mips_entry", "rea_mips_probe"]) {
const matches = [
...symbols.stdout.matchAll(
new RegExp(
`^\\s*\\d+:\\s+([\\da-f]+)\\s+\\d+\\s+FUNC\\s+GLOBAL\\s+\\S+\\s+\\d+\\s+${name}\\s*$`,
"gmu",
),
),
];
assert.equal(
matches.length,
1,
`Missing or ambiguous readelf symbol ${name}`,
);
parsed[name] = `0x${BigInt(`0x${matches[0][1]}`).toString(16)}`;
}
return {
flags: Number(flags),
limitations: [],
symbols: parsed,
command,
version: version.stdout.split("\n")[0],
raw: {
header: header.stdout,
abi: abi.stdout,
symbols: symbols.stdout,
},
};
}
export function assertMipsLoadedImage(observations, byteOrder, sha256) {
assert.equal(observations.language_id, language(byteOrder));
assert.equal(observations.compiler_spec_id, "default");
+97 -6
View File
@@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
@@ -12,11 +12,13 @@ export async function buildMipsFixture(
directory,
byteOrder,
floatAbi = "hard",
options = {},
) {
if (byteOrder !== "little" && byteOrder !== "big")
throw new Error("MIPS fixture byte order must be little or big");
if (floatAbi !== "hard" && floatAbi !== "soft")
throw new Error("MIPS fixture float ABI must be hard or soft");
const legacy = options.legacy === true;
const soft = floatAbi === "soft";
const target = byteOrder === "little" ? "mipsel-linux-gnu" : "mips-linux-gnu";
const source = fileURLToPath(new URL("../c/mips.c", import.meta.url));
@@ -24,10 +26,13 @@ export async function buildMipsFixture(
new URL("../c/mips-soft-float.c", import.meta.url),
);
const probe = fileURLToPath(new URL("./mips-probe.S", import.meta.url));
const path = join(directory, `${target}${soft ? "-soft-float" : ""}.elf`);
const path = join(
directory,
`${target}${soft ? "-soft-float" : ""}${legacy ? "-legacy" : ""}.elf`,
);
const args = [
`--target=${target}`,
"-march=mips32r2",
`-march=${legacy ? "mips32" : "mips32r2"}`,
"-mabi=32",
...(soft ? ["-msoft-float"] : []),
"-mno-abicalls",
@@ -45,8 +50,88 @@ export async function buildMipsFixture(
"-o",
path,
];
const compiled = await compileMipsFixture(path, byteOrder, 2, args, {
expectedFlags: legacy ? 0x50001001 : 0x70001001,
});
// Legacy variant: retype the ABI record declarations out of the image and
// zero the header ABI field, matching pre-ABI-flags 2000s firmware toolchains
// (their compilers predate the record; the header leaves the ABI unspecified).
let attested = compiled;
if (legacy) {
const raw = await readFile(path);
const write16 = (offset, value) =>
byteOrder === "little"
? raw.writeUInt16LE(value, offset)
: raw.writeUInt16BE(value, offset);
const write32 = (offset, value) =>
byteOrder === "little"
? raw.writeUInt32LE(value, offset)
: raw.writeUInt32BE(value, offset);
const read16 = (offset) =>
byteOrder === "little"
? raw.readUInt16LE(offset)
: raw.readUInt16BE(offset);
const read32 = (offset) =>
byteOrder === "little"
? raw.readUInt32LE(offset)
: raw.readUInt32BE(offset);
const flags = read32(36);
if ((flags & 0xf0000000) !== 0x50000000)
throw new Error("Legacy fixture did not build as EF_MIPS_ARCH_32");
write32(36, flags & ~0x0000f000);
const programOffset = read32(28);
const programCount = read16(44);
if (read16(42) !== 32)
throw new Error("Legacy fixture has non-standard program headers");
for (let index = 0; index < programCount; index += 1) {
const offset = programOffset + index * 32;
if (read32(offset) === 0x70000003) write32(offset, 0);
}
const sectionOffset = read32(32);
const sectionCount = read16(48);
if (read16(46) !== 40)
throw new Error("Legacy fixture has non-standard section headers");
for (let index = 0; index < sectionCount; index += 1) {
const offset = sectionOffset + index * 40;
if (read32(offset + 4) === 0x7000002a) write32(offset + 4, 1);
}
// GNU readelf resolves the record by section name, not type, so a retyped
// header alone keeps a named declaration. Replace the name in the string
// table in place, preserving every remaining name and offset.
const stringTableIndex = read16(50);
const stringTableOffset =
stringTableIndex < sectionCount
? read32(sectionOffset + stringTableIndex * 40 + 16)
: -1;
const named =
stringTableOffset === -1
? -1
: raw.indexOf(
Buffer.from(".MIPS.abiflags", "ascii"),
stringTableOffset,
);
if (named === -1)
throw new Error("Legacy fixture lacks the expected ABI section name");
raw.write("x.rea.no.flags", named, "ascii");
await writeFile(path, raw);
const patched = await readFile(path);
const patchedFlags =
byteOrder === "little"
? patched.readUInt32LE(36)
: patched.readUInt32BE(36);
if (patchedFlags !== 0x50000001)
throw new Error(
"Legacy fixture did not end with the recordless o32 declaration",
);
attested = {
...compiled,
flags: patchedFlags,
sha256: createHash("sha256").update(patched).digest("hex"),
};
}
return {
...(await compileMipsFixture(path, byteOrder, 2, args)),
...attested,
legacy,
float_abi: floatAbi,
source_sha256: createHash("sha256")
.update(await readFile(source))
@@ -108,7 +193,13 @@ export async function buildMipsRelocatableFixture(
}
/** Compile and attest the shared ELF32/o32/MIPS32r2 fixture boundary. */
async function compileMipsFixture(path, byteOrder, objectType, args) {
async function compileMipsFixture(
path,
byteOrder,
objectType,
args,
options = {},
) {
const compiler = process.env.REA_MIPS_CLANG ?? "clang";
const version = await execute(compiler, ["--version"], { timeout: 10000 });
await execute(compiler, args, { timeout: 60000 });
@@ -128,7 +219,7 @@ async function compileMipsFixture(path, byteOrder, objectType, args) {
bytes[5] !== (byteOrder === "little" ? 1 : 2) ||
read16(16) !== objectType ||
read16(18) !== 8 ||
read32(36) !== 0x70001001
read32(36) !== (options.expectedFlags ?? 0x70001001)
)
throw new Error(
"Compiler did not produce the declared ELF32/o32/MIPS32r2 fixture",
+25 -20
View File
@@ -4,26 +4,6 @@ import { resolve } from "node:path";
import { describe, expect, it } from "vitest";
import { PRODUCT_IDENTITY } from "../../src/identity.js";
const translatedReadmes = [
"README_zh.md",
"README_zh-TW.md",
"README_ja.md",
"README_ko.md",
"README_tr.md",
"README_ru.md",
"README_vi.md",
"README_th.md",
"README_de.md",
"README_es.md",
"README_fr.md",
"README_uk.md",
"README_pl.md",
"README_pt-BR.md",
"README_ar.md",
"README_fa.md",
"README_id.md",
] as const;
const jsonExamples = (content: string): unknown[] =>
[...content.matchAll(/```json\s*([\s\S]*?)```/gu)].map((match): unknown =>
JSON.parse(match[1] ?? ""),
@@ -64,6 +44,14 @@ describe("onboarding documentation product facts", () => {
});
it("keeps translated onboarding commands aligned with the English examples", async () => {
const inventory = JSON.parse(
await readFile(resolve("website/locales/languages.json"), "utf8"),
) as { code: string; readme: string }[];
const translatedReadmes = inventory
.filter((language) => language.code !== "en")
.map((language) => language.readme);
expect(translatedReadmes).toContain("docs/readme/README_it.md");
expect(translatedReadmes).toHaveLength(18);
const english = shellExamples(await readFile(resolve("README.md"), "utf8"));
for (const path of translatedReadmes)
expect(
@@ -71,4 +59,21 @@ describe("onboarding documentation product facts", () => {
path,
).toEqual(english);
});
it("preserves Persian joiners in links to GitHub-rendered headings", async () => {
const persian = await readFile(resolve("docs/readme/README_fa.md"), "utf8");
// Confirmed against GitHub's rendered README, which retains U+200C.
for (const heading of [
"REA چگونه کار می‌کند",
"چه چیزهایی را می‌توان تحلیل کرد",
"نمونه‌های عملی",
"پرسش‌های متداول",
"به‌روزرسانی REA",
]) {
expect(persian).toContain(`## ${heading}`);
expect(persian).toContain(
`](#${heading.toLowerCase().replaceAll(" ", "-")})`,
);
}
});
});
@@ -52,15 +52,22 @@ public class ReaSwiftDecoderRegressions extends HeadlessScript {
"malformed small-string UTF-8 was accepted");
rejectLarge("rea_swift_decoder_bad_terminator", 0xd000000000000003L);
rejectLarge("rea_swift_decoder_bad_large_utf8", 0x1000000000000001L);
Object exhausted = decode(copied, 0);
require(literals(exhausted).isEmpty(), "capacity failure published decoded small text");
// Zero covers provenance-state exhaustion; 8192 admits both register
// cache entries and then rejects the retained literal/text charge.
assertCapacityOmission(copied, 0);
assertCapacityOmission(copied, 8192);
require(getScriptArgs().length == 1, "Expected owned result path");
Files.writeString(Path.of(getScriptArgs()[0]), "REA_SWIFT_DECODER_SAFEGUARDS_OK\n");
}
private void assertCapacityOmission(Function function, long remaining) throws Exception {
Object exhausted = decode(function, remaining);
require(literals(exhausted).isEmpty(), "capacity failure published decoded small text at budget " + remaining);
JsonArray omitted = omissions(exhausted);
require(omitted.size() == 1, "capacity failure did not stop subsequent use sites");
require(omitted.size() == 1, "capacity failure did not stop subsequent use sites at budget " + remaining);
JsonObject omission = omitted.get(0).getAsJsonObject();
require(omission.get("reason").getAsString().equals("capacity"), "capacity omission lost its reason");
require(!omission.has("value"), "omission claimed a retained literal value");
require(getScriptArgs().length == 1, "Expected owned result path");
Files.writeString(Path.of(getScriptArgs()[0]), "REA_SWIFT_DECODER_SAFEGUARDS_OK\n");
}
private Object decode(Function function, long remaining) throws Exception {