fix(cli): stream large JavaScript JSON results (#1024)

* fix(javascript): report application analysis progress on the CLI

* test(cli): preserve progress in filtered output checks

* fix(cli): stream large JavaScript JSON results

* test(cli): include streaming output in dispatcher fixture
This commit is contained in:
Ysopwnable
2026-10-08 03:44:51 +08:00
committed by GitHub
parent ed725dace4
commit 0e25befe78
16 changed files with 781 additions and 34 deletions
+3 -1
View File
@@ -2,12 +2,14 @@ import type { Logger } from "../logger.js";
import { registerCoreBinaryCommands } from "./coreBinaryCommands.js";
import { registerCoreNativeCommands } from "./coreNativeCommands.js";
import type { CliInstance } from "./types.js";
import type { CliResultOutput } from "./streamedJsonOutput.js";
/** Register core binary and native deep-analysis CLI commands. */
export const registerCoreAnalysisCommands = (
cli: CliInstance,
logger: Logger,
resultOutput?: CliResultOutput,
): void => {
registerCoreBinaryCommands(cli, logger);
registerCoreBinaryCommands(cli, logger, resultOutput);
registerCoreNativeCommands(cli, logger);
};
+25 -7
View File
@@ -13,13 +13,18 @@ import {
} from "./options.js";
import type { CliInstance } from "./types.js";
import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnalysis.js";
import type {
CliCommandOutput,
CliResultOutput,
} from "./streamedJsonOutput.js";
/** Register provider-neutral binary overview and procedure CLI commands. */
export const registerCoreBinaryCommands = (
cli: CliInstance,
logger: Logger,
resultOutput?: CliResultOutput,
): void => {
registerOverviewCommands(cli, logger);
registerOverviewCommands(cli, logger, resultOutput);
registerFunctionCommand(cli, logger);
registerInstructionsCommand(cli, logger);
registerSearchCommand(cli, logger);
@@ -27,7 +32,11 @@ export const registerCoreBinaryCommands = (
registerTraceCommand(cli, logger);
};
const registerOverviewCommands = (cli: CliInstance, logger: Logger): void => {
const registerOverviewCommands = (
cli: CliInstance,
logger: Logger,
resultOutput?: CliResultOutput,
): void => {
const overviewOptions = z.object({
snapshot: z
.string()
@@ -43,9 +52,16 @@ const registerOverviewCommands = (cli: CliInstance, logger: Logger): void => {
path: z.string().describe("App, program, or analysis database path"),
}),
options: overviewOptions,
run: ({ args, options }) =>
run: ({ args, options, format }) =>
logCliCommand(logger, "analyze", () =>
runRoutedOverview(args.path, options, logger),
runRoutedOverview(
args.path,
options,
logger,
resultOutput === undefined
? undefined
: { output: resultOutput, command: CLI_COMMANDS.analyze, format },
),
),
});
cli.command(CLI_COMMANDS.inspect, {
@@ -109,6 +125,7 @@ const runRoutedOverview = async (
readonly provider?: string | undefined;
},
logger: Logger,
output?: CliCommandOutput,
) => {
if (
options.provider === undefined &&
@@ -116,9 +133,10 @@ const runRoutedOverview = async (
options["target-format"] === undefined &&
(await isJavaScriptApplicationPath(path))
)
return runCliJavaScriptApplicationAnalysis({
input_path: resolve(path),
});
return runCliJavaScriptApplicationAnalysis(
{ input_path: resolve(path) },
output,
);
return runDirectAnalysis(
path,
"binary_overview",
+15 -6
View File
@@ -24,15 +24,17 @@ import {
javascriptApplicationOptions,
} from "../cliObservationOptions.js";
import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnalysis.js";
import type { CliResultOutput } from "./streamedJsonOutput.js";
/** Register CLI equivalents of the Electron MCP tools. */
export const registerElectronCommands = (
cli: ReturnType<typeof Cli.create>,
logger: Logger,
resultOutput?: CliResultOutput,
): void => {
registerElectronObservationCommands(cli, logger);
registerElectronActiveCommand(cli, logger);
registerJavaScriptApplicationCommand(cli, logger);
registerJavaScriptApplicationCommand(cli, logger, resultOutput);
registerJavaScriptRuntimeReconciliationCommand(cli, logger);
};
@@ -172,6 +174,7 @@ const registerElectronPageInspection = (
const registerJavaScriptApplicationCommand = (
cli: ReturnType<typeof Cli.create>,
logger: Logger,
resultOutput?: CliResultOutput,
): void => {
cli.command(CLI_COMMANDS.analyzeJavaScriptApplication, {
description:
@@ -180,12 +183,18 @@ const registerJavaScriptApplicationCommand = (
path: z.string().describe("ASAR or extracted application path"),
}),
options: javascriptApplicationOptions,
run: ({ args, options }) =>
run: ({ args, options, format }) =>
logCliCommand(logger, CLI_COMMANDS.analyzeJavaScriptApplication, () =>
runCliJavaScriptApplicationAnalysis({
input_path: args.path,
format: options.artifactFormat,
}),
runCliJavaScriptApplicationAnalysis(
{ input_path: args.path, format: options.artifactFormat },
resultOutput === undefined
? undefined
: {
output: resultOutput,
command: CLI_COMMANDS.analyzeJavaScriptApplication,
format,
},
),
),
});
};
+15 -2
View File
@@ -2,11 +2,14 @@ import { analyzeJavaScriptApplication } from "../application/javascript/JavaScri
import { createProgressReporter } from "../application/ProgressReporter.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import type { JsonValue } from "../domain/jsonValue.js";
import type { CliCommandOutput } from "./streamedJsonOutput.js";
/** Execute the shared one-shot CLI boundary for static JavaScript analysis. */
export const runCliJavaScriptApplicationAnalysis = async (
input: unknown,
): Promise<JsonValue> => {
output?: CliCommandOutput,
): Promise<JsonValue | undefined> => {
const started = performance.now();
const progress = createProgressReporter(
async (update) => {
process.stderr.write(`${JSON.stringify({ rea_progress: update })}\n`);
@@ -14,7 +17,17 @@ export const runCliJavaScriptApplicationAnalysis = async (
{ minimumIntervalMs: 0 },
);
const result = await analyzeJavaScriptApplication(input, { progress });
return result.ok ? result.value : cliError(result.error);
const value = result.ok ? result.value : cliError(result.error);
if (
output !== undefined &&
(await output.output.write(value, {
command: output.command,
format: output.format,
duration: `${Math.round(performance.now() - started)}ms`,
}))
)
return undefined;
return value;
};
const cliError = (
+149
View File
@@ -0,0 +1,149 @@
import type { Writable } from "node:stream";
type JsonOutputFormat = "json" | "jsonl";
/** Write ordinary JSON without constructing a document-sized string. */
export const writeJsonOutput = async (
value: unknown,
destination: Writable,
format: JsonOutputFormat = "json",
): Promise<void> => {
for (const part of bufferedParts(documentParts(value, format)))
await writePart(destination, part);
};
const writePart = (destination: Writable, part: string): Promise<void> =>
new Promise((resolve, reject) => {
const onError = (cause: unknown): void => reject(cause);
destination.once("error", onError);
try {
destination.write(part, (cause) => {
if (cause) {
// A Writable can emit its error after invoking the write callback.
setImmediate(() => destination.off("error", onError));
reject(cause);
} else {
destination.off("error", onError);
resolve();
}
});
} catch (cause: unknown) {
destination.off("error", onError);
reject(cause);
}
});
function* documentParts(
value: unknown,
format: JsonOutputFormat,
): Generator<string> {
if (format === "jsonl" && Array.isArray(value)) {
for (const item of value) {
yield* encodeValue(item, false, 0, new Set());
yield "\n";
}
return;
}
yield* encodeValue(value, format === "json", 0, new Set());
yield "\n";
}
function* bufferedParts(parts: Iterable<string>): Generator<string> {
let buffer = "";
for (const part of parts) {
if (buffer.length + part.length > 64 * 1024 && buffer !== "") {
yield buffer;
buffer = "";
}
buffer += part;
}
if (buffer !== "") yield buffer;
}
function* encodeString(value: string): Generator<string> {
yield '"';
for (let start = 0; start < value.length;) {
let end = Math.min(start + 8192, value.length);
const last = value.charCodeAt(end - 1);
const next = value.charCodeAt(end);
// Keep a surrogate pair in the same JSON.stringify and UTF-8 write.
if (
end < value.length &&
last >= 0xd800 &&
last <= 0xdbff &&
next >= 0xdc00 &&
next <= 0xdfff
)
end -= 1;
yield JSON.stringify(value.slice(start, end)).slice(1, -1);
start = end;
}
yield '"';
}
function* encodeValue(
value: unknown,
pretty: boolean,
depth: number,
ancestors: Set<object>,
): Generator<string> {
if (typeof value === "string") {
yield* encodeString(value);
return;
}
if (value === null || typeof value === "boolean") {
yield value === null ? "null" : String(value);
return;
}
if (typeof value === "number" && Number.isFinite(value)) {
yield String(value);
return;
}
if (typeof value !== "object" || value === null)
throw new TypeError("CLI JSON output requires ordinary JSON values");
if (ancestors.has(value))
throw new TypeError("CLI JSON output contains a circular reference");
if (
!Array.isArray(value) &&
Object.prototype.toString.call(value) !== "[object Object]"
)
throw new TypeError("CLI JSON output requires ordinary JSON objects");
ancestors.add(value);
try {
const array = Array.isArray(value);
const keys = array
? undefined
: Object.keys(value).filter((key) => {
const item: unknown = Reflect.get(value, key);
// Incur's filter can select inherited methods. Match native JSON's
// omission of fields that have no JSON representation.
return (
item !== undefined &&
typeof item !== "function" &&
typeof item !== "symbol"
);
});
const count = array ? value.length : (keys?.length ?? 0);
yield array ? "[" : "{";
for (let index = 0; index < count; index += 1) {
if (index > 0) yield ",";
if (pretty) yield `\n${" ".repeat(depth + 1)}`;
let item: unknown;
if (array) {
item = value[index];
} else {
const key = keys?.[index];
if (key === undefined)
throw new TypeError("CLI JSON object changed during serialization");
yield* encodeString(key);
yield pretty ? ": " : ":";
item = Reflect.get(value, key);
}
yield* encodeValue(item, pretty, depth + 1, ancestors);
}
if (pretty && count > 0) yield `\n${" ".repeat(depth)}`;
yield array ? "]" : "}";
} finally {
ancestors.delete(value);
}
}
+84
View File
@@ -0,0 +1,84 @@
import { type Writable } from "node:stream";
import { Filter } from "incur";
import { parseCliOutputArguments } from "../cliOutput.js";
import type { JsonValue } from "../domain/jsonValue.js";
import { writeJsonOutput } from "./jsonOutput.js";
/** Result metadata supplied by the command that actually ran. */
export interface CliResultOutputMetadata {
readonly command: string;
readonly duration: string;
readonly format: string;
}
/** Optional executable-owned result surface for one CLI invocation. */
export interface CliResultOutput {
readonly handled: boolean;
readonly failed: boolean;
write(value: JsonValue, metadata: CliResultOutputMetadata): Promise<boolean>;
}
/** Selected command output context, independent of application workflows. */
export interface CliCommandOutput {
readonly output: CliResultOutput;
readonly command: string;
readonly format: string;
}
/** Preserve Incur's JSON controls while streaming the complete result. */
export const createStreamedCliJsonOutput = (
arguments_: readonly string[],
destination: Writable,
): CliResultOutput | undefined => {
const options = parseCliOutputArguments(arguments_);
if (
options.parseError ||
options.tokenCount ||
options.tokenWindow ||
(options.format !== "json" && options.format !== "jsonl")
)
return undefined;
const format = options.format;
let handled = false;
let failed = false;
return {
get handled() {
return handled;
},
get failed() {
return failed;
},
async write(value, metadata) {
if (metadata.format !== format) return false;
const filtered: unknown = options.filterOutput
? Filter.apply(value, Filter.parse(options.filterOutput))
: value;
const meta = {
command: metadata.command,
duration: metadata.duration,
};
const document = options.fullOutput
? filtered === undefined
? { ok: true, meta }
: { ok: true, data: filtered, meta }
: filtered === undefined ||
filtered === null ||
typeof filtered === "function" ||
typeof filtered === "symbol"
? {}
: filtered;
// Once writing starts, a failed destination must not receive a second
// document from Incur's error formatter.
handled = true;
try {
await writeJsonOutput(document, destination, format);
} catch (cause: unknown) {
failed = true;
throw cause;
}
return true;
},
};
};