fix(hopper): diagnose MCP provider failures with stage, state, and request context (#1044)
* fix(hopper): diagnose MCP provider failures with stage, state, and request context * fix(hopper): retain health at the actual provider lifecycle boundary * test(hopper): distinguish socket loss from later observed process exit * test(hopper): cover observed exit separately from socket-first replies --------- Co-authored-by: N0zoM1z0 <1705004398@qq.com>
This commit is contained in:
@@ -14,6 +14,7 @@ import { jsonValueSchema } from "../domain/jsonValue.js";
|
||||
import type { ProgressReporter } from "./ProgressReporter.js";
|
||||
import type { ProviderRejectionCode } from "../contracts/providerSelection.js";
|
||||
import type { ProcessLineageObservation } from "../process/ProcessOwnership.js";
|
||||
import type { ProviderOperationHealth } from "../domain/providerOperationHealth.js";
|
||||
|
||||
export interface ExecutionOptions {
|
||||
readonly signal?: AbortSignal;
|
||||
@@ -99,6 +100,8 @@ export interface AnalysisClient extends AnalysisOperationPort {
|
||||
runtimeLineageSnapshots?(): readonly ProviderRuntimeLineageSnapshot[];
|
||||
/** Current provider work that may outlive a timed-out or cancelled caller. */
|
||||
requestActivitySnapshots?(): readonly ProviderRequestActivitySnapshot[];
|
||||
/** Current operational health for the active provider bridge. */
|
||||
operationHealthSnapshot?(): ProviderOperationHealth | undefined;
|
||||
/** Close with a typed result when the provider can verify cleanup. */
|
||||
closeWithOutcome?(
|
||||
options?: Pick<ExecutionOptions, "progress"> & {
|
||||
|
||||
@@ -339,6 +339,8 @@ export class BinarySession
|
||||
this.#active?.client.runtimeLineageSnapshots?.() ?? [],
|
||||
requestActivitySnapshots:
|
||||
this.#active?.client.requestActivitySnapshots?.() ?? [],
|
||||
providerOperationHealth:
|
||||
this.#active?.client.operationHealthSnapshot?.() ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { BinaryTarget } from "../../domain/binaryTarget.js";
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import type { ProviderOperationHealth } from "../../domain/providerOperationHealth.js";
|
||||
import type {
|
||||
CapabilityDescriptor,
|
||||
ProviderRequestActivitySnapshot,
|
||||
@@ -21,6 +22,7 @@ interface BinarySessionStatusInput {
|
||||
readonly runId: string | undefined;
|
||||
readonly runtimeLineageSnapshots: readonly ProviderRuntimeLineageSnapshot[];
|
||||
readonly requestActivitySnapshots: readonly ProviderRequestActivitySnapshot[];
|
||||
readonly providerOperationHealth?: ProviderOperationHealth | null;
|
||||
}
|
||||
|
||||
/** Project internal provider routing state into the caller-visible session status. */
|
||||
@@ -32,6 +34,7 @@ export const binarySessionStatus = ({
|
||||
runId,
|
||||
runtimeLineageSnapshots,
|
||||
requestActivitySnapshots,
|
||||
providerOperationHealth,
|
||||
}: BinarySessionStatusInput): JsonValue => {
|
||||
const configuredProvider = router.configuredIdentity();
|
||||
const provider = providerSummary(configuredProvider);
|
||||
@@ -85,6 +88,9 @@ export const binarySessionStatus = ({
|
||||
process_lineage: processLineageStatus(runtimeLineageSnapshots),
|
||||
},
|
||||
analysis_activity: requestActivityStatus(requestActivitySnapshots),
|
||||
provider_operation_health: providerOperationHealthStatus(
|
||||
providerOperationHealth,
|
||||
),
|
||||
};
|
||||
return target === undefined
|
||||
? { open: false, ...common }
|
||||
@@ -203,3 +209,20 @@ const capabilityStatus = (
|
||||
},
|
||||
limitations: [...descriptor.limitations],
|
||||
});
|
||||
|
||||
const providerOperationHealthStatus = (
|
||||
health: ProviderOperationHealth | null | undefined,
|
||||
): JsonValue => {
|
||||
if (health === undefined || health === null) return null;
|
||||
return {
|
||||
state: health.state,
|
||||
stage: health.stage,
|
||||
retry_action: health.retryAction,
|
||||
exit_code: health.exitCode,
|
||||
requests: health.requests.map((request) => ({
|
||||
request_id: request.requestId,
|
||||
operation: request.operation,
|
||||
stage: request.stage,
|
||||
})),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -152,6 +152,18 @@ export class CompositeProvider implements AnalysisProvider {
|
||||
.sort((left, right) =>
|
||||
left.provider.id.localeCompare(right.provider.id),
|
||||
),
|
||||
operationHealthSnapshot: () => {
|
||||
for (const client of clients.values()) {
|
||||
const snapshot = client.operationHealthSnapshot?.();
|
||||
if (snapshot !== undefined && snapshot.state !== "idle")
|
||||
return snapshot;
|
||||
}
|
||||
for (const client of clients.values()) {
|
||||
const snapshot = client.operationHealthSnapshot?.();
|
||||
if (snapshot !== undefined) return snapshot;
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
closeWithOutcome,
|
||||
close: async () => {
|
||||
await closeWithOutcome();
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { BinaryTarget } from "../../domain/binaryTarget.js";
|
||||
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
|
||||
import { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
||||
import type { ProviderOperationHealth } from "../../domain/providerOperationHealth.js";
|
||||
import { err } from "../../domain/result.js";
|
||||
import { ABORTED, waitForAbortable } from "./AbortablePromise.js";
|
||||
import type {
|
||||
@@ -115,6 +116,10 @@ class LazyAnalysisClient implements AnalysisClient {
|
||||
return this.#client?.requestActivitySnapshots?.() ?? [];
|
||||
}
|
||||
|
||||
operationHealthSnapshot(): ProviderOperationHealth | undefined {
|
||||
return this.#client?.operationHealthSnapshot?.();
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
if (this.#loading === undefined) return;
|
||||
|
||||
@@ -14,6 +14,11 @@ import {
|
||||
PROVIDER_REJECTION_CODES,
|
||||
type ProviderRejectionCode,
|
||||
} from "./providerSelection.js";
|
||||
import {
|
||||
PROVIDER_FAILURE_STAGES,
|
||||
PROVIDER_OPERATION_STATES,
|
||||
PROVIDER_RETRY_ACTIONS,
|
||||
} from "../domain/providerOperationHealth.js";
|
||||
import { analysisErrorProjectionSchema } from "./errorSchemas.js";
|
||||
import { prefixedDigestSchema } from "./../domain/digests.js";
|
||||
|
||||
@@ -169,6 +174,22 @@ export const analysisActivity = z.object({
|
||||
),
|
||||
});
|
||||
|
||||
const providerOperationRequest = z.object({
|
||||
request_id: z.number().int().min(1),
|
||||
operation: z.string().min(1),
|
||||
stage: z.enum(PROVIDER_FAILURE_STAGES),
|
||||
});
|
||||
|
||||
export const providerOperationHealth = z
|
||||
.object({
|
||||
state: z.enum(PROVIDER_OPERATION_STATES),
|
||||
stage: z.enum(PROVIDER_FAILURE_STAGES).nullable(),
|
||||
retry_action: z.enum(PROVIDER_RETRY_ACTIONS).nullable(),
|
||||
exit_code: z.number().int().nullable(),
|
||||
requests: z.array(providerOperationRequest),
|
||||
})
|
||||
.nullable();
|
||||
|
||||
const providerRejectionCode: z.ZodType<ProviderRejectionCode> = z.enum(
|
||||
PROVIDER_REJECTION_CODES,
|
||||
);
|
||||
@@ -315,6 +336,7 @@ export const sessionProvider = z
|
||||
capabilities: z.array(providerCapability),
|
||||
analysis_run: analysisRun,
|
||||
analysis_activity: analysisActivity,
|
||||
provider_operation_health: providerOperationHealth,
|
||||
analysis_provider_binding: z
|
||||
.object({
|
||||
provider: providerIdentity,
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
} from "./hopperErrors.js";
|
||||
import { ProviderSelectionError } from "./providerSelectionError.js";
|
||||
import { UnknownRegistryError } from "./unknownRegistryError.js";
|
||||
import { providerRetryAction } from "./providerOperationHealth.js";
|
||||
import {
|
||||
type AnalysisError,
|
||||
type AnalysisErrorTag,
|
||||
@@ -49,7 +50,7 @@ export const analysisErrorRemediationAction = (
|
||||
? "Check binary_session.analysis_activity, wait for the active Hopper request to finish, then retry."
|
||||
: "Check binary_session for Hopper health, then retry the operation.";
|
||||
if (error instanceof HopperProcessError)
|
||||
return "Check binary_session provider health. Restart the owned Hopper process only when it has stopped.";
|
||||
return hopperProcessRemediation(error);
|
||||
if (error instanceof HopperStartError)
|
||||
return error.ownerRunId === undefined
|
||||
? "Check the Hopper launcher and target details, then retry opening the target."
|
||||
@@ -191,8 +192,7 @@ const hopperErrorUserMessage = (error: AnalysisError): string | undefined => {
|
||||
? `Hopper timed out during ${request} while the provider remained busy. Check binary_session.analysis_activity, wait for the active request to finish, then retry.`
|
||||
: `Hopper timed out during ${request} before it started. Check binary_session for provider health, then retry.`;
|
||||
}
|
||||
if (error instanceof HopperProcessError)
|
||||
return `Hopper stopped during ${error.operation ?? "connection"}${error.exitCode === null ? "; its exit was not observed" : ` with exit code ${String(error.exitCode)}`}.${error.userMessage === undefined ? "" : ` ${error.userMessage}`} Check binary_session provider health before retrying.`;
|
||||
if (error instanceof HopperProcessError) return hopperProcessMessage(error);
|
||||
if (error instanceof HopperStartError)
|
||||
return (
|
||||
error.userMessage ??
|
||||
@@ -203,6 +203,40 @@ const hopperErrorUserMessage = (error: AnalysisError): string | undefined => {
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const RESTART_OWNED_PROVIDER =
|
||||
"Restart the owned provider by calling close_binary, then open the target again and retry.";
|
||||
|
||||
const hopperProcessRemediation = (error: HopperProcessError): string => {
|
||||
if (error.failureCode !== undefined && error.userMessage !== undefined)
|
||||
return error.userMessage;
|
||||
if (providerRetryAction(error.providerState) === "restart_provider")
|
||||
return RESTART_OWNED_PROVIDER;
|
||||
return "Read binary_session.provider_operation_health for this request before retrying or restarting the owned provider.";
|
||||
};
|
||||
|
||||
const hopperProcessMessage = (error: HopperProcessError): string => {
|
||||
const stage = error.stage;
|
||||
const where = error.operation ?? stage;
|
||||
const request =
|
||||
error.requestId === undefined
|
||||
? ""
|
||||
: ` Request id ${String(error.requestId)}.`;
|
||||
const startup =
|
||||
error.userMessage === undefined ? "" : ` ${error.userMessage}`;
|
||||
if (error.providerState === "exited") {
|
||||
const exit =
|
||||
error.exitCode === null
|
||||
? "no exit code was observed"
|
||||
: `exit code ${String(error.exitCode)}`;
|
||||
const recovery =
|
||||
error.failureCode === undefined ? ` ${RESTART_OWNED_PROVIDER}` : "";
|
||||
return `Hopper exited during ${where} (${exit}).${startup}${recovery}${request}`;
|
||||
}
|
||||
if (error.providerState === "unreachable")
|
||||
return `Hopper became unreachable during ${where}; its process exit was not observed. ${RESTART_OWNED_PROVIDER}${startup}${request}`;
|
||||
return `Hopper failed during ${where} and REA could not determine whether the provider is busy, exited, or unreachable.${startup} Read binary_session.provider_operation_health for this request before retrying or restarting.${request}`;
|
||||
};
|
||||
|
||||
const standardErrorMessage = (tag: AnalysisErrorTag): string | undefined => {
|
||||
if (UNREADABLE_OUTPUT_TAGS.has(tag))
|
||||
return "Analysis returned an unreadable result. Retry once; if it continues, run `rea doctor`.";
|
||||
|
||||
@@ -390,3 +390,40 @@ describe("analysis error projection: caller contract", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("analysis error projection: operational diagnostics", () => {
|
||||
it("distinguishes an observed Hopper exit from an undetermined provider failure", () => {
|
||||
const exited = projectAnalysisError(
|
||||
new HopperProcessError(7, undefined, "search_strings", 4),
|
||||
);
|
||||
expect(exited).toMatchObject({
|
||||
code: "provider_unavailable",
|
||||
retryable: true,
|
||||
details: {
|
||||
stage: "analysis",
|
||||
provider_state: "exited",
|
||||
retry_action: "restart_provider",
|
||||
operation: "search_strings",
|
||||
request_id: 4,
|
||||
exit_code: 7,
|
||||
},
|
||||
});
|
||||
expect(exited.message).not.toContain("Run `rea doctor`, then try again.");
|
||||
expect(exited.remediation.action).toContain("Restart the owned provider");
|
||||
|
||||
const unknown = projectAnalysisError(
|
||||
new HopperProcessError(null, undefined, "search_strings", 5),
|
||||
);
|
||||
expect(unknown.details).toMatchObject({
|
||||
stage: "analysis",
|
||||
provider_state: "unknown",
|
||||
retry_action: "unknown",
|
||||
operation: "search_strings",
|
||||
request_id: 5,
|
||||
exit_code: null,
|
||||
});
|
||||
expect(unknown.message).toContain("could not determine");
|
||||
expect(unknown.message).not.toContain("stopped");
|
||||
expect(unknown.remediation.action).toContain("provider_operation_health");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -25,11 +25,13 @@ import {
|
||||
HopperCancelledError,
|
||||
HopperProcessError,
|
||||
HopperRemoteError,
|
||||
HopperStartError,
|
||||
HopperTimeoutError,
|
||||
} from "./hopperErrors.js";
|
||||
import { ProviderAdapterError } from "./providerAdapterError.js";
|
||||
import { ProviderSelectionError } from "./providerSelectionError.js";
|
||||
import { UnknownRegistryError } from "./unknownRegistryError.js";
|
||||
import { providerRetryAction } from "./providerOperationHealth.js";
|
||||
import {
|
||||
type AnalysisError,
|
||||
type AnalysisErrorTag,
|
||||
@@ -327,10 +329,16 @@ const providerErrorDetails = (
|
||||
...(error.operation === undefined ? {} : { operation: error.operation }),
|
||||
...(error.requestId === undefined ? {} : { request_id: error.requestId }),
|
||||
};
|
||||
if (error instanceof HopperProcessError)
|
||||
if (error instanceof HopperProcessError) {
|
||||
const stage = error.stage;
|
||||
return {
|
||||
exit_code: error.exitCode,
|
||||
stage: error.operation === undefined ? "connection" : "analysis",
|
||||
stage,
|
||||
provider_state: error.providerState,
|
||||
retry_action: providerRetryAction(
|
||||
error.providerState,
|
||||
error.failureCode !== undefined,
|
||||
),
|
||||
...(error.failureCode === undefined
|
||||
? {}
|
||||
: { failure_code: error.failureCode }),
|
||||
@@ -340,6 +348,16 @@ const providerErrorDetails = (
|
||||
? {}
|
||||
: { diagnostics: { ...error.diagnostic } }),
|
||||
};
|
||||
}
|
||||
if (error instanceof HopperStartError)
|
||||
return {
|
||||
stage: "launch",
|
||||
provider_state: "unknown",
|
||||
retry_action: error.ownerRunId === undefined ? "retry" : "unknown",
|
||||
...(error.ownerRunId === undefined
|
||||
? {}
|
||||
: { owner_run_id: error.ownerRunId }),
|
||||
};
|
||||
return undefined;
|
||||
};
|
||||
|
||||
@@ -369,9 +387,10 @@ const lifecycleErrorDetails = (
|
||||
return { operation: error.operation, timeout_ms: error.timeoutMs };
|
||||
if (error instanceof HopperTimeoutError)
|
||||
return {
|
||||
stage: error.operation === undefined ? "startup" : "analysis",
|
||||
stage: error.operation === undefined ? "startup" : error.stage,
|
||||
timeout_ms: error.timeoutMs,
|
||||
provider_state: error.providerState,
|
||||
retry_action: error.providerState === "busy" ? "wait" : "retry",
|
||||
...(error.operation === undefined ? {} : { operation: error.operation }),
|
||||
...(error.requestId === undefined ? {} : { request_id: error.requestId }),
|
||||
};
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { AnalysisError } from "./analysisErrorBase.js";
|
||||
import type {
|
||||
HopperProcessProviderState,
|
||||
ProviderFailureStage,
|
||||
} from "./providerOperationHealth.js";
|
||||
import {
|
||||
hopperStartupFailure,
|
||||
type HopperStartupDiagnostic,
|
||||
@@ -18,6 +22,9 @@ export class HopperTimeoutError extends HopperError {
|
||||
readonly operation?: string,
|
||||
readonly requestId?: number,
|
||||
readonly providerState: "busy" | "not_started" = "not_started",
|
||||
readonly stage: ProviderFailureStage = operation === undefined
|
||||
? "launch"
|
||||
: "analysis",
|
||||
) {
|
||||
super(
|
||||
`Hopper ${operation === undefined ? "startup" : operation} timed out after ${String(timeoutMs)}ms`,
|
||||
@@ -76,17 +83,30 @@ export class HopperProcessError extends HopperError {
|
||||
readonly _tag = "HopperProcessError";
|
||||
readonly failureCode: HopperStartupFailureCode | undefined;
|
||||
override readonly userMessage: string | undefined;
|
||||
readonly providerState: HopperProcessProviderState;
|
||||
readonly stage: ProviderFailureStage;
|
||||
|
||||
constructor(
|
||||
readonly exitCode: number | null,
|
||||
readonly diagnostic?: HopperStartupFailureDiagnostic,
|
||||
readonly operation?: string,
|
||||
readonly requestId?: number,
|
||||
providerState?: HopperProcessProviderState,
|
||||
stage?: ProviderFailureStage,
|
||||
) {
|
||||
super(`Hopper bridge stopped unexpectedly with code ${String(exitCode)}`);
|
||||
const failure = hopperStartupFailure(exitCode);
|
||||
this.stage =
|
||||
stage ??
|
||||
(failure === undefined
|
||||
? operation === undefined
|
||||
? "connection"
|
||||
: "analysis"
|
||||
: "launch");
|
||||
this.failureCode = failure?.code;
|
||||
this.userMessage = failure?.message;
|
||||
this.providerState =
|
||||
providerState ?? (exitCode === null ? "unknown" : "exited");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/** Caller-visible operational state for one provider bridge. */
|
||||
export const PROVIDER_OPERATION_STATES = [
|
||||
"idle",
|
||||
"busy",
|
||||
"exited",
|
||||
"unreachable",
|
||||
"not_started",
|
||||
"unknown",
|
||||
] as const;
|
||||
|
||||
/** Operational state REA can support with evidence, or unknown. */
|
||||
export type ProviderOperationState = (typeof PROVIDER_OPERATION_STATES)[number];
|
||||
|
||||
/** States a Hopper process failure can establish. */
|
||||
export type HopperProcessProviderState = "exited" | "unreachable" | "unknown";
|
||||
|
||||
/** What the caller should do next for this provider state. */
|
||||
export const PROVIDER_RETRY_ACTIONS = [
|
||||
"retry",
|
||||
"wait",
|
||||
"restart_provider",
|
||||
"unknown",
|
||||
] as const;
|
||||
|
||||
/** Recovery choice for a provider operation failure. */
|
||||
export type ProviderRetryAction = (typeof PROVIDER_RETRY_ACTIONS)[number];
|
||||
|
||||
/** Stage that was in progress when the provider operation failed. */
|
||||
export const PROVIDER_FAILURE_STAGES = [
|
||||
"launch",
|
||||
"connection",
|
||||
"analysis",
|
||||
"decompilation",
|
||||
"unknown",
|
||||
] as const;
|
||||
|
||||
/** Known provider-operation stage, including an explicit unknown. */
|
||||
export type ProviderFailureStage = (typeof PROVIDER_FAILURE_STAGES)[number];
|
||||
|
||||
/** One provider request retained with a failure or an in-flight operation. */
|
||||
export interface ProviderOperationRequest {
|
||||
readonly requestId: number;
|
||||
readonly operation: string;
|
||||
readonly stage: ProviderFailureStage;
|
||||
}
|
||||
|
||||
/** Sanitized provider health a session can show after the request returns. */
|
||||
export interface ProviderOperationHealth {
|
||||
readonly state: ProviderOperationState;
|
||||
readonly stage: ProviderFailureStage | null;
|
||||
readonly retryAction: ProviderRetryAction | null;
|
||||
readonly exitCode: number | null;
|
||||
readonly requests: readonly ProviderOperationRequest[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Choose a recovery action from the observed state.
|
||||
* Startup failures keep their existing install or configuration guidance.
|
||||
*/
|
||||
export const providerRetryAction = (
|
||||
state: ProviderOperationState,
|
||||
startupFailure = false,
|
||||
): ProviderRetryAction | null => {
|
||||
if (state === "idle") return null;
|
||||
if (startupFailure && state === "exited") return "retry";
|
||||
switch (state) {
|
||||
case "busy":
|
||||
return "wait";
|
||||
case "exited":
|
||||
case "unreachable":
|
||||
return "restart_provider";
|
||||
case "not_started":
|
||||
return "retry";
|
||||
case "unknown":
|
||||
return "unknown";
|
||||
}
|
||||
};
|
||||
|
||||
/** Use one stage when every retained request agrees, otherwise unknown. */
|
||||
export const sharedProviderFailureStage = (
|
||||
requests: readonly ProviderOperationRequest[],
|
||||
): ProviderFailureStage | null => {
|
||||
const first = requests[0]?.stage;
|
||||
if (first === undefined) return null;
|
||||
return requests.every((request) => request.stage === first)
|
||||
? first
|
||||
: "unknown";
|
||||
};
|
||||
+254
-6
@@ -19,6 +19,12 @@ import {
|
||||
hopperStartupFailure,
|
||||
type HopperStartupFailureDiagnostic,
|
||||
} from "../domain/hopperStartupFailure.js";
|
||||
import {
|
||||
providerRetryAction,
|
||||
sharedProviderFailureStage,
|
||||
type ProviderOperationHealth,
|
||||
type ProviderOperationRequest,
|
||||
} from "../domain/providerOperationHealth.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import type { ProgressReporter } from "../application/ProgressReporter.js";
|
||||
@@ -31,6 +37,7 @@ import {
|
||||
ProviderProcessSupervisor,
|
||||
} from "../process/ProviderProcess.js";
|
||||
import type { BridgeLaunch, BridgeLauncher } from "./BridgeLauncher.js";
|
||||
import { hopperOperationStage } from "./HopperOperationStage.js";
|
||||
import type { HopperDiagnostic } from "./HopperDiagnostics.js";
|
||||
import { cleanupHopperSession } from "./HopperCleanup.js";
|
||||
import {
|
||||
@@ -94,6 +101,7 @@ export class HopperClient {
|
||||
#closing = false;
|
||||
#launcherExitCode: number | null | undefined;
|
||||
#launcherFailureDiagnostic: HopperStartupFailureDiagnostic | undefined;
|
||||
#operationFailure: OperationFailureRecord | undefined;
|
||||
#startupController: AbortController | undefined;
|
||||
#startPromise: Promise<Result<HopperServerInfo, HopperError>> | undefined;
|
||||
#closePromise: Promise<Result<null, AnalysisError>> | undefined;
|
||||
@@ -101,10 +109,46 @@ export class HopperClient {
|
||||
this.#responses.push(chunk);
|
||||
};
|
||||
readonly #onSocketError = (): void => {
|
||||
this.#failAll(new HopperProcessError(null));
|
||||
const error = new HopperProcessError(
|
||||
null,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
"unreachable",
|
||||
);
|
||||
this.#failAll(error);
|
||||
this.#rememberProcessFailure(error, []);
|
||||
};
|
||||
readonly #onSocketClose = (): void => {
|
||||
if (!this.#closing) this.#failAll(new HopperProcessError(null));
|
||||
if (this.#closing) return;
|
||||
const launch = this.#launch;
|
||||
if (launch?.ownsProcessLifetime && this.#launcherExitCode !== undefined)
|
||||
return;
|
||||
if (launch?.ownsProcessLifetime) {
|
||||
setImmediate(() => {
|
||||
if (this.#closing || this.#launcherExitCode !== undefined) return;
|
||||
const exitCode = launch.process.exitCode ?? null;
|
||||
const error = new HopperProcessError(
|
||||
exitCode,
|
||||
this.#launcherFailureDiagnostic,
|
||||
undefined,
|
||||
undefined,
|
||||
exitCode !== null ? "exited" : "unreachable",
|
||||
);
|
||||
this.#failAll(error);
|
||||
this.#rememberProcessFailure(error, []);
|
||||
});
|
||||
return;
|
||||
}
|
||||
const error = new HopperProcessError(
|
||||
null,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
"unreachable",
|
||||
);
|
||||
this.#failAll(error);
|
||||
this.#rememberProcessFailure(error, []);
|
||||
};
|
||||
|
||||
constructor(options: HopperClientOptions) {
|
||||
@@ -147,6 +191,59 @@ export class HopperClient {
|
||||
return this.#requests.activity();
|
||||
}
|
||||
|
||||
/**
|
||||
* Latest operational health for this owned bridge.
|
||||
* A recorded exit remains visible after request activity returns to idle.
|
||||
*/
|
||||
operationHealth(): ProviderOperationHealth {
|
||||
const failure = this.#operationFailure;
|
||||
if (failure !== undefined) {
|
||||
const requests: readonly ProviderOperationRequest[] =
|
||||
failure.requests.map((request) => ({
|
||||
...request,
|
||||
stage: failure.startupFailure
|
||||
? "launch"
|
||||
: hopperOperationStage(request.operation),
|
||||
}));
|
||||
return {
|
||||
state: failure.state,
|
||||
stage:
|
||||
requests.length === 0
|
||||
? failure.startupFailure
|
||||
? "launch"
|
||||
: "connection"
|
||||
: sharedProviderFailureStage(requests),
|
||||
retryAction: providerRetryAction(failure.state, failure.startupFailure),
|
||||
exitCode: failure.exitCode,
|
||||
requests,
|
||||
};
|
||||
}
|
||||
const activity = this.requestActivity();
|
||||
if (activity !== null) {
|
||||
const stage = hopperOperationStage(activity.operation);
|
||||
return {
|
||||
state: "busy",
|
||||
stage,
|
||||
retryAction: "wait",
|
||||
exitCode: null,
|
||||
requests: [
|
||||
{
|
||||
requestId: activity.requestId,
|
||||
operation: activity.operation,
|
||||
stage,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return {
|
||||
state: "idle",
|
||||
stage: null,
|
||||
retryAction: null,
|
||||
exitCode: null,
|
||||
requests: [],
|
||||
};
|
||||
}
|
||||
|
||||
/** Launch the bridge once and complete its authenticated health handshake. */
|
||||
start(signal?: AbortSignal): Promise<Result<HopperServerInfo, HopperError>> {
|
||||
if (this.#closePromise !== undefined) {
|
||||
@@ -175,7 +272,25 @@ export class HopperClient {
|
||||
void started.then(
|
||||
(result) => {
|
||||
release();
|
||||
if (!result.ok) reset();
|
||||
if (!result.ok) {
|
||||
if (result.error instanceof HopperProcessError)
|
||||
this.#rememberProcessFailure(result.error, []);
|
||||
else if (
|
||||
this.#operationFailure === undefined &&
|
||||
(result.error instanceof HopperTimeoutError ||
|
||||
result.error instanceof HopperStartError)
|
||||
)
|
||||
this.#operationFailure = {
|
||||
state:
|
||||
result.error instanceof HopperTimeoutError
|
||||
? "not_started"
|
||||
: "unknown",
|
||||
exitCode: null,
|
||||
startupFailure: true,
|
||||
requests: [],
|
||||
};
|
||||
reset();
|
||||
}
|
||||
},
|
||||
(cause: unknown) => {
|
||||
this.#logger.debug(
|
||||
@@ -193,6 +308,7 @@ export class HopperClient {
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<HopperServerInfo, HopperError>> {
|
||||
if (isAborted(signal)) return err(new HopperCancelledError());
|
||||
this.#operationFailure = undefined;
|
||||
if (this.#socket !== undefined || this.#runtimeRoot !== undefined) {
|
||||
return err(new HopperProtocolError("Hopper client is already started"));
|
||||
}
|
||||
@@ -393,6 +509,9 @@ export class HopperClient {
|
||||
new HopperProcessError(
|
||||
this.#launcherExitCode,
|
||||
this.#launcherFailureDiagnostic,
|
||||
undefined,
|
||||
undefined,
|
||||
"exited",
|
||||
),
|
||||
);
|
||||
try {
|
||||
@@ -432,7 +551,16 @@ export class HopperClient {
|
||||
const socket = this.#socket;
|
||||
const token = this.#token;
|
||||
if (socket === undefined || socket.destroyed || token === undefined) {
|
||||
return err(new HopperProcessError(null, undefined, method, id));
|
||||
return this.#unavailableRequest(
|
||||
new HopperProcessError(
|
||||
null,
|
||||
undefined,
|
||||
method,
|
||||
id,
|
||||
"unreachable",
|
||||
hopperOperationStage(method),
|
||||
),
|
||||
);
|
||||
}
|
||||
if (options.signal?.aborted === true)
|
||||
return err(new HopperCancelledError());
|
||||
@@ -441,6 +569,7 @@ export class HopperClient {
|
||||
...(options.signal === undefined ? {} : { signal: options.signal }),
|
||||
...(options.progress !== undefined ? { progress: options.progress } : {}),
|
||||
});
|
||||
this.#retainRequestHealth(result, id, method);
|
||||
this.#logger[result.ok ? "debug" : "warn"](
|
||||
{
|
||||
method,
|
||||
@@ -525,10 +654,18 @@ export class HopperClient {
|
||||
type: "launcher-exit",
|
||||
code: event.code,
|
||||
});
|
||||
if (launch.ownsProcessLifetime && !this.#closing)
|
||||
if (launch.ownsProcessLifetime && !this.#closing) {
|
||||
this.#rememberExit(event.code);
|
||||
this.#failAll(
|
||||
new HopperProcessError(event.code, this.#launcherFailureDiagnostic),
|
||||
new HopperProcessError(
|
||||
event.code,
|
||||
this.#launcherFailureDiagnostic,
|
||||
undefined,
|
||||
undefined,
|
||||
"exited",
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#abortProtocol(message: string, cause?: Error): void {
|
||||
@@ -539,8 +676,119 @@ export class HopperClient {
|
||||
#failAll(error: HopperError): void {
|
||||
this.#requests.failAll(error);
|
||||
}
|
||||
|
||||
#unavailableRequest(
|
||||
error: HopperProcessError,
|
||||
): Result<JsonValue, HopperProcessError> {
|
||||
this.#rememberProcessFailure(
|
||||
error,
|
||||
error.requestId === undefined || error.operation === undefined
|
||||
? []
|
||||
: [{ requestId: error.requestId, operation: error.operation }],
|
||||
);
|
||||
return err(error);
|
||||
}
|
||||
|
||||
#retainRequestHealth(
|
||||
result: Result<JsonValue, HopperError>,
|
||||
requestId: number,
|
||||
operation: string,
|
||||
): void {
|
||||
if (this.#closing) return;
|
||||
if (result.ok) return;
|
||||
if (!(result.error instanceof HopperProcessError)) return;
|
||||
this.#rememberProcessFailure(result.error, [
|
||||
{
|
||||
requestId: result.error.requestId ?? requestId,
|
||||
operation: result.error.operation ?? operation,
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
#rememberExit(code: number | null): void {
|
||||
if (this.#closing) return;
|
||||
const previous = this.#operationFailure;
|
||||
this.#operationFailure = {
|
||||
state: "exited",
|
||||
exitCode: code ?? previous?.exitCode ?? null,
|
||||
startupFailure:
|
||||
hopperStartupFailure(code) !== undefined ||
|
||||
previous?.startupFailure === true,
|
||||
requests: previous?.requests ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
#rememberProcessFailure(
|
||||
error: HopperProcessError,
|
||||
requests: readonly FailedProviderRequest[],
|
||||
): void {
|
||||
if (this.#closing) return;
|
||||
const previous = this.#operationFailure;
|
||||
const state = processFailureState(error);
|
||||
const startupFailure =
|
||||
error.failureCode !== undefined || previous?.startupFailure === true;
|
||||
const merged = mergeFailedRequests(previous?.requests ?? [], requests);
|
||||
if (
|
||||
previous !== undefined &&
|
||||
PROCESS_FAILURE_RANK[previous.state] > PROCESS_FAILURE_RANK[state]
|
||||
) {
|
||||
this.#operationFailure = {
|
||||
...previous,
|
||||
startupFailure,
|
||||
requests: merged,
|
||||
};
|
||||
return;
|
||||
}
|
||||
this.#operationFailure = {
|
||||
state,
|
||||
exitCode:
|
||||
state === "exited"
|
||||
? (error.exitCode ?? previous?.exitCode ?? null)
|
||||
: null,
|
||||
startupFailure,
|
||||
requests: merged,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
interface FailedProviderRequest {
|
||||
readonly requestId: number;
|
||||
readonly operation: string;
|
||||
}
|
||||
|
||||
interface OperationFailureRecord {
|
||||
readonly state: "exited" | "unreachable" | "unknown" | "not_started";
|
||||
readonly exitCode: number | null;
|
||||
readonly startupFailure: boolean;
|
||||
readonly requests: readonly FailedProviderRequest[];
|
||||
}
|
||||
|
||||
const PROCESS_FAILURE_RANK = {
|
||||
not_started: 0,
|
||||
unknown: 0,
|
||||
unreachable: 1,
|
||||
exited: 2,
|
||||
} as const;
|
||||
|
||||
const processFailureState = (
|
||||
error: HopperProcessError,
|
||||
): OperationFailureRecord["state"] =>
|
||||
error.providerState === "exited" || error.exitCode !== null
|
||||
? "exited"
|
||||
: error.providerState;
|
||||
|
||||
const mergeFailedRequests = (
|
||||
current: readonly FailedProviderRequest[],
|
||||
added: readonly FailedProviderRequest[],
|
||||
): readonly FailedProviderRequest[] => {
|
||||
const byId = new Map<number, FailedProviderRequest>();
|
||||
for (const request of [...current, ...added])
|
||||
byId.set(request.requestId, request);
|
||||
return [...byId.values()].sort(
|
||||
(left, right) => left.requestId - right.requestId,
|
||||
);
|
||||
};
|
||||
|
||||
const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true;
|
||||
|
||||
const startupInterruption = (
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import type { ProviderFailureStage } from "../domain/providerOperationHealth.js";
|
||||
|
||||
/** Classify Hopper wire operations at their provider boundary. */
|
||||
export const hopperOperationStage = (
|
||||
operation: string,
|
||||
): ProviderFailureStage => {
|
||||
if (operation === "health") return "connection";
|
||||
return operation === "procedure_pseudo_code" ? "decompilation" : "analysis";
|
||||
};
|
||||
@@ -235,6 +235,7 @@ export class HopperProvider implements AnalysisProviderCandidate {
|
||||
},
|
||||
];
|
||||
},
|
||||
operationHealthSnapshot: () => client.operationHealth(),
|
||||
closeWithOutcome: (options) => client.closeWithOutcome(options),
|
||||
close: () => client.close(),
|
||||
};
|
||||
|
||||
@@ -2,6 +2,7 @@ import type {
|
||||
ProgressReporter,
|
||||
ProgressUpdate,
|
||||
} from "../application/ProgressReporter.js";
|
||||
import { hopperOperationStage } from "./HopperOperationStage.js";
|
||||
import type { HopperBridgeEvent } from "./protocol.js";
|
||||
import {
|
||||
HopperCancelledError,
|
||||
@@ -101,23 +102,12 @@ export class HopperRequestQueue {
|
||||
const entry = this.#active;
|
||||
if (entry === undefined || entry.id !== id) return false;
|
||||
this.#releaseWire(entry);
|
||||
if (!entry.callerSettled) {
|
||||
const contextual =
|
||||
!result.ok && result.error instanceof HopperRemoteError
|
||||
? err(
|
||||
new HopperRemoteError(
|
||||
result.error.code,
|
||||
result.error.safeMessage,
|
||||
{
|
||||
diagnosticType: result.error.diagnosticType,
|
||||
operation: entry.method,
|
||||
requestId: entry.id,
|
||||
},
|
||||
),
|
||||
)
|
||||
: result;
|
||||
this.#settleCaller(entry, contextual, "waiting");
|
||||
}
|
||||
if (!entry.callerSettled)
|
||||
this.#settleCaller(
|
||||
entry,
|
||||
result.ok ? result : err(withRequestContext(result.error, entry)),
|
||||
"waiting",
|
||||
);
|
||||
this.#active = undefined;
|
||||
this.#drain();
|
||||
return true;
|
||||
@@ -216,7 +206,20 @@ export class HopperRequestQueue {
|
||||
try {
|
||||
this.send(
|
||||
{ id: entry.id, method: entry.method, params: entry.params },
|
||||
() => this.accept(entry.id, err(new HopperProcessError(null))),
|
||||
() =>
|
||||
this.accept(
|
||||
entry.id,
|
||||
err(
|
||||
new HopperProcessError(
|
||||
null,
|
||||
undefined,
|
||||
entry.method,
|
||||
entry.id,
|
||||
"unreachable",
|
||||
hopperOperationStage(entry.method),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
this.accept(
|
||||
@@ -289,12 +292,23 @@ export class HopperRequestQueue {
|
||||
const withRequestContext = (
|
||||
error: HopperError,
|
||||
entry: QueuedRequest,
|
||||
): HopperError =>
|
||||
error instanceof HopperProcessError
|
||||
? new HopperProcessError(
|
||||
error.exitCode,
|
||||
error.diagnostic,
|
||||
entry.method,
|
||||
entry.id,
|
||||
)
|
||||
: error;
|
||||
): HopperError => {
|
||||
if (error instanceof HopperProcessError)
|
||||
return new HopperProcessError(
|
||||
error.exitCode,
|
||||
error.diagnostic,
|
||||
error.operation ?? entry.method,
|
||||
error.requestId ?? entry.id,
|
||||
error.providerState,
|
||||
error.operation === undefined && error.stage === "connection"
|
||||
? hopperOperationStage(entry.method)
|
||||
: error.stage,
|
||||
);
|
||||
if (error instanceof HopperRemoteError)
|
||||
return new HopperRemoteError(error.code, error.safeMessage, {
|
||||
diagnosticType: error.diagnosticType,
|
||||
operation: error.operation ?? entry.method,
|
||||
requestId: error.requestId ?? entry.id,
|
||||
});
|
||||
return error;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user