fix(hopper): diagnose MCP provider failures with stage, state, and request context (#1044)

* fix(hopper): diagnose MCP provider failures with stage, state, and request context

* fix(hopper): retain health at the actual provider lifecycle boundary

* test(hopper): distinguish socket loss from later observed process exit

* test(hopper): cover observed exit separately from socket-first replies

---------

Co-authored-by: N0zoM1z0 <1705004398@qq.com>
This commit is contained in:
Alan Vo
2026-10-09 00:33:56 +08:00
committed by GitHub
co-authored by N0zoM1z0
parent b5c2954ad6
commit 5a67c7fc54
20 changed files with 990 additions and 62 deletions
+254 -6
View File
@@ -19,6 +19,12 @@ import {
hopperStartupFailure,
type HopperStartupFailureDiagnostic,
} from "../domain/hopperStartupFailure.js";
import {
providerRetryAction,
sharedProviderFailureStage,
type ProviderOperationHealth,
type ProviderOperationRequest,
} from "../domain/providerOperationHealth.js";
import { err, ok, type Result } from "../domain/result.js";
import type { JsonValue } from "../domain/jsonValue.js";
import type { ProgressReporter } from "../application/ProgressReporter.js";
@@ -31,6 +37,7 @@ import {
ProviderProcessSupervisor,
} from "../process/ProviderProcess.js";
import type { BridgeLaunch, BridgeLauncher } from "./BridgeLauncher.js";
import { hopperOperationStage } from "./HopperOperationStage.js";
import type { HopperDiagnostic } from "./HopperDiagnostics.js";
import { cleanupHopperSession } from "./HopperCleanup.js";
import {
@@ -94,6 +101,7 @@ export class HopperClient {
#closing = false;
#launcherExitCode: number | null | undefined;
#launcherFailureDiagnostic: HopperStartupFailureDiagnostic | undefined;
#operationFailure: OperationFailureRecord | undefined;
#startupController: AbortController | undefined;
#startPromise: Promise<Result<HopperServerInfo, HopperError>> | undefined;
#closePromise: Promise<Result<null, AnalysisError>> | undefined;
@@ -101,10 +109,46 @@ export class HopperClient {
this.#responses.push(chunk);
};
readonly #onSocketError = (): void => {
this.#failAll(new HopperProcessError(null));
const error = new HopperProcessError(
null,
undefined,
undefined,
undefined,
"unreachable",
);
this.#failAll(error);
this.#rememberProcessFailure(error, []);
};
readonly #onSocketClose = (): void => {
if (!this.#closing) this.#failAll(new HopperProcessError(null));
if (this.#closing) return;
const launch = this.#launch;
if (launch?.ownsProcessLifetime && this.#launcherExitCode !== undefined)
return;
if (launch?.ownsProcessLifetime) {
setImmediate(() => {
if (this.#closing || this.#launcherExitCode !== undefined) return;
const exitCode = launch.process.exitCode ?? null;
const error = new HopperProcessError(
exitCode,
this.#launcherFailureDiagnostic,
undefined,
undefined,
exitCode !== null ? "exited" : "unreachable",
);
this.#failAll(error);
this.#rememberProcessFailure(error, []);
});
return;
}
const error = new HopperProcessError(
null,
undefined,
undefined,
undefined,
"unreachable",
);
this.#failAll(error);
this.#rememberProcessFailure(error, []);
};
constructor(options: HopperClientOptions) {
@@ -147,6 +191,59 @@ export class HopperClient {
return this.#requests.activity();
}
/**
* Latest operational health for this owned bridge.
* A recorded exit remains visible after request activity returns to idle.
*/
operationHealth(): ProviderOperationHealth {
const failure = this.#operationFailure;
if (failure !== undefined) {
const requests: readonly ProviderOperationRequest[] =
failure.requests.map((request) => ({
...request,
stage: failure.startupFailure
? "launch"
: hopperOperationStage(request.operation),
}));
return {
state: failure.state,
stage:
requests.length === 0
? failure.startupFailure
? "launch"
: "connection"
: sharedProviderFailureStage(requests),
retryAction: providerRetryAction(failure.state, failure.startupFailure),
exitCode: failure.exitCode,
requests,
};
}
const activity = this.requestActivity();
if (activity !== null) {
const stage = hopperOperationStage(activity.operation);
return {
state: "busy",
stage,
retryAction: "wait",
exitCode: null,
requests: [
{
requestId: activity.requestId,
operation: activity.operation,
stage,
},
],
};
}
return {
state: "idle",
stage: null,
retryAction: null,
exitCode: null,
requests: [],
};
}
/** Launch the bridge once and complete its authenticated health handshake. */
start(signal?: AbortSignal): Promise<Result<HopperServerInfo, HopperError>> {
if (this.#closePromise !== undefined) {
@@ -175,7 +272,25 @@ export class HopperClient {
void started.then(
(result) => {
release();
if (!result.ok) reset();
if (!result.ok) {
if (result.error instanceof HopperProcessError)
this.#rememberProcessFailure(result.error, []);
else if (
this.#operationFailure === undefined &&
(result.error instanceof HopperTimeoutError ||
result.error instanceof HopperStartError)
)
this.#operationFailure = {
state:
result.error instanceof HopperTimeoutError
? "not_started"
: "unknown",
exitCode: null,
startupFailure: true,
requests: [],
};
reset();
}
},
(cause: unknown) => {
this.#logger.debug(
@@ -193,6 +308,7 @@ export class HopperClient {
signal?: AbortSignal,
): Promise<Result<HopperServerInfo, HopperError>> {
if (isAborted(signal)) return err(new HopperCancelledError());
this.#operationFailure = undefined;
if (this.#socket !== undefined || this.#runtimeRoot !== undefined) {
return err(new HopperProtocolError("Hopper client is already started"));
}
@@ -393,6 +509,9 @@ export class HopperClient {
new HopperProcessError(
this.#launcherExitCode,
this.#launcherFailureDiagnostic,
undefined,
undefined,
"exited",
),
);
try {
@@ -432,7 +551,16 @@ export class HopperClient {
const socket = this.#socket;
const token = this.#token;
if (socket === undefined || socket.destroyed || token === undefined) {
return err(new HopperProcessError(null, undefined, method, id));
return this.#unavailableRequest(
new HopperProcessError(
null,
undefined,
method,
id,
"unreachable",
hopperOperationStage(method),
),
);
}
if (options.signal?.aborted === true)
return err(new HopperCancelledError());
@@ -441,6 +569,7 @@ export class HopperClient {
...(options.signal === undefined ? {} : { signal: options.signal }),
...(options.progress !== undefined ? { progress: options.progress } : {}),
});
this.#retainRequestHealth(result, id, method);
this.#logger[result.ok ? "debug" : "warn"](
{
method,
@@ -525,10 +654,18 @@ export class HopperClient {
type: "launcher-exit",
code: event.code,
});
if (launch.ownsProcessLifetime && !this.#closing)
if (launch.ownsProcessLifetime && !this.#closing) {
this.#rememberExit(event.code);
this.#failAll(
new HopperProcessError(event.code, this.#launcherFailureDiagnostic),
new HopperProcessError(
event.code,
this.#launcherFailureDiagnostic,
undefined,
undefined,
"exited",
),
);
}
}
#abortProtocol(message: string, cause?: Error): void {
@@ -539,8 +676,119 @@ export class HopperClient {
#failAll(error: HopperError): void {
this.#requests.failAll(error);
}
#unavailableRequest(
error: HopperProcessError,
): Result<JsonValue, HopperProcessError> {
this.#rememberProcessFailure(
error,
error.requestId === undefined || error.operation === undefined
? []
: [{ requestId: error.requestId, operation: error.operation }],
);
return err(error);
}
#retainRequestHealth(
result: Result<JsonValue, HopperError>,
requestId: number,
operation: string,
): void {
if (this.#closing) return;
if (result.ok) return;
if (!(result.error instanceof HopperProcessError)) return;
this.#rememberProcessFailure(result.error, [
{
requestId: result.error.requestId ?? requestId,
operation: result.error.operation ?? operation,
},
]);
}
#rememberExit(code: number | null): void {
if (this.#closing) return;
const previous = this.#operationFailure;
this.#operationFailure = {
state: "exited",
exitCode: code ?? previous?.exitCode ?? null,
startupFailure:
hopperStartupFailure(code) !== undefined ||
previous?.startupFailure === true,
requests: previous?.requests ?? [],
};
}
#rememberProcessFailure(
error: HopperProcessError,
requests: readonly FailedProviderRequest[],
): void {
if (this.#closing) return;
const previous = this.#operationFailure;
const state = processFailureState(error);
const startupFailure =
error.failureCode !== undefined || previous?.startupFailure === true;
const merged = mergeFailedRequests(previous?.requests ?? [], requests);
if (
previous !== undefined &&
PROCESS_FAILURE_RANK[previous.state] > PROCESS_FAILURE_RANK[state]
) {
this.#operationFailure = {
...previous,
startupFailure,
requests: merged,
};
return;
}
this.#operationFailure = {
state,
exitCode:
state === "exited"
? (error.exitCode ?? previous?.exitCode ?? null)
: null,
startupFailure,
requests: merged,
};
}
}
interface FailedProviderRequest {
readonly requestId: number;
readonly operation: string;
}
interface OperationFailureRecord {
readonly state: "exited" | "unreachable" | "unknown" | "not_started";
readonly exitCode: number | null;
readonly startupFailure: boolean;
readonly requests: readonly FailedProviderRequest[];
}
const PROCESS_FAILURE_RANK = {
not_started: 0,
unknown: 0,
unreachable: 1,
exited: 2,
} as const;
const processFailureState = (
error: HopperProcessError,
): OperationFailureRecord["state"] =>
error.providerState === "exited" || error.exitCode !== null
? "exited"
: error.providerState;
const mergeFailedRequests = (
current: readonly FailedProviderRequest[],
added: readonly FailedProviderRequest[],
): readonly FailedProviderRequest[] => {
const byId = new Map<number, FailedProviderRequest>();
for (const request of [...current, ...added])
byId.set(request.requestId, request);
return [...byId.values()].sort(
(left, right) => left.requestId - right.requestId,
);
};
const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true;
const startupInterruption = (
+9
View File
@@ -0,0 +1,9 @@
import type { ProviderFailureStage } from "../domain/providerOperationHealth.js";
/** Classify Hopper wire operations at their provider boundary. */
export const hopperOperationStage = (
operation: string,
): ProviderFailureStage => {
if (operation === "health") return "connection";
return operation === "procedure_pseudo_code" ? "decompilation" : "analysis";
};
+1
View File
@@ -235,6 +235,7 @@ export class HopperProvider implements AnalysisProviderCandidate {
},
];
},
operationHealthSnapshot: () => client.operationHealth(),
closeWithOutcome: (options) => client.closeWithOutcome(options),
close: () => client.close(),
};
+41 -27
View File
@@ -2,6 +2,7 @@ import type {
ProgressReporter,
ProgressUpdate,
} from "../application/ProgressReporter.js";
import { hopperOperationStage } from "./HopperOperationStage.js";
import type { HopperBridgeEvent } from "./protocol.js";
import {
HopperCancelledError,
@@ -101,23 +102,12 @@ export class HopperRequestQueue {
const entry = this.#active;
if (entry === undefined || entry.id !== id) return false;
this.#releaseWire(entry);
if (!entry.callerSettled) {
const contextual =
!result.ok && result.error instanceof HopperRemoteError
? err(
new HopperRemoteError(
result.error.code,
result.error.safeMessage,
{
diagnosticType: result.error.diagnosticType,
operation: entry.method,
requestId: entry.id,
},
),
)
: result;
this.#settleCaller(entry, contextual, "waiting");
}
if (!entry.callerSettled)
this.#settleCaller(
entry,
result.ok ? result : err(withRequestContext(result.error, entry)),
"waiting",
);
this.#active = undefined;
this.#drain();
return true;
@@ -216,7 +206,20 @@ export class HopperRequestQueue {
try {
this.send(
{ id: entry.id, method: entry.method, params: entry.params },
() => this.accept(entry.id, err(new HopperProcessError(null))),
() =>
this.accept(
entry.id,
err(
new HopperProcessError(
null,
undefined,
entry.method,
entry.id,
"unreachable",
hopperOperationStage(entry.method),
),
),
),
);
} catch (cause: unknown) {
this.accept(
@@ -289,12 +292,23 @@ export class HopperRequestQueue {
const withRequestContext = (
error: HopperError,
entry: QueuedRequest,
): HopperError =>
error instanceof HopperProcessError
? new HopperProcessError(
error.exitCode,
error.diagnostic,
entry.method,
entry.id,
)
: error;
): HopperError => {
if (error instanceof HopperProcessError)
return new HopperProcessError(
error.exitCode,
error.diagnostic,
error.operation ?? entry.method,
error.requestId ?? entry.id,
error.providerState,
error.operation === undefined && error.stage === "connection"
? hopperOperationStage(entry.method)
: error.stage,
);
if (error instanceof HopperRemoteError)
return new HopperRemoteError(error.code, error.safeMessage, {
diagnosticType: error.diagnosticType,
operation: error.operation ?? entry.method,
requestId: error.requestId ?? entry.id,
});
return error;
};