Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
110151e523 | ||
|
|
b93e585d68 | ||
|
|
929aa67336 | ||
|
|
4e8475e965 | ||
|
|
557148ed3f | ||
|
|
f839046776 | ||
|
|
281c4b8ec5 | ||
|
|
a0da18d7fa | ||
|
|
2c6e074cac | ||
|
|
29c871e51b | ||
|
|
c0018ca403 | ||
|
|
ad16d250e3 | ||
|
|
af2f55cda9 | ||
|
|
4d5d1f5daa | ||
|
|
0aff997b31 | ||
|
|
e8361b7715 | ||
|
|
72e931f3a3 | ||
|
|
0266e6552b | ||
|
|
7d972a7e85 | ||
|
|
e1fb40460d | ||
|
|
553cd6e358 |
@@ -169,14 +169,6 @@ repeat parsing. Returned-row exhaustion
|
||||
does not prove complete metadata or absence under partial/unavailable coverage;
|
||||
preserve build-local tokens and their artifact/MVID scope.
|
||||
|
||||
`inspect_managed_members` always returns the assembly-wide `member_refs`, even
|
||||
for one selected type. For questions about a selected type, select
|
||||
`detail: "summary"` with `type_names`; it returns the selection (with unmatched
|
||||
names), coverage and per-collection counts. Page rows with
|
||||
`view: {kind: "managed-members", collection: "call_edges", offset: 0, limit: 64}`
|
||||
over `types`, `exported_types`, `fields`, `methods`, `member_refs`, `call_edges`,
|
||||
`field_accesses` or `user_strings`.
|
||||
|
||||
### .NET NativeAOT
|
||||
|
||||
NativeAOT binaries contain native machine code, so ordinary CIL decompilers
|
||||
|
||||
+133
-12
@@ -11,6 +11,7 @@ import math
|
||||
import re
|
||||
import resource
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote_to_bytes
|
||||
|
||||
from mitmproxy import ctx, version
|
||||
from mitmproxy.io import tnetstring
|
||||
@@ -41,6 +42,113 @@ class BoundedReader:
|
||||
return self.handle.read(count)
|
||||
|
||||
|
||||
SPECIAL_SCHEMES = {"ftp", "file", "http", "https", "ws", "wss"}
|
||||
SCHEME_PREFIX = r"^[\x00-\x20]*([a-z][a-z0-9+.-]*):"
|
||||
NETWORK_PREFIX = r"^[\x00-\x20]*[/\\]{2}"
|
||||
URL_IGNORABLE_CONTROLS = "\t\n\r"
|
||||
URL_CONTROL_TRANSLATION = {ord(char): None for char in URL_IGNORABLE_CONTROLS}
|
||||
FORBIDDEN_DOMAIN_CODE_POINTS = {chr(code) for code in range(0x21)} | set("#%/:<>?@[\\]^|\x7f")
|
||||
|
||||
|
||||
def authority_start(text, base_scheme=None):
|
||||
"""Return where an authority begins and whether it is special, or None.
|
||||
|
||||
Follows WHATWG URL parsing as src/domain/recordedTransportUrl.ts does:
|
||||
special schemes (and scheme-relative references) skip every leading "/"
|
||||
or "\\" before the authority, so `https://\\user:pass@host` still has
|
||||
userinfo. File URLs never carry userinfo: an "@" in a file host makes
|
||||
WHATWG reject the whole URL, so there are no credentials to remove.
|
||||
"""
|
||||
scheme = re.match(SCHEME_PREFIX, text, flags=re.I)
|
||||
name = scheme.group(1).lower() if scheme is not None else None
|
||||
if name == "file" or (name is None and base_scheme == "file"):
|
||||
return None
|
||||
if name is not None and name in SPECIAL_SCHEMES:
|
||||
# A same-scheme Location without two slash-equivalent separators is
|
||||
# relative to the recorded request, so its @ belongs to the path.
|
||||
following = text[scheme.end():scheme.end() + 2]
|
||||
if name == base_scheme and (len(following) < 2 or any(char not in "/\\" for char in following)):
|
||||
return None
|
||||
start = scheme.end()
|
||||
else:
|
||||
if scheme is not None:
|
||||
if text[scheme.end():scheme.end() + 2] != "//":
|
||||
return None
|
||||
return scheme.end() + 2, False
|
||||
# A non-special base treats backslashes as authority data and only
|
||||
# recognizes two forward slashes as a network reference.
|
||||
if base_scheme is not None and base_scheme not in SPECIAL_SCHEMES:
|
||||
prefix = re.match(r"^[\x00-\x20]*//", text)
|
||||
return None if prefix is None else (prefix.end(), False)
|
||||
prefix = re.match(NETWORK_PREFIX, text)
|
||||
if prefix is None:
|
||||
return None
|
||||
start = prefix.end()
|
||||
while start < len(text) and text[start] in "/\\":
|
||||
start += 1
|
||||
return start, True
|
||||
|
||||
|
||||
def special_host_parses(host_port):
|
||||
"""Whether WHATWG could accept a special authority's host and port.
|
||||
|
||||
A rejected host fails the whole URL, so it has no credentials to remove.
|
||||
This mirrors the forbidden-code-point, empty-host and port rules; IDNA,
|
||||
IPv4 range and IPv6 syntax failures are not modeled and still redact.
|
||||
"""
|
||||
if host_port.startswith("["):
|
||||
host_end = host_port.find("]") + 1
|
||||
if host_end == 0:
|
||||
return False
|
||||
else:
|
||||
colon = host_port.find(":")
|
||||
host_end = len(host_port) if colon < 0 else colon
|
||||
host = unquote_to_bytes(host_port[:host_end]).decode("utf-8", errors="replace")
|
||||
if host == "" or any(char in FORBIDDEN_DOMAIN_CODE_POINTS for char in host):
|
||||
return False
|
||||
port = host_port[host_end:]
|
||||
if port == "":
|
||||
return True
|
||||
digits = port[1:]
|
||||
return port[0] == ":" and (digits == "" or (digits.isascii() and digits.isdigit() and int(digits) <= 65535))
|
||||
|
||||
|
||||
def without_userinfo(value, base_scheme=None):
|
||||
"""Remove an authority's userinfo using the WHATWG authority boundary.
|
||||
|
||||
Special schemes and scheme-relative references also end the authority
|
||||
at a backslash, so later path text containing "@" is not userinfo.
|
||||
"""
|
||||
binary = isinstance(value, bytes)
|
||||
text = value.decode("latin-1") if binary else value
|
||||
# WHATWG removes these controls before interpreting scheme/separators.
|
||||
# Keep that parse view separate from the exact source spelling and bytes.
|
||||
parsed_text = text.translate(URL_CONTROL_TRANSLATION)
|
||||
located = authority_start(parsed_text, base_scheme)
|
||||
if located is None:
|
||||
return value
|
||||
start, special = located
|
||||
terminators = "/?#\\" if special else "/?#"
|
||||
end = start
|
||||
while end < len(parsed_text) and parsed_text[end] not in terminators:
|
||||
end += 1
|
||||
at = parsed_text.rfind("@", start, end)
|
||||
if at < start or (special and not special_host_parses(parsed_text[at + 1 : end])):
|
||||
return value
|
||||
parsed_offset = 0
|
||||
raw_start = 0
|
||||
for raw_offset, char in enumerate(text):
|
||||
if char in URL_IGNORABLE_CONTROLS:
|
||||
continue
|
||||
if parsed_offset == start:
|
||||
raw_start = raw_offset
|
||||
if parsed_offset == at:
|
||||
safe = text[:raw_start] + text[raw_offset + 1 :]
|
||||
return safe.encode("latin-1") if binary else safe
|
||||
parsed_offset += 1
|
||||
raise CaptureFailure("decoder", "Normalized URL authority could not be bound to its source.")
|
||||
|
||||
|
||||
def project_record(state, secrets):
|
||||
binaries, numbers, redactions = [], [], []
|
||||
byte_secrets = []
|
||||
@@ -58,16 +166,23 @@ def project_record(state, secrets):
|
||||
return None
|
||||
return text
|
||||
|
||||
def transport_value(value, path, transport_url):
|
||||
def transport_value(value, path, transport_url, base_scheme):
|
||||
if not (transport_url or re.fullmatch(r"/(?:backup/)*request/(?:path|authority)", path)):
|
||||
return value, False
|
||||
binary = isinstance(value, bytes)
|
||||
pattern = rb"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@" if binary else r"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@"
|
||||
replacement = rb"\1" if binary else r"\1"
|
||||
separator = b"@" if binary else "@"
|
||||
safe_url = re.sub(pattern, replacement, value, flags=re.I)
|
||||
if path.endswith("/authority") and separator in safe_url:
|
||||
safe_url = safe_url.rsplit(separator, 1)[1]
|
||||
if transport_url:
|
||||
safe_url = without_userinfo(value, base_scheme)
|
||||
elif path.endswith("/authority"):
|
||||
safe_url = value.rsplit(separator, 1)[-1]
|
||||
else:
|
||||
# Native request.path is origin-form unless it explicitly carries
|
||||
# a scheme (absolute-form). Leading separators are path evidence,
|
||||
# not a header URL reference's network authority.
|
||||
text = value.decode("latin-1") if binary else value
|
||||
safe_url = without_userinfo(value) if re.match(
|
||||
SCHEME_PREFIX, text.translate(URL_CONTROL_TRANSLATION), flags=re.I
|
||||
) else value
|
||||
changed = safe_url != value
|
||||
if changed:
|
||||
redactions.append({"pointer": path, "reason": "transport-credential"})
|
||||
@@ -80,7 +195,7 @@ def project_record(state, secrets):
|
||||
del numbers[before[1]:]
|
||||
del redactions[before[2]:]
|
||||
|
||||
def visit(value, path, depth=0, credential=False, transport_url=False):
|
||||
def visit(value, path, depth=0, credential=False, transport_url=False, base_scheme=None):
|
||||
if depth > 64:
|
||||
raise CaptureFailure("input-limit", "Native capture exceeds the 64-level evidence nesting budget.", path)
|
||||
if credential:
|
||||
@@ -90,7 +205,7 @@ def project_record(state, secrets):
|
||||
binaries.append({"pointer": path, "representation": "producer-bytes", "state": "redacted", "content_base64": None, "bytes": None, "sha256": None})
|
||||
return None
|
||||
if isinstance(value, bytes):
|
||||
safe_bytes, credential_url = transport_value(value, path, transport_url)
|
||||
safe_bytes, credential_url = transport_value(value, path, transport_url, base_scheme)
|
||||
content_base64 = base64.b64encode(value).decode("ascii")
|
||||
digest = hashlib.sha256(value).hexdigest()
|
||||
hidden = any(secret in value or secret in safe_bytes for secret in byte_secrets) or any(secret in content_base64 or secret in digest or secret in str(len(value)) for secret in secrets)
|
||||
@@ -107,7 +222,7 @@ def project_record(state, secrets):
|
||||
return None
|
||||
return safe_text
|
||||
if isinstance(value, str):
|
||||
safe_text, _ = transport_value(value, path, transport_url)
|
||||
safe_text, _ = transport_value(value, path, transport_url, base_scheme)
|
||||
return redact(safe_text, path, value)
|
||||
if isinstance(value, bool) or value is None:
|
||||
return value
|
||||
@@ -125,10 +240,16 @@ def project_record(state, secrets):
|
||||
for index, field in enumerate(value):
|
||||
if not isinstance(field, (list, tuple)) or len(field) != 2 or not isinstance(field[0], bytes):
|
||||
raise CaptureFailure("format", "Native header/trailer is not an ordered byte pair.", pointer(path, index))
|
||||
result.append([visit(field[0], pointer(pointer(path, index), 0), depth + 2), visit(field[1], pointer(pointer(path, index), 1), depth + 2, field[0].lower() in CREDENTIALS, field[0].lower() in {b"location", b"referer", b"origin"})])
|
||||
result.append([visit(field[0], pointer(pointer(path, index), 0), depth + 2), visit(field[1], pointer(pointer(path, index), 1), depth + 2, field[0].lower() in CREDENTIALS, field[0].lower() in {b"location", b"referer", b"origin"}, base_scheme if field[0].lower() == b"location" else None)])
|
||||
return result
|
||||
return [visit(item, pointer(path, index), depth + 1) for index, item in enumerate(value)]
|
||||
return [visit(item, pointer(path, index), depth + 1, base_scheme=base_scheme) for index, item in enumerate(value)]
|
||||
if isinstance(value, dict):
|
||||
if path == "" or re.fullmatch(r"/backup(?:/backup)*", path):
|
||||
request = value.get("request")
|
||||
scheme = request.get("scheme") if isinstance(request, dict) else None
|
||||
if isinstance(scheme, bytes):
|
||||
scheme = scheme.decode("latin-1")
|
||||
base_scheme = scheme.lower() if isinstance(scheme, str) and re.fullmatch(r"[a-z][a-z0-9+.-]*", scheme, flags=re.I) else None
|
||||
result = {}
|
||||
for key, item in value.items():
|
||||
if not isinstance(key, str):
|
||||
@@ -139,7 +260,7 @@ def project_record(state, secrets):
|
||||
validate_omitted(item, pointer(path, key), depth + 1)
|
||||
redactions.append({"pointer": path, "reason": "explicit-sensitive-value", "scope": "property-name"})
|
||||
continue
|
||||
result[key] = visit(item, pointer(path, key), depth + 1)
|
||||
result[key] = visit(item, pointer(path, key), depth + 1, base_scheme=base_scheme)
|
||||
return result
|
||||
raise CaptureFailure("decoder", "Native parser returned an unexpected value type: " + type(value).__name__ + ".", path)
|
||||
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Source-owned tests for the mitmproxy adapter's record projection."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
def load_capture():
|
||||
"""Load the adapter with fresh mitmproxy stand-ins scoped to its import."""
|
||||
fake_mitmproxy = types.ModuleType("mitmproxy")
|
||||
fake_mitmproxy.ctx = types.SimpleNamespace()
|
||||
fake_mitmproxy.version = types.SimpleNamespace(VERSION="12.2.3")
|
||||
fake_io = types.ModuleType("mitmproxy.io")
|
||||
fake_io.tnetstring = types.ModuleType("mitmproxy.io.tnetstring")
|
||||
stand_ins = {
|
||||
"mitmproxy": fake_mitmproxy,
|
||||
"mitmproxy.io": fake_io,
|
||||
"mitmproxy.io.tnetstring": fake_io.tnetstring,
|
||||
}
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"rea_mitmproxy_capture", os.path.join(os.path.dirname(__file__), "capture.py")
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
with mock.patch.dict(sys.modules, stand_ins):
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
capture = load_capture()
|
||||
|
||||
|
||||
def location(value, scheme=None):
|
||||
state = {"response": {"headers": [[b"Location", value]]}}
|
||||
if scheme is not None:
|
||||
state["request"] = {"scheme": scheme}
|
||||
record = capture.project_record(state, [])
|
||||
sidecar = next(
|
||||
field for field in record["binary_fields"] if field["pointer"] == "/response/headers/0/1"
|
||||
)
|
||||
return record["reported"]["response"]["headers"][0][1], sidecar, record["redactions"]
|
||||
|
||||
|
||||
class TransportUserinfoTests(unittest.TestCase):
|
||||
def test_ignorable_controls_cannot_hide_authority_credentials(self):
|
||||
for control in [b"\t", b"\n", b"\r"]:
|
||||
for value in [b"https://" + control + b"/user:pass@host/path", b"ht" + control + b"tps://user:" + control + b"pass@host/path"]:
|
||||
with self.subTest(value=value):
|
||||
text, sidecar, redactions = location(value, b"https")
|
||||
self.assertNotIn("user", text)
|
||||
self.assertNotIn("pass", text)
|
||||
self.assertEqual(sidecar["state"], "redacted")
|
||||
self.assertEqual(len(redactions), 1)
|
||||
|
||||
def test_mixed_network_separators_remove_credentials(self):
|
||||
for prefix in [b"\\\\", b"/\\", b"\\/"]:
|
||||
with self.subTest(prefix=prefix):
|
||||
text, sidecar, redactions = location(prefix + b"user:pass@host/path", b"https")
|
||||
self.assertEqual(text, prefix.decode("ascii") + "host/path")
|
||||
self.assertEqual(sidecar["state"], "redacted")
|
||||
self.assertEqual(len(redactions), 1)
|
||||
|
||||
def test_non_special_base_keeps_backslashes_inside_network_authorities(self):
|
||||
text, sidecar, redactions = location(b"//user\\name:pass@host/x", b"foo")
|
||||
self.assertEqual(text, "//host/x")
|
||||
self.assertEqual(sidecar["state"], "redacted")
|
||||
self.assertEqual(len(redactions), 1)
|
||||
text, sidecar, redactions = location(b"/\\invoice@customer", b"foo")
|
||||
self.assertEqual(text, "/\\invoice@customer")
|
||||
self.assertEqual(sidecar["state"], "retained")
|
||||
self.assertEqual(redactions, [])
|
||||
|
||||
def test_native_request_paths_preserve_origin_form_and_redact_absolute_form(self):
|
||||
for value in [b"/\\invoice@customer", b"//invoice@customer", b"\\\\invoice@customer"]:
|
||||
with self.subTest(value=value):
|
||||
record = capture.project_record({"request": {"path": value}}, [])
|
||||
self.assertEqual(record["reported"]["request"]["path"], value.decode("ascii"))
|
||||
self.assertEqual(record["binary_fields"][0]["state"], "retained")
|
||||
self.assertEqual(record["redactions"], [])
|
||||
record = capture.project_record({"request": {"path": b"https://user:pass@host/x"}}, [])
|
||||
self.assertEqual(record["reported"]["request"]["path"], "https://host/x")
|
||||
self.assertEqual(record["binary_fields"][0]["state"], "redacted")
|
||||
|
||||
def test_same_scheme_locations_preserve_relative_path_text_and_bytes(self):
|
||||
for value in [b"https:invoice@customer", b"https:/invoice@customer", b"https:\\invoice@customer"]:
|
||||
with self.subTest(value=value):
|
||||
text, sidecar, redactions = location(value, b"https")
|
||||
self.assertEqual(text, value.decode("ascii"))
|
||||
self.assertEqual(sidecar["state"], "retained")
|
||||
self.assertEqual(redactions, [])
|
||||
|
||||
def test_location_authorities_still_remove_credentials_with_a_base_scheme(self):
|
||||
for value, scheme, expected in [
|
||||
(b"https://user:pass@host/path", b"https", "https://host/path"),
|
||||
(b"https:\\\\user:pass@host/path", b"https", "https:\\\\host/path"),
|
||||
(b"https:user:pass@host/path", b"http", "https:host/path"),
|
||||
]:
|
||||
with self.subTest(value=value, scheme=scheme):
|
||||
text, sidecar, redactions = location(value, scheme)
|
||||
self.assertEqual(text, expected)
|
||||
self.assertEqual(sidecar["state"], "redacted")
|
||||
self.assertEqual(len(redactions), 1)
|
||||
|
||||
def test_backup_location_uses_its_own_request_scheme(self):
|
||||
record = capture.project_record({
|
||||
"request": {"scheme": b"http"},
|
||||
"backup": {
|
||||
"request": {"scheme": b"https"},
|
||||
"response": {"headers": [[b"Location", b"https:invoice@customer"]]},
|
||||
},
|
||||
}, [])
|
||||
self.assertEqual(record["reported"]["backup"]["response"]["headers"][0][1], "https:invoice@customer")
|
||||
self.assertEqual(record["redactions"], [])
|
||||
|
||||
def test_backslash_ends_a_special_scheme_authority(self):
|
||||
text, sidecar, redactions = location(b"https://example.test\\part@tail")
|
||||
self.assertEqual(text, "https://example.test\\part@tail")
|
||||
self.assertEqual(sidecar["state"], "retained")
|
||||
self.assertIsNotNone(sidecar["content_base64"])
|
||||
self.assertEqual(redactions, [])
|
||||
|
||||
def test_userinfo_before_a_backslash_is_still_removed(self):
|
||||
text, sidecar, redactions = location(b"https://user:pw@example.test\\path")
|
||||
self.assertEqual(text, "https://example.test\\path")
|
||||
self.assertEqual(sidecar["state"], "redacted")
|
||||
self.assertEqual(
|
||||
redactions,
|
||||
[{"pointer": "/response/headers/0/1", "reason": "transport-credential"}],
|
||||
)
|
||||
|
||||
def test_ordinary_userinfo_is_removed(self):
|
||||
text, _sidecar, _redactions = location(b"https://user:pw@example.test/a@b")
|
||||
self.assertEqual(text, "https://example.test/a@b")
|
||||
|
||||
def test_empty_userinfo_marker_is_removed(self):
|
||||
self.assertEqual(
|
||||
capture.without_userinfo("https://@host"), "https://host"
|
||||
)
|
||||
|
||||
def test_special_schemes_skip_leading_slashes_before_userinfo(self):
|
||||
for value, expected in [
|
||||
(b"https://\\user:pass@host/path", b"https://\\host/path"),
|
||||
(b"https:\\\\user:pass@host/path", b"https:\\\\host/path"),
|
||||
(b"https:user:pass@host", b"https:host"),
|
||||
(b"//\\user:pass@host", b"//\\host"),
|
||||
]:
|
||||
with self.subTest(value=value):
|
||||
self.assertEqual(capture.without_userinfo(value), expected)
|
||||
text, sidecar, redactions = location(b"https://\\user:pass@host/path")
|
||||
self.assertEqual(text, "https://\\host/path")
|
||||
self.assertEqual(sidecar["state"], "redacted")
|
||||
self.assertEqual(len(redactions), 1)
|
||||
|
||||
def test_file_base_keeps_paths_after_the_fixed_network_boundary(self):
|
||||
for value in [b"///u@h", b"////u@h"]:
|
||||
with self.subTest(value=value):
|
||||
text, sidecar, redactions = location(value, b"file")
|
||||
self.assertEqual(text, value.decode("ascii"))
|
||||
self.assertEqual(sidecar["state"], "retained")
|
||||
self.assertIsNotNone(sidecar["content_base64"])
|
||||
self.assertEqual(redactions, [])
|
||||
|
||||
def test_file_urls_keep_path_text(self):
|
||||
self.assertEqual(capture.without_userinfo("file:///a@b"), "file:///a@b")
|
||||
|
||||
def test_file_urls_reject_userinfo_and_keep_their_source(self):
|
||||
# WHATWG rejects an "@" in a file host, so nothing parses as credentials.
|
||||
self.assertEqual(
|
||||
capture.without_userinfo("file://user:pass@host/x"), "file://user:pass@host/x"
|
||||
)
|
||||
for value in [b"\\\\user:pass@host/x", b"/\\user:pass@host/x", b"//user:pass@host/x"]:
|
||||
with self.subTest(value=value):
|
||||
text, sidecar, redactions = location(value, b"file")
|
||||
self.assertEqual(text, value.decode("ascii"))
|
||||
self.assertEqual(sidecar["state"], "retained")
|
||||
self.assertEqual(redactions, [])
|
||||
|
||||
def test_rejected_special_hosts_keep_their_source(self):
|
||||
# A rejected host or port fails the whole URL, so it has no userinfo.
|
||||
for value in [
|
||||
"https:user@%",
|
||||
"https://user@",
|
||||
"https://user:pass@h^st/x",
|
||||
"https://user@ho st/x",
|
||||
"https://user@host:99999/x",
|
||||
"https://user@host:12a",
|
||||
"https://user@[::1",
|
||||
]:
|
||||
with self.subTest(value=value):
|
||||
self.assertEqual(capture.without_userinfo(value), value)
|
||||
text, sidecar, redactions = location(b"https://user:pass@h^st/x", b"https")
|
||||
self.assertEqual(text, "https://user:pass@h^st/x")
|
||||
self.assertEqual(sidecar["state"], "retained")
|
||||
self.assertEqual(redactions, [])
|
||||
|
||||
def test_accepted_special_hosts_still_remove_userinfo(self):
|
||||
for value, expected in [
|
||||
("https:user@%41", "https:%41"),
|
||||
("https://user@[::1]:8443/x", "https://[::1]:8443/x"),
|
||||
("https://user@host:/x", "https://host:/x"),
|
||||
("wss://user:pass@host:65535", "wss://host:65535"),
|
||||
]:
|
||||
with self.subTest(value=value):
|
||||
self.assertEqual(capture.without_userinfo(value), expected)
|
||||
|
||||
def test_non_special_schemes_keep_backslashes_in_the_authority(self):
|
||||
self.assertEqual(capture.without_userinfo("foo://a\\b@host/x"), "foo://host/x")
|
||||
self.assertEqual(capture.without_userinfo(b"//a\\b@host"), b"//a\\b@host")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -81,40 +81,6 @@ including the existing `inspect-analysis-view` CLI. Session record lifetime and
|
||||
export/import semantics remain unchanged. A server without Evidence retention
|
||||
rejects selected summary before analysis; complete delivery remains available.
|
||||
|
||||
### Member summary and retained member rows
|
||||
|
||||
Complete `inspect_managed_members` delivery always includes the assembly-wide
|
||||
`member_refs` and `exported_types`, which `type_names` does not narrow, so even
|
||||
one selected or unmatched type can return megabytes. Select
|
||||
`{"path":"/absolute/path/Assembly.dll","type_names":["Namespace.Type"],"detail":"summary"}`
|
||||
to keep the complete source in the session and receive identity, metadata
|
||||
status, the selection with its unmatched names, coverage/issues, the separate
|
||||
user-string coverage, limitations and per-collection counts. Then page original
|
||||
rows with view kind `managed-members`:
|
||||
|
||||
```json
|
||||
{
|
||||
"source": {
|
||||
"kind": "retained-evidence",
|
||||
"evidence_id": "<parent_evidence_id>"
|
||||
},
|
||||
"view": {
|
||||
"kind": "managed-members",
|
||||
"collection": "call_edges",
|
||||
"offset": 0,
|
||||
"limit": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Collections are `types`, `exported_types`, `fields`, `methods`, `member_refs`,
|
||||
`call_edges`, `field_accesses` and `user_strings` (the literals). Each page names
|
||||
its collection, retains original rows, order, tokens, signatures and body facts,
|
||||
and carries the source selection and coverage; `user_strings` pages carry the
|
||||
literal pool's own coverage. Older Evidence that never examined exported types
|
||||
or user strings reports a null count and refuses those pages rather than
|
||||
returning an empty one.
|
||||
|
||||
## Shipped scope
|
||||
|
||||
The canonical parser admits PE/CLI bytes and their metadata/CIL without loading
|
||||
|
||||
+1
-1
@@ -138,7 +138,7 @@
|
||||
"test": "npm run test:with-build",
|
||||
"test:with-build": "npm run test:prepare && vitest run",
|
||||
"test:run": "vitest run",
|
||||
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v",
|
||||
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py bridge/mitmproxy/test_capture.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v",
|
||||
"test:local": "node scripts/run-development-tests.mjs local",
|
||||
"test:fast": "node scripts/ci/test-group.mjs source",
|
||||
"test:built": "node scripts/ci/test-group.mjs built",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
export function assertCleanLinuxHopperRuntime(
|
||||
launcherPaths?: readonly string[],
|
||||
procRoot?: string,
|
||||
listProcesses?: () => Promise<{ stdout: string }>,
|
||||
): Promise<void>;
|
||||
|
||||
@@ -17,8 +17,9 @@ export const assertCleanLinuxHopperRuntime = async (
|
||||
: [process.env.HOPPER_LAUNCHER_PATH]),
|
||||
],
|
||||
procRoot = "/proc",
|
||||
listProcesses = () => execFileAsync("ps", ["-ax", "-o", "command="]),
|
||||
) => {
|
||||
const { stdout } = await execFileAsync("ps", ["-ax", "-o", "command="]);
|
||||
const { stdout } = await listProcesses();
|
||||
if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(stdout))
|
||||
throw new Error("Clean Linux runner retained a Hopper demo process");
|
||||
const launchers = [];
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -21,13 +21,13 @@ export const managedTargetInputSchema = z.strictObject({
|
||||
.optional(),
|
||||
});
|
||||
|
||||
/** Delivery selection belongs to artifact and member inspection, not native queries. */
|
||||
/** Delivery selection belongs only to artifact inspection, not member/native queries. */
|
||||
export const managedArtifactInputSchema = managedTargetInputSchema.extend({
|
||||
detail: analysisDetailSchema.unwrap().optional(),
|
||||
});
|
||||
|
||||
/** Member inspection may select declaring types before members are decoded. */
|
||||
export const managedMembersInputSchema = managedArtifactInputSchema.extend({
|
||||
export const managedMembersInputSchema = managedTargetInputSchema.extend({
|
||||
type_names: z
|
||||
.array(
|
||||
z
|
||||
@@ -87,7 +87,7 @@ export const MANAGED_TOOL_CONTRACTS = [
|
||||
name: "inspect_managed_members",
|
||||
...toolContractMetadata("inspect_managed_members"),
|
||||
description:
|
||||
'Inspect PE/CLI metadata members, signatures, raw CIL hashes, decoded-instruction-tuple hashes, separately reported exception regions, call edges, field-access anchors, and the #US string literals that ldstr loads, without loading or executing target code. Select type_names to decode only the members of those exact declaring types and the literals their bodies load, which keeps large assemblies such as a base class library within the representation budget. Metadata tokens are reported as build-local coordinates bound to the artifact SHA-256 and MVID; the tuple hash does not resolve them or fully commit control flow. Omitted or complete detail returns every collection, including the assembly-wide member_refs and exported_types, which type_names does not narrow. Select detail: "summary" for identity, metadata status, selection with unmatched names, coverage/issues, limitations and per-collection counts; it retains complete Evidence and returns a linked inspect_analysis_view summary. Page original rows with view kind managed-members, normalized_result.parent_evidence_id and collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings.',
|
||||
"Inspect PE/CLI metadata members, signatures, raw CIL hashes, decoded-instruction-tuple hashes, separately reported exception regions, call edges, field-access anchors, and the #US string literals that ldstr loads, without loading or executing target code. Select type_names to decode only the members of those exact declaring types and the literals their bodies load, which keeps large assemblies such as a base class library within the representation budget. Metadata tokens are reported as build-local coordinates bound to the artifact SHA-256 and MVID; the tuple hash does not resolve them or fully commit control flow.",
|
||||
kind: "managed-provider",
|
||||
inputSchema: managedMembersInputSchema,
|
||||
outputSchema: memberOutputSchema,
|
||||
@@ -103,14 +103,6 @@ export const MANAGED_TOOL_CONTRACTS = [
|
||||
type_names: ["System.Threading.Monitor"],
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Summarize selected managed types and retain their members",
|
||||
input: {
|
||||
path: "/tmp/mscorlib.dll",
|
||||
type_names: ["System.Threading.Monitor"],
|
||||
detail: "summary",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -4,7 +4,6 @@ import {
|
||||
artifactInventorySummaryViewResultSchema,
|
||||
machoSummaryViewResultSchema,
|
||||
managedArtifactSummaryViewResultSchema,
|
||||
managedMemberSummaryViewResultSchema,
|
||||
} from "../domain/analysisView/analysisView.js";
|
||||
import { evidenceBundleSummarySchema } from "../domain/evidenceRecordSummary.js";
|
||||
import { BINARY_ARCHITECTURES } from "../domain/binaryTargetTypes.js";
|
||||
@@ -419,12 +418,7 @@ export const managedOutputSchemas: Readonly<Record<string, z.ZodObject>> = {
|
||||
managedArtifactSummaryViewResultSchema,
|
||||
]),
|
||||
),
|
||||
inspect_managed_members: evidenceResultOf(
|
||||
z.union([
|
||||
managedMemberInspectionSchema,
|
||||
managedMemberSummaryViewResultSchema,
|
||||
]),
|
||||
),
|
||||
inspect_managed_members: evidenceResultOf(managedMemberInspectionSchema),
|
||||
inspect_managed_native_boundaries: evidenceResultOf(
|
||||
managedNativeBoundaryInspectionSchema,
|
||||
),
|
||||
|
||||
@@ -49,13 +49,6 @@ import {
|
||||
managedSummarySchema,
|
||||
managedViewContextSchema,
|
||||
} from "./managedArtifactView.js";
|
||||
import {
|
||||
projectManagedMemberView,
|
||||
managedMembersViewRequestSchema,
|
||||
managedMembersViewPageSchema,
|
||||
managedMemberSummarySchema,
|
||||
managedMembersViewContextSchema,
|
||||
} from "./managedMemberView.js";
|
||||
import {
|
||||
projectWasmArtifactView,
|
||||
wasmViewRequestSchema,
|
||||
@@ -215,7 +208,6 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [
|
||||
machoViewRequestSchema,
|
||||
evmViewRequestSchema,
|
||||
managedViewRequestSchema,
|
||||
managedMembersViewRequestSchema,
|
||||
androidViewRequestSchema,
|
||||
apiSurfaceViewSchema,
|
||||
captureViewRequestSchema,
|
||||
@@ -336,7 +328,6 @@ const viewResultBase = {
|
||||
"inspect_macho",
|
||||
"inspect_evm_interface",
|
||||
"inspect_managed_artifact",
|
||||
"inspect_managed_members",
|
||||
"inspect_web_network_capture",
|
||||
"inspect_android_package",
|
||||
"inspect_pcap_capture",
|
||||
@@ -405,13 +396,6 @@ export const managedArtifactSummaryViewResultSchema = z.strictObject({
|
||||
summary: managedSummarySchema,
|
||||
});
|
||||
|
||||
/** Exact member selection, coverage and counts for the selected delivery. */
|
||||
export const managedMemberSummaryViewResultSchema = z.strictObject({
|
||||
...summaryViewResultBase,
|
||||
parent_operation: z.literal("inspect_managed_members"),
|
||||
summary: managedMemberSummarySchema,
|
||||
});
|
||||
|
||||
/** Exact historical capture summary, with no body or member-inventory payload. */
|
||||
export const webNetworkCaptureSummaryViewResultSchema = z.strictObject({
|
||||
...summaryViewResultBase,
|
||||
@@ -462,7 +446,6 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [
|
||||
machoSummarySchema,
|
||||
evmSummarySchema,
|
||||
managedSummarySchema,
|
||||
managedMemberSummarySchema,
|
||||
captureSummarySchema,
|
||||
androidPackageSummarySchema,
|
||||
pcapSummarySchema,
|
||||
@@ -546,14 +529,6 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [
|
||||
...managedViewContextSchema.shape,
|
||||
parent_operation: z.literal("inspect_managed_artifact"),
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("managed-members"),
|
||||
view: managedMembersViewRequestSchema,
|
||||
page: managedMembersViewPageSchema,
|
||||
...viewResultBase,
|
||||
...managedMembersViewContextSchema.shape,
|
||||
parent_operation: z.literal("inspect_managed_members"),
|
||||
}),
|
||||
z.strictObject({
|
||||
kind: z.literal("android"),
|
||||
view: androidViewRequestSchema,
|
||||
@@ -647,10 +622,6 @@ export type UnsignedAnalysisView =
|
||||
Extract<AnalysisViewResult, { readonly kind: "managed" }>,
|
||||
"view_digest"
|
||||
>
|
||||
| Omit<
|
||||
Extract<AnalysisViewResult, { readonly kind: "managed-members" }>,
|
||||
"view_digest"
|
||||
>
|
||||
| Omit<
|
||||
Extract<AnalysisViewResult, { readonly kind: "android" }>,
|
||||
"view_digest"
|
||||
@@ -703,7 +674,7 @@ const unsupportedParent = (
|
||||
new AnalysisUnsupportedTargetError(
|
||||
INSPECT_ANALYSIS_VIEW_OPERATION,
|
||||
parentArtifactPath(parent),
|
||||
`Parent Evidence operation ${JSON.stringify(parent.operation)} does not support analysis views. Use completed binary layout, JavaScript application, native function, artifact inventory/inspection, WASM, Mach-O, EVM, managed artifact or member, Android package, Dart AOT, or web network capture Evidence.`,
|
||||
`Parent Evidence operation ${JSON.stringify(parent.operation)} does not support analysis views. Use completed binary layout, JavaScript application, native function, artifact inventory/inspection, WASM, Mach-O, EVM, managed artifact, Android package, Dart AOT, or web network capture Evidence.`,
|
||||
);
|
||||
|
||||
const parentArtifactPath = (parent: AnalysisViewParent): string => {
|
||||
@@ -733,7 +704,6 @@ const retainedViewProjectors = new Map<string, typeof projectWasmArtifactView>([
|
||||
["inspect_macho", projectMachoView],
|
||||
["inspect_evm_interface", projectEvmInterfaceView],
|
||||
["inspect_managed_artifact", projectManagedArtifactView],
|
||||
["inspect_managed_members", projectManagedMemberView],
|
||||
["inspect_android_package", projectAndroidPackageView],
|
||||
["inspect_dart_aot", projectDartAotView],
|
||||
["inspect_artifact", projectArtifactInventoryView],
|
||||
|
||||
@@ -51,24 +51,6 @@ export const managedViewContextSchema = z.strictObject({
|
||||
source_coverage: managedArtifactInspectionSchema.shape.coverage,
|
||||
});
|
||||
|
||||
/** Token scope and identity limits shared by every managed metadata view. */
|
||||
export const managedViewIdentityUnknowns = (
|
||||
parent: AnalysisViewParent,
|
||||
mvid: string | null,
|
||||
): string[] => [
|
||||
"Metadata tokens are build-local coordinates bound to the recorded artifact SHA-256 and module MVID; this view does not load or execute the assembly.",
|
||||
...(parent.subject == null
|
||||
? [
|
||||
"The parent has no recorded subject; its normalized artifact identity cannot be cross-checked against a subject digest.",
|
||||
]
|
||||
: []),
|
||||
...(mvid === null
|
||||
? [
|
||||
"The module MVID is unknown; tokens cannot establish a cross-observation module match.",
|
||||
]
|
||||
: []),
|
||||
];
|
||||
|
||||
/** Project authenticated metadata without reading a path or invoking a provider. */
|
||||
export const projectManagedArtifactView = (
|
||||
parent: AnalysisViewParent,
|
||||
@@ -114,7 +96,17 @@ export const projectManagedArtifactView = (
|
||||
limitations: uniqueSorted([...parent.limitations, ...report.limitations]),
|
||||
unknowns: [
|
||||
"Collection counts and pages cover producer-returned rows; metadata completeness remains the original metadata status and coverage, not an absence claim about unreturned rows.",
|
||||
...managedViewIdentityUnknowns(parent, identity_scope.requires_mvid),
|
||||
"Metadata tokens are build-local coordinates bound to the recorded artifact SHA-256 and module MVID; this view does not load or execute the assembly.",
|
||||
...(parent.subject == null
|
||||
? [
|
||||
"The parent has no recorded subject; its normalized artifact identity cannot be cross-checked against a subject digest.",
|
||||
]
|
||||
: []),
|
||||
...(identity_scope.requires_mvid === null
|
||||
? [
|
||||
"The module MVID is unknown; tokens cannot establish a cross-observation module match.",
|
||||
]
|
||||
: []),
|
||||
],
|
||||
};
|
||||
if (view.kind === "summary") {
|
||||
|
||||
@@ -1,270 +0,0 @@
|
||||
import { z } from "zod";
|
||||
import type { AnalysisError } from "../analysisErrorBase.js";
|
||||
import { uniqueSorted } from "../canonicalOrdering.js";
|
||||
import { analysisInputErrorFromIssues } from "../inputIssueProjection.js";
|
||||
import {
|
||||
managedMemberInspectionSchema,
|
||||
type ManagedMemberInspection,
|
||||
} from "../managed/managedArtifact.js";
|
||||
import { err, ok, type Result } from "../result.js";
|
||||
import {
|
||||
analysisViewInputError,
|
||||
type AnalysisViewParent,
|
||||
type AnalysisViewRequest,
|
||||
type UnsignedAnalysisView,
|
||||
} from "./analysisView.js";
|
||||
import {
|
||||
completeWithinViewCoverage,
|
||||
pageViewCoverage,
|
||||
} from "./analysisViewCoverage.js";
|
||||
import { managedViewIdentityUnknowns } from "./managedArtifactView.js";
|
||||
|
||||
const member = managedMemberInspectionSchema.shape;
|
||||
|
||||
/** Original member rows; offsets count entries in the returned parent collection. */
|
||||
export const managedMembersViewRequestSchema = z.strictObject({
|
||||
kind: z.literal("managed-members"),
|
||||
collection: z.enum([
|
||||
"types",
|
||||
"exported_types",
|
||||
"fields",
|
||||
"methods",
|
||||
"member_refs",
|
||||
"call_edges",
|
||||
"field_accesses",
|
||||
"user_strings",
|
||||
]),
|
||||
offset: z.number().int().nonnegative().default(0),
|
||||
limit: z.number().int().positive().default(64),
|
||||
});
|
||||
|
||||
const rowsOf = <C extends string, S extends z.ZodArray>(
|
||||
collection: C,
|
||||
rows: S,
|
||||
) => z.strictObject({ collection: z.literal(collection), rows });
|
||||
|
||||
/**
|
||||
* Rows of one collection, keyed by that collection: several row schemas accept
|
||||
* each other's rows after stripping keys, so no union may pick one by order.
|
||||
*/
|
||||
export const managedMembersViewPageSchema = z.discriminatedUnion("collection", [
|
||||
rowsOf("types", member.types),
|
||||
rowsOf("exported_types", member.exported_types.unwrap().unwrap()),
|
||||
rowsOf("fields", member.fields),
|
||||
rowsOf("methods", member.methods),
|
||||
rowsOf("member_refs", member.member_refs),
|
||||
rowsOf("call_edges", member.call_edges),
|
||||
rowsOf("field_accesses", member.field_accesses),
|
||||
rowsOf("user_strings", member.user_strings.unwrap().unwrap().shape.literals),
|
||||
]);
|
||||
|
||||
type ManagedMembersViewPage = z.infer<typeof managedMembersViewPageSchema>;
|
||||
|
||||
const nullableCount = z.number().int().nonnegative().nullable();
|
||||
|
||||
/** Keep identity, selection and coverage; member collections remain retained. */
|
||||
export const managedMemberSummarySchema = z.strictObject({
|
||||
artifact: member.artifact,
|
||||
module: member.module,
|
||||
metadata: member.metadata,
|
||||
identity_scope: member.identity_scope,
|
||||
selection: member.selection.unwrap(),
|
||||
coverage: member.coverage,
|
||||
user_strings: z
|
||||
.strictObject({
|
||||
state: member.user_strings.unwrap().unwrap().shape.state,
|
||||
issues: member.user_strings.unwrap().unwrap().shape.issues,
|
||||
})
|
||||
.nullable(),
|
||||
limitations: member.limitations,
|
||||
reported_counts: z.strictObject({
|
||||
types: z.number().int().nonnegative(),
|
||||
exported_types: nullableCount,
|
||||
fields: z.number().int().nonnegative(),
|
||||
methods: z.number().int().nonnegative(),
|
||||
member_refs: z.number().int().nonnegative(),
|
||||
call_edges: z.number().int().nonnegative(),
|
||||
field_accesses: z.number().int().nonnegative(),
|
||||
user_strings: nullableCount,
|
||||
}),
|
||||
});
|
||||
|
||||
export const managedMembersViewContextSchema = z.strictObject({
|
||||
identity_scope: member.identity_scope,
|
||||
selection: member.selection.unwrap(),
|
||||
source_metadata_status: member.metadata.shape.status,
|
||||
source_coverage: member.coverage,
|
||||
});
|
||||
|
||||
type ManagedMembersViewRequest = z.infer<
|
||||
typeof managedMembersViewRequestSchema
|
||||
>;
|
||||
type MemberCollection = ManagedMembersViewRequest["collection"];
|
||||
|
||||
const slice = <C extends MemberCollection, T>(
|
||||
collection: C,
|
||||
rows: readonly T[] | null | undefined,
|
||||
{ offset, limit }: { readonly offset: number; readonly limit: number },
|
||||
) =>
|
||||
rows == null
|
||||
? null
|
||||
: {
|
||||
page: { collection, rows: rows.slice(offset, offset + limit) },
|
||||
total: rows.length,
|
||||
};
|
||||
|
||||
/** One page of a collection, or null when the parent never examined it. */
|
||||
const collectionPage = (
|
||||
report: ManagedMemberInspection,
|
||||
view: ManagedMembersViewRequest,
|
||||
): { readonly page: ManagedMembersViewPage; readonly total: number } | null => {
|
||||
const { collection } = view;
|
||||
switch (collection) {
|
||||
case "types":
|
||||
return slice(collection, report.types, view);
|
||||
case "exported_types":
|
||||
return slice(collection, report.exported_types, view);
|
||||
case "fields":
|
||||
return slice(collection, report.fields, view);
|
||||
case "methods":
|
||||
return slice(collection, report.methods, view);
|
||||
case "member_refs":
|
||||
return slice(collection, report.member_refs, view);
|
||||
case "call_edges":
|
||||
return slice(collection, report.call_edges, view);
|
||||
case "field_accesses":
|
||||
return slice(collection, report.field_accesses, view);
|
||||
case "user_strings":
|
||||
return slice(collection, report.user_strings?.literals, view);
|
||||
}
|
||||
};
|
||||
|
||||
/** Everything but the member collections, which are replaced by their counts. */
|
||||
const summarizeMembers = (
|
||||
report: ManagedMemberInspection,
|
||||
): z.infer<typeof managedMemberSummarySchema> => {
|
||||
const {
|
||||
types,
|
||||
exported_types,
|
||||
fields,
|
||||
methods,
|
||||
member_refs,
|
||||
call_edges,
|
||||
field_accesses,
|
||||
user_strings,
|
||||
...identity
|
||||
} = report;
|
||||
return {
|
||||
...identity,
|
||||
user_strings:
|
||||
user_strings === null
|
||||
? null
|
||||
: { state: user_strings.state, issues: user_strings.issues },
|
||||
reported_counts: {
|
||||
types: types.length,
|
||||
exported_types: exported_types?.length ?? null,
|
||||
fields: fields.length,
|
||||
methods: methods.length,
|
||||
member_refs: member_refs.length,
|
||||
call_edges: call_edges.length,
|
||||
field_accesses: field_accesses.length,
|
||||
user_strings: user_strings?.literals.length ?? null,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/** Project authenticated member Evidence without reading a path or invoking a provider. */
|
||||
export const projectManagedMemberView = (
|
||||
parent: AnalysisViewParent,
|
||||
view: AnalysisViewRequest,
|
||||
): Result<UnsignedAnalysisView, AnalysisError> => {
|
||||
if (view.kind !== "summary" && view.kind !== "managed-members")
|
||||
return err(
|
||||
analysisViewInputError([
|
||||
{
|
||||
path: ["view"],
|
||||
reason: "invalid_value",
|
||||
message:
|
||||
"Managed member inspection supports summary and managed-members views.",
|
||||
},
|
||||
]),
|
||||
);
|
||||
const parsed = managedMemberInspectionSchema.safeParse(
|
||||
parent.normalizedResult,
|
||||
);
|
||||
if (!parsed.success)
|
||||
return err(
|
||||
analysisInputErrorFromIssues(
|
||||
"inspect_analysis_view",
|
||||
parsed.error.issues.map((issue) => ({
|
||||
...issue,
|
||||
path: ["source", "normalized_result", ...issue.path],
|
||||
})),
|
||||
{ source: { normalized_result: parent.normalizedResult } },
|
||||
{ cause: parsed.error },
|
||||
),
|
||||
);
|
||||
const report = parsed.data;
|
||||
const shared = {
|
||||
parent_evidence_id: parent.evidenceId,
|
||||
parent_operation: "inspect_managed_members" as const,
|
||||
parent_digest: parent.evidenceId.slice(3),
|
||||
artifact: { path: report.artifact.path, sha256: report.artifact.sha256 },
|
||||
limitations: uniqueSorted([...parent.limitations, ...report.limitations]),
|
||||
unknowns: [
|
||||
"Collection counts and pages cover producer-returned rows; member completeness remains the original coverage, not an absence claim about unreturned rows.",
|
||||
...(report.selection === null
|
||||
? []
|
||||
: [
|
||||
"Exported types and member references cover the whole assembly; the other collections cover only the selected declaring types.",
|
||||
]),
|
||||
...managedViewIdentityUnknowns(
|
||||
parent,
|
||||
report.identity_scope.requires_mvid,
|
||||
),
|
||||
],
|
||||
};
|
||||
if (view.kind === "summary") {
|
||||
const summary = summarizeMembers(report);
|
||||
const total = Object.values(summary.reported_counts).reduce<number>(
|
||||
(sum, count) => sum + (count ?? 0),
|
||||
0,
|
||||
);
|
||||
return ok({
|
||||
kind: "summary",
|
||||
view,
|
||||
...shared,
|
||||
summary,
|
||||
coverage: completeWithinViewCoverage(total, total),
|
||||
});
|
||||
}
|
||||
const selected = collectionPage(report, view);
|
||||
if (selected === null)
|
||||
return err(
|
||||
analysisViewInputError([
|
||||
{
|
||||
path: ["view", "collection"],
|
||||
reason: "invalid_value",
|
||||
message: `The parent Evidence did not examine ${view.collection}; repeat inspect_managed_members with the current runtime to record it.`,
|
||||
},
|
||||
]),
|
||||
);
|
||||
const { page, total } = selected;
|
||||
return ok({
|
||||
kind: "managed-members",
|
||||
view,
|
||||
page,
|
||||
...shared,
|
||||
identity_scope: report.identity_scope,
|
||||
selection: report.selection,
|
||||
source_metadata_status: report.metadata.status,
|
||||
source_coverage:
|
||||
view.collection === "user_strings" && report.user_strings !== null
|
||||
? {
|
||||
state: report.user_strings.state,
|
||||
issues: report.user_strings.issues,
|
||||
}
|
||||
: report.coverage,
|
||||
coverage: pageViewCoverage(view.offset, page.rows.length, total),
|
||||
});
|
||||
};
|
||||
@@ -1,4 +1,3 @@
|
||||
import { relationshipCoverageLimitations } from "./relationshipCoverage.js";
|
||||
import {
|
||||
callPathInputSchema,
|
||||
callPathResultSchema,
|
||||
@@ -241,12 +240,15 @@ const inspectSearch = ({
|
||||
const coverage = snapshot.evidence.relationship_coverage;
|
||||
for (const limitation of snapshot.limitations)
|
||||
limitations.push(limitation);
|
||||
if (coverage !== null) {
|
||||
for (const limitation of relationshipCoverageLimitations(coverage))
|
||||
if (coverage?.status === "partial")
|
||||
for (const limitation of coverage.limitations)
|
||||
limitations.push(limitation);
|
||||
for (const diagnostic of coverage.diagnostics ?? [])
|
||||
limitations.push(diagnostic);
|
||||
}
|
||||
else if (coverage?.status === "unknown")
|
||||
limitations.push(coverage.reason);
|
||||
for (const diagnostic of coverage !== null && "diagnostics" in coverage
|
||||
? coverage.diagnostics
|
||||
: [])
|
||||
limitations.push(diagnostic);
|
||||
// The requested goal is a terminal for this search. Its outgoing edges
|
||||
// cannot add a path to the goal, so its dossier limits are contextual
|
||||
// but do not make the searched start-to-goal closure incomplete.
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
import {
|
||||
relationshipCoverageLimitations,
|
||||
type RelationshipCoverage,
|
||||
} from "./relationshipCoverage.js";
|
||||
import type { RelationshipCoverage } from "./relationshipCoverage.js";
|
||||
import type { FunctionDossier } from "./hopperValues.js";
|
||||
import { uniqueSorted } from "./canonicalOrdering.js";
|
||||
|
||||
@@ -18,7 +15,11 @@ export const combineFunctionRelationshipCoverage = (
|
||||
{ direction: "callees" as const, coverage: callees },
|
||||
];
|
||||
const diagnostics = uniqueSorted(
|
||||
reads.flatMap(({ coverage }) => coverage?.diagnostics ?? []),
|
||||
reads.flatMap(({ coverage }) =>
|
||||
coverage !== undefined && "diagnostics" in coverage
|
||||
? coverage.diagnostics
|
||||
: [],
|
||||
),
|
||||
);
|
||||
const details = diagnostics.length > 0 ? { diagnostics } : {};
|
||||
const callersLimited = callers?.status !== "complete";
|
||||
@@ -34,9 +35,15 @@ export const combineFunctionRelationshipCoverage = (
|
||||
const reasons = reads.flatMap(({ direction, coverage }) => {
|
||||
const label =
|
||||
direction === "callers" ? callerCoveragePrefix : calleeCoveragePrefix;
|
||||
return relationshipCoverageLimitations(coverage).map(
|
||||
(reason) => `${label}${reason}`,
|
||||
);
|
||||
const limitations =
|
||||
coverage === undefined
|
||||
? ["Provider did not report relationship coverage."]
|
||||
: coverage.status === "unknown"
|
||||
? [coverage.reason]
|
||||
: coverage.status === "partial"
|
||||
? coverage.limitations
|
||||
: [];
|
||||
return limitations.map((reason) => `${label}${reason}`);
|
||||
});
|
||||
if (
|
||||
reads.some(
|
||||
@@ -83,11 +90,24 @@ export const functionCallCoverageLimitations = (
|
||||
coverage: RelationshipCoverage | null,
|
||||
dossier: Pick<FunctionDossier, "unresolved_calls">,
|
||||
): string[] => {
|
||||
return uniqueSorted([
|
||||
...relationshipCoverageLimitations(coverage),
|
||||
...(coverage?.status === "complete" ? [] : (coverage?.diagnostics ?? [])),
|
||||
const limitations =
|
||||
coverage === null
|
||||
? ["Provider did not report relationship coverage."]
|
||||
: coverage.status === "unknown"
|
||||
? [
|
||||
coverage.reason,
|
||||
...("diagnostics" in coverage ? coverage.diagnostics : []),
|
||||
]
|
||||
: coverage.status === "partial"
|
||||
? [
|
||||
...coverage.limitations,
|
||||
...("diagnostics" in coverage ? coverage.diagnostics : []),
|
||||
]
|
||||
: [];
|
||||
limitations.push(
|
||||
...(dossier.unresolved_calls ?? []).map(
|
||||
({ address, reason }) => `Unresolved call at ${address}: ${reason}`,
|
||||
),
|
||||
]);
|
||||
);
|
||||
return uniqueSorted(limitations);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { parseRecordedTransportUrl } from "./recordedTransportUrl.js";
|
||||
|
||||
it("recognizes an empty userinfo marker at the authority start", () => {
|
||||
expect(parseRecordedTransportUrl("https://@host")).toMatchObject({
|
||||
origin: "https://host",
|
||||
path: "/",
|
||||
credentials: { start: 8, end: 9 },
|
||||
});
|
||||
});
|
||||
@@ -33,19 +33,3 @@ export const relationshipCoverageSchema = z.discriminatedUnion("status", [
|
||||
]);
|
||||
|
||||
export type RelationshipCoverage = z.infer<typeof relationshipCoverageSchema>;
|
||||
|
||||
/** Reasons the observed relationship inventory cannot establish completeness. */
|
||||
export const relationshipCoverageLimitations = (
|
||||
coverage: RelationshipCoverage | null | undefined,
|
||||
): readonly string[] => {
|
||||
switch (coverage?.status) {
|
||||
case undefined:
|
||||
return ["Provider did not report relationship coverage."];
|
||||
case "complete":
|
||||
return [];
|
||||
case "partial":
|
||||
return coverage.limitations;
|
||||
case "unknown":
|
||||
return [coverage.reason];
|
||||
}
|
||||
};
|
||||
|
||||
@@ -16,15 +16,6 @@ import { registerEvidenceTools } from "./registerEvidenceTools.js";
|
||||
import { runManagedProviderExecution } from "../composition/directAnalysis.js";
|
||||
import { isManagedToolName } from "../contracts/managed/managedToolContracts.js";
|
||||
|
||||
/** Operations whose caller may select a retained summary instead of complete delivery. */
|
||||
const summarizesRetainedEvidence = (
|
||||
operation: string,
|
||||
parameters: Readonly<Record<string, unknown>>,
|
||||
): boolean =>
|
||||
(operation === "inspect_managed_artifact" ||
|
||||
operation === "inspect_managed_members") &&
|
||||
parameters.detail === "summary";
|
||||
|
||||
/** Register execution-free managed PE/CLI inspection. */
|
||||
export const registerManagedTools = (
|
||||
server: EvidenceMcpServer,
|
||||
@@ -43,8 +34,11 @@ export const registerManagedTools = (
|
||||
execute: async (operation, parameters, executionOptions) => {
|
||||
if (!isManagedToolName(operation))
|
||||
return analysis.execute(operation, parameters, executionOptions);
|
||||
const { detail: _detail, ...providerParameters } = parameters;
|
||||
if (summarizesRetainedEvidence(operation, parameters)) {
|
||||
const { detail: _detail, ...artifactParameters } = parameters;
|
||||
if (
|
||||
operation === "inspect_managed_artifact" &&
|
||||
parameters.detail === "summary"
|
||||
) {
|
||||
const ready = requireAnalysisViewRetention(operation, {
|
||||
recordEvidence: options.recordEvidence,
|
||||
evidenceById: options.evidenceById,
|
||||
@@ -75,7 +69,9 @@ export const registerManagedTools = (
|
||||
path,
|
||||
operation,
|
||||
executionOptions?.signal,
|
||||
providerParameters,
|
||||
operation === "inspect_managed_artifact"
|
||||
? artifactParameters
|
||||
: parameters,
|
||||
);
|
||||
if (execution.ok && typeof requestedPath === "string")
|
||||
selectedManagedPath = requestedPath;
|
||||
@@ -86,7 +82,8 @@ export const registerManagedTools = (
|
||||
...options,
|
||||
analysisForAdmission,
|
||||
projectEvidence: (operation, parameters, evidence) =>
|
||||
summarizesRetainedEvidence(operation, parameters)
|
||||
operation === "inspect_managed_artifact" &&
|
||||
parameters.detail === "summary"
|
||||
? summarizeRetainedAnalysis(evidence, {
|
||||
recordEvidence: options.recordEvidence,
|
||||
evidenceById: options.evidenceById,
|
||||
|
||||
@@ -49,13 +49,16 @@ it("returns a first-call overview and pages the unchanged source after its file
|
||||
const ajv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
expect(ajv.validateSchema(definition.inputSchema)).toBe(true);
|
||||
expect(ajv.validateSchema(outputSchema)).toBe(true);
|
||||
expect(
|
||||
ajv.validate(
|
||||
catalog.find((x) => x.name === "inspect_managed_native_boundaries")!
|
||||
.inputSchema,
|
||||
{ path, detail: "summary" },
|
||||
),
|
||||
).toBe(false);
|
||||
for (const name of [
|
||||
"inspect_managed_members",
|
||||
"inspect_managed_native_boundaries",
|
||||
])
|
||||
expect(
|
||||
ajv.validate(catalog.find((x) => x.name === name)!.inputSchema, {
|
||||
path,
|
||||
detail: "summary",
|
||||
}),
|
||||
).toBe(false);
|
||||
const reply = await client.callTool({
|
||||
name: "inspect_managed_artifact",
|
||||
arguments: { path, detail: "summary" },
|
||||
@@ -236,15 +239,11 @@ it("refuses selected delivery before file access when the server cannot retain E
|
||||
const [a, b] = InMemoryTransport.createLinkedPair();
|
||||
await server.connect(b);
|
||||
await client.connect(a);
|
||||
for (const name of ["inspect_managed_artifact", "inspect_managed_members"]) {
|
||||
const rejected = await client.callTool({
|
||||
name,
|
||||
arguments: { path: join(directory, "absent.dll"), detail: "summary" },
|
||||
});
|
||||
expect(parseMcpToolError(rejected).error.code).toBe(
|
||||
"capability_unavailable",
|
||||
);
|
||||
}
|
||||
const rejected = await client.callTool({
|
||||
name: "inspect_managed_artifact",
|
||||
arguments: { path: join(directory, "absent.dll"), detail: "summary" },
|
||||
});
|
||||
expect(parseMcpToolError(rejected).error.code).toBe("capability_unavailable");
|
||||
const complete = await client.callTool({
|
||||
name: "inspect_managed_artifact",
|
||||
arguments: { path },
|
||||
|
||||
@@ -1,200 +0,0 @@
|
||||
import { writeFile, unlink } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { Ajv2020 } from "ajv/dist/2020.js";
|
||||
import { expect, it, onTestFinished } from "vitest";
|
||||
import { composeBinarySession } from "../../../src/application/binary/BinarySessionComposition.js";
|
||||
import { AnalysisProviderRegistry } from "../../../src/application/binary/AnalysisProviderRegistry.js";
|
||||
import { inspectAnalysisView } from "../../../src/application/analysisView/AnalysisViewService.js";
|
||||
import { ManagedStaticProvider } from "../../../src/dotnet/ManagedStaticProvider.js";
|
||||
import { buildManagedPeFixture } from "../../../src/dotnet/ManagedPe.fixture.js";
|
||||
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import {
|
||||
analysisViewResultSchema,
|
||||
managedMemberSummaryViewResultSchema,
|
||||
} from "../../../src/domain/analysisView/analysisView.js";
|
||||
import { managedMemberInspectionSchema } from "../../../src/domain/managed/managedArtifact.js";
|
||||
import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
const COLLECTIONS = [
|
||||
"types",
|
||||
"exported_types",
|
||||
"fields",
|
||||
"methods",
|
||||
"member_refs",
|
||||
"call_edges",
|
||||
"field_accesses",
|
||||
"user_strings",
|
||||
] as const;
|
||||
|
||||
const createHarness = async () => {
|
||||
const session = composeBinarySession(new AnalysisProviderRegistry([]), [
|
||||
new ManagedStaticProvider(),
|
||||
]);
|
||||
const harness = await createApplicationMcpHarness({ session });
|
||||
onTestFinished(harness.close);
|
||||
return harness;
|
||||
};
|
||||
|
||||
it("summarizes selected members and pages every original collection after the file disappears", async () => {
|
||||
const directory = await createTestTempDirectory("rea-member-summary-");
|
||||
const path = join(directory, "fixture.dll");
|
||||
await writeFile(path, buildManagedPeFixture({ methodRowCount: 2 }));
|
||||
const { client, session } = await createHarness();
|
||||
const definition = (await client.listTools()).tools.find(
|
||||
(x) => x.name === "inspect_managed_members",
|
||||
);
|
||||
if (definition?.outputSchema === undefined)
|
||||
throw new Error("Managed member schemas are absent");
|
||||
const outputSchema: Record<string, unknown> = definition.outputSchema;
|
||||
const ajv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
expect(ajv.validateSchema(definition.inputSchema)).toBe(true);
|
||||
expect(ajv.validateSchema(outputSchema)).toBe(true);
|
||||
const type_names = ["Fixture.Program", "Missing.Type"];
|
||||
expect(
|
||||
ajv.validate(definition.inputSchema, {
|
||||
path,
|
||||
type_names,
|
||||
detail: "summary",
|
||||
}),
|
||||
).toBe(true);
|
||||
const reply = await client.callTool({
|
||||
name: "inspect_managed_members",
|
||||
arguments: { path, type_names, detail: "summary" },
|
||||
});
|
||||
expect(reply.isError).not.toBe(true);
|
||||
expect(
|
||||
ajv.validate(outputSchema, reply.structuredContent),
|
||||
JSON.stringify(ajv.errors),
|
||||
).toBe(true);
|
||||
const receipt = parseEvidence(reply.structuredContent);
|
||||
const view = managedMemberSummaryViewResultSchema.parse(
|
||||
receipt.normalized_result,
|
||||
);
|
||||
const parent = session.evidenceById(view.parent_evidence_id);
|
||||
if (parent === undefined)
|
||||
throw new Error("Complete member source was not retained");
|
||||
expect(parent.operation).toBe("inspect_managed_members");
|
||||
expect(receipt.evidence_links).toEqual([parent.evidence_id]);
|
||||
const report = managedMemberInspectionSchema.parse(parent.normalized_result);
|
||||
expect(view.summary.selection).toEqual({
|
||||
type_names,
|
||||
unmatched_type_names: ["Missing.Type"],
|
||||
});
|
||||
expect(view.summary.coverage).toEqual(report.coverage);
|
||||
expect(view.summary.identity_scope).toEqual(report.identity_scope);
|
||||
expect(view.summary).not.toHaveProperty("methods");
|
||||
expect(report.methods.length).toBeGreaterThan(0);
|
||||
expect(report.member_refs.length).toBeGreaterThan(0);
|
||||
for (const detail of [undefined, "complete"]) {
|
||||
const complete = parseEvidence(
|
||||
(
|
||||
await client.callTool({
|
||||
name: "inspect_managed_members",
|
||||
arguments: { path, type_names, ...(detail ? { detail } : {}) },
|
||||
})
|
||||
).structuredContent,
|
||||
);
|
||||
expect(complete.operation).toBe("inspect_managed_members");
|
||||
expect(complete.normalized_result).toEqual(parent.normalized_result);
|
||||
}
|
||||
const original = JSON.stringify(parent);
|
||||
await unlink(path);
|
||||
for (const collection of COLLECTIONS) {
|
||||
const rows =
|
||||
collection === "user_strings"
|
||||
? report.user_strings?.literals
|
||||
: report[collection];
|
||||
if (rows == null) throw new Error(`${collection} was not examined`);
|
||||
expect(view.summary.reported_counts[collection]).toBe(rows.length);
|
||||
const paged: unknown[] = [];
|
||||
for (let offset: number | null = 0; offset !== null;) {
|
||||
const result = await client.callTool({
|
||||
name: "inspect_analysis_view",
|
||||
arguments: {
|
||||
source: {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: parent.evidence_id,
|
||||
},
|
||||
view: { kind: "managed-members", collection, offset, limit: 1 },
|
||||
},
|
||||
});
|
||||
expect(result.isError).not.toBe(true);
|
||||
const page = analysisViewResultSchema.parse(
|
||||
parseEvidence(result.structuredContent).normalized_result,
|
||||
);
|
||||
if (page.kind !== "managed-members")
|
||||
throw new Error("Unexpected page kind");
|
||||
expect(page.page.collection).toBe(collection);
|
||||
expect(page.selection).toEqual(report.selection);
|
||||
expect(page.coverage.total).toBe(rows.length);
|
||||
paged.push(...page.page.rows);
|
||||
offset = page.coverage.next_offset;
|
||||
}
|
||||
// Method rows carry body facts that a looser row schema would strip.
|
||||
expect(paged).toEqual(rows);
|
||||
}
|
||||
expect(JSON.stringify(session.evidenceById(parent.evidence_id))).toBe(
|
||||
original,
|
||||
);
|
||||
const portable = inspectAnalysisView({
|
||||
source: { kind: "inline", evidence: parent },
|
||||
view: { kind: "summary" },
|
||||
});
|
||||
if (!portable.ok) throw portable.error;
|
||||
expect(portable.value.normalized_result).toEqual(view);
|
||||
}, 30000);
|
||||
|
||||
it("keeps collections that historical member Evidence never examined unknown", async () => {
|
||||
const directory = await createTestTempDirectory("rea-member-historical-");
|
||||
const path = join(directory, "fixture.dll");
|
||||
await writeFile(path, buildManagedPeFixture());
|
||||
const { client } = await createHarness();
|
||||
const parent = parseEvidence(
|
||||
(
|
||||
await client.callTool({
|
||||
name: "inspect_managed_members",
|
||||
arguments: { path },
|
||||
})
|
||||
).structuredContent,
|
||||
);
|
||||
const {
|
||||
exported_types: _exported,
|
||||
user_strings: _strings,
|
||||
selection: _selection,
|
||||
...recorded
|
||||
} = managedMemberInspectionSchema.parse(parent.normalized_result);
|
||||
const historical = createEvidence(undefined, parent.provider, {
|
||||
operation: parent.operation,
|
||||
parameters: parent.parameters,
|
||||
result: recorded,
|
||||
limitations: parent.limitations,
|
||||
authority: parent.authority,
|
||||
});
|
||||
const summary = inspectAnalysisView({
|
||||
source: { kind: "inline", evidence: historical },
|
||||
view: { kind: "summary" },
|
||||
});
|
||||
if (!summary.ok) throw summary.error;
|
||||
const view = managedMemberSummaryViewResultSchema.parse(
|
||||
summary.value.normalized_result,
|
||||
);
|
||||
expect(view.summary.reported_counts.exported_types).toBeNull();
|
||||
expect(view.summary.reported_counts.user_strings).toBeNull();
|
||||
expect(view.summary.user_strings).toBeNull();
|
||||
expect(view.summary.selection).toBeNull();
|
||||
expect(view.unknowns.join(" ")).toContain("no recorded subject");
|
||||
for (const collection of ["exported_types", "user_strings"] as const)
|
||||
expect(
|
||||
inspectAnalysisView({
|
||||
source: { kind: "inline", evidence: historical },
|
||||
view: { kind: "managed-members", collection, offset: 0, limit: 1 },
|
||||
}),
|
||||
).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } });
|
||||
expect(
|
||||
inspectAnalysisView({
|
||||
source: { kind: "inline", evidence: historical },
|
||||
view: { kind: "managed", collection: "attributes", offset: 0, limit: 1 },
|
||||
}),
|
||||
).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } });
|
||||
}, 30000);
|
||||
@@ -34,6 +34,10 @@ it
|
||||
const procRoot = join(directory, "proc");
|
||||
await mkdir(procRoot);
|
||||
const alternateMode = mode === "alternate launcher";
|
||||
let orphanPid: number | undefined;
|
||||
const isolatedProcessList = async () => ({
|
||||
stdout: alternateMode || orphanPid === undefined ? "" : "Xvfb",
|
||||
});
|
||||
if (alternateMode) {
|
||||
await copyFile("/usr/bin/sleep", launcher);
|
||||
await chmod(launcher, 0o700);
|
||||
@@ -47,6 +51,7 @@ it
|
||||
await assertCleanLinuxHopperRuntime(
|
||||
alternateMode ? [launcher] : [],
|
||||
procRoot,
|
||||
isolatedProcessList,
|
||||
);
|
||||
const command = alternateMode ? alternate : process.execPath;
|
||||
const arguments_ = alternateMode ? ["30"] : ["-e", childCode];
|
||||
@@ -69,7 +74,6 @@ it
|
||||
const parentPid = parent.child.pid;
|
||||
if (parentPid === undefined)
|
||||
throw new Error("Fixture parent did not spawn");
|
||||
let orphanPid: number | undefined;
|
||||
try {
|
||||
const { stdout } = await parent;
|
||||
const { pid } = JSON.parse(stdout) as { pid: number };
|
||||
@@ -115,6 +119,7 @@ it
|
||||
assertCleanLinuxHopperRuntime(
|
||||
alternateMode ? [launcher] : [],
|
||||
procRoot,
|
||||
isolatedProcessList,
|
||||
),
|
||||
).rejects.toThrow("Clean Linux runner retained a Hopper demo process");
|
||||
// Observation must not terminate the fixture, even though it resembles Xvfb.
|
||||
|
||||
Reference in New Issue
Block a user