fix(capture): end special-scheme URL authorities at a backslash
The native mitmproxy adapter removed transport userinfo with `^(...//)[^/?#]*@`, so the authority ran to the last "@" before "/", "?" or "#". For `https://example.test\part@tail` it treated `example.test\part` as userinfo, reported `https://tail` as the Location, and withdrew the original bytes as a transport credential. WHATWG URL parsing, used by the shared TypeScript owner (recordedTransportUrl.ts) and by HAR captures, ends a special-scheme or scheme-relative authority at a backslash, so that "@" is ordinary path text. Locate the authority with the same boundary rule before removing userinfo. Bytes are handled as Latin-1 text so unrelated bytes stay identical. Userinfo before the backslash is still removed and redacted. Add source unit tests for the adapter projection to test:python. Fixes #2671 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9f3f392eb8
commit
553cd6e358
@@ -41,6 +41,35 @@ class BoundedReader:
|
||||
return self.handle.read(count)
|
||||
|
||||
|
||||
SPECIAL_SCHEMES = {"ftp", "file", "http", "https", "ws", "wss"}
|
||||
URL_PREFIX = r"^([\x00-\x20]*(?:([a-z][a-z0-9+.-]*):)?//)"
|
||||
|
||||
|
||||
def without_userinfo(value):
|
||||
"""Remove an authority's userinfo using the WHATWG authority boundary.
|
||||
|
||||
Special schemes (and scheme-relative references) also end the authority
|
||||
at a backslash, so later path text containing "@" is not userinfo. This
|
||||
matches src/domain/recordedTransportUrl.ts.
|
||||
"""
|
||||
binary = isinstance(value, bytes)
|
||||
text = value.decode("latin-1") if binary else value
|
||||
prefix = re.match(URL_PREFIX, text, flags=re.I)
|
||||
if prefix is None:
|
||||
return value
|
||||
scheme = prefix.group(2)
|
||||
special = scheme is None or scheme.lower() in SPECIAL_SCHEMES
|
||||
terminators = "/?#\\" if special else "/?#"
|
||||
end = prefix.end()
|
||||
while end < len(text) and text[end] not in terminators:
|
||||
end += 1
|
||||
at = text.rfind("@", prefix.end(), end)
|
||||
if at < 0:
|
||||
return value
|
||||
safe = text[: prefix.end()] + text[at + 1 :]
|
||||
return safe.encode("latin-1") if binary else safe
|
||||
|
||||
|
||||
def project_record(state, secrets):
|
||||
binaries, numbers, redactions = [], [], []
|
||||
byte_secrets = []
|
||||
@@ -62,10 +91,8 @@ def project_record(state, secrets):
|
||||
if not (transport_url or re.fullmatch(r"/(?:backup/)*request/(?:path|authority)", path)):
|
||||
return value, False
|
||||
binary = isinstance(value, bytes)
|
||||
pattern = rb"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@" if binary else r"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@"
|
||||
replacement = rb"\1" if binary else r"\1"
|
||||
separator = b"@" if binary else "@"
|
||||
safe_url = re.sub(pattern, replacement, value, flags=re.I)
|
||||
safe_url = without_userinfo(value)
|
||||
if path.endswith("/authority") and separator in safe_url:
|
||||
safe_url = safe_url.rsplit(separator, 1)[1]
|
||||
changed = safe_url != value
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Source-owned tests for the mitmproxy adapter's record projection."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
|
||||
|
||||
fake_mitmproxy = sys.modules.setdefault("mitmproxy", types.ModuleType("mitmproxy"))
|
||||
fake_mitmproxy.ctx = types.SimpleNamespace()
|
||||
fake_mitmproxy.version = types.SimpleNamespace(VERSION="12.2.3")
|
||||
fake_io = sys.modules.setdefault("mitmproxy.io", types.ModuleType("mitmproxy.io"))
|
||||
fake_io.tnetstring = sys.modules.setdefault(
|
||||
"mitmproxy.io.tnetstring", types.ModuleType("mitmproxy.io.tnetstring")
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"rea_mitmproxy_capture", os.path.join(os.path.dirname(__file__), "capture.py")
|
||||
)
|
||||
capture = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(capture)
|
||||
|
||||
|
||||
def location(value):
|
||||
record = capture.project_record({"response": {"headers": [[b"Location", value]]}}, [])
|
||||
sidecar = next(
|
||||
field for field in record["binary_fields"] if field["pointer"] == "/response/headers/0/1"
|
||||
)
|
||||
return record["reported"]["response"]["headers"][0][1], sidecar, record["redactions"]
|
||||
|
||||
|
||||
class TransportUserinfoTests(unittest.TestCase):
|
||||
def test_backslash_ends_a_special_scheme_authority(self):
|
||||
text, sidecar, redactions = location(b"https://example.test\\part@tail")
|
||||
self.assertEqual(text, "https://example.test\\part@tail")
|
||||
self.assertEqual(sidecar["state"], "retained")
|
||||
self.assertIsNotNone(sidecar["content_base64"])
|
||||
self.assertEqual(redactions, [])
|
||||
|
||||
def test_userinfo_before_a_backslash_is_still_removed(self):
|
||||
text, sidecar, redactions = location(b"https://user:pw@example.test\\path")
|
||||
self.assertEqual(text, "https://example.test\\path")
|
||||
self.assertEqual(sidecar["state"], "redacted")
|
||||
self.assertEqual(
|
||||
redactions,
|
||||
[{"pointer": "/response/headers/0/1", "reason": "transport-credential"}],
|
||||
)
|
||||
|
||||
def test_ordinary_userinfo_is_removed(self):
|
||||
text, _sidecar, _redactions = location(b"https://user:pw@example.test/a@b")
|
||||
self.assertEqual(text, "https://example.test/a@b")
|
||||
|
||||
def test_non_special_schemes_keep_backslashes_in_the_authority(self):
|
||||
self.assertEqual(capture.without_userinfo("foo://a\\b@host/x"), "foo://host/x")
|
||||
self.assertEqual(capture.without_userinfo(b"//a\\b@host"), b"//a\\b@host")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+1
-1
@@ -130,7 +130,7 @@
|
||||
"test": "npm run test:with-build",
|
||||
"test:with-build": "npm run test:prepare && vitest run",
|
||||
"test:run": "vitest run",
|
||||
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v",
|
||||
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py bridge/mitmproxy/test_capture.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v",
|
||||
"test:local": "node scripts/run-development-tests.mjs local",
|
||||
"test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation",
|
||||
"pretest:boundary": "npm run test:prepare",
|
||||
|
||||
Reference in New Issue
Block a user