fix(capture): end special-scheme URL authorities at a backslash

The native mitmproxy adapter removed transport userinfo with
`^(...//)[^/?#]*@`, so the authority ran to the last "@" before "/", "?"
or "#". For `https://example.test\part@tail` it treated
`example.test\part` as userinfo, reported `https://tail` as the
Location, and withdrew the original bytes as a transport credential.
WHATWG URL parsing, used by the shared TypeScript owner
(recordedTransportUrl.ts) and by HAR captures, ends a special-scheme or
scheme-relative authority at a backslash, so that "@" is ordinary path
text.

Locate the authority with the same boundary rule before removing
userinfo. Bytes are handled as Latin-1 text so unrelated bytes stay
identical. Userinfo before the backslash is still removed and redacted.
Add source unit tests for the adapter projection to test:python.

Fixes #2671

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kaoru0822
2026-10-13 11:06:25 +00:00
co-authored by Claude Opus 5.5
parent 9f3f392eb8
commit 553cd6e358
3 changed files with 90 additions and 4 deletions
+30 -3
View File
@@ -41,6 +41,35 @@ class BoundedReader:
return self.handle.read(count)
SPECIAL_SCHEMES = {"ftp", "file", "http", "https", "ws", "wss"}
URL_PREFIX = r"^([\x00-\x20]*(?:([a-z][a-z0-9+.-]*):)?//)"
def without_userinfo(value):
"""Remove an authority's userinfo using the WHATWG authority boundary.
Special schemes (and scheme-relative references) also end the authority
at a backslash, so later path text containing "@" is not userinfo. This
matches src/domain/recordedTransportUrl.ts.
"""
binary = isinstance(value, bytes)
text = value.decode("latin-1") if binary else value
prefix = re.match(URL_PREFIX, text, flags=re.I)
if prefix is None:
return value
scheme = prefix.group(2)
special = scheme is None or scheme.lower() in SPECIAL_SCHEMES
terminators = "/?#\\" if special else "/?#"
end = prefix.end()
while end < len(text) and text[end] not in terminators:
end += 1
at = text.rfind("@", prefix.end(), end)
if at < 0:
return value
safe = text[: prefix.end()] + text[at + 1 :]
return safe.encode("latin-1") if binary else safe
def project_record(state, secrets):
binaries, numbers, redactions = [], [], []
byte_secrets = []
@@ -62,10 +91,8 @@ def project_record(state, secrets):
if not (transport_url or re.fullmatch(r"/(?:backup/)*request/(?:path|authority)", path)):
return value, False
binary = isinstance(value, bytes)
pattern = rb"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@" if binary else r"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@"
replacement = rb"\1" if binary else r"\1"
separator = b"@" if binary else "@"
safe_url = re.sub(pattern, replacement, value, flags=re.I)
safe_url = without_userinfo(value)
if path.endswith("/authority") and separator in safe_url:
safe_url = safe_url.rsplit(separator, 1)[1]
changed = safe_url != value
+59
View File
@@ -0,0 +1,59 @@
"""Source-owned tests for the mitmproxy adapter's record projection."""
import importlib.util
import os
import sys
import types
import unittest
fake_mitmproxy = sys.modules.setdefault("mitmproxy", types.ModuleType("mitmproxy"))
fake_mitmproxy.ctx = types.SimpleNamespace()
fake_mitmproxy.version = types.SimpleNamespace(VERSION="12.2.3")
fake_io = sys.modules.setdefault("mitmproxy.io", types.ModuleType("mitmproxy.io"))
fake_io.tnetstring = sys.modules.setdefault(
"mitmproxy.io.tnetstring", types.ModuleType("mitmproxy.io.tnetstring")
)
spec = importlib.util.spec_from_file_location(
"rea_mitmproxy_capture", os.path.join(os.path.dirname(__file__), "capture.py")
)
capture = importlib.util.module_from_spec(spec)
spec.loader.exec_module(capture)
def location(value):
record = capture.project_record({"response": {"headers": [[b"Location", value]]}}, [])
sidecar = next(
field for field in record["binary_fields"] if field["pointer"] == "/response/headers/0/1"
)
return record["reported"]["response"]["headers"][0][1], sidecar, record["redactions"]
class TransportUserinfoTests(unittest.TestCase):
def test_backslash_ends_a_special_scheme_authority(self):
text, sidecar, redactions = location(b"https://example.test\\part@tail")
self.assertEqual(text, "https://example.test\\part@tail")
self.assertEqual(sidecar["state"], "retained")
self.assertIsNotNone(sidecar["content_base64"])
self.assertEqual(redactions, [])
def test_userinfo_before_a_backslash_is_still_removed(self):
text, sidecar, redactions = location(b"https://user:pw@example.test\\path")
self.assertEqual(text, "https://example.test\\path")
self.assertEqual(sidecar["state"], "redacted")
self.assertEqual(
redactions,
[{"pointer": "/response/headers/0/1", "reason": "transport-credential"}],
)
def test_ordinary_userinfo_is_removed(self):
text, _sidecar, _redactions = location(b"https://user:pw@example.test/a@b")
self.assertEqual(text, "https://example.test/a@b")
def test_non_special_schemes_keep_backslashes_in_the_authority(self):
self.assertEqual(capture.without_userinfo("foo://a\\b@host/x"), "foo://host/x")
self.assertEqual(capture.without_userinfo(b"//a\\b@host"), b"//a\\b@host")
if __name__ == "__main__":
unittest.main()
+1 -1
View File
@@ -130,7 +130,7 @@
"test": "npm run test:with-build",
"test:with-build": "npm run test:prepare && vitest run",
"test:run": "vitest run",
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v",
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py bridge/mitmproxy/test_capture.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v",
"test:local": "node scripts/run-development-tests.mjs local",
"test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation",
"pretest:boundary": "npm run test:prepare",