From 058a2af21d4e5a760eed3a42f3caa39c4caab05a Mon Sep 17 00:00:00 2001 From: EminY <57416756+emincan2000@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:22:13 +0000 Subject: [PATCH] Trim the same whitespace as JavaScript in parse() str.strip() and \s also remove U+001C to U+001F and U+0085, which String.prototype.trim() and the JS \s keep, and they miss U+FEFF, which JS removes. So parse("/u\x85") lost the trailing character and markdown-it-py rendered such links differently from markdown-it. --- src/mdurl/_parse.py | 14 ++++++++++++-- tests/test_parse.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/mdurl/_parse.py b/src/mdurl/_parse.py index 8d64748..2cf6fc9 100644 --- a/src/mdurl/_parse.py +++ b/src/mdurl/_parse.py @@ -53,8 +53,18 @@ PROTOCOL_PATTERN = re.compile(r"^([a-z0-9.+-]+:)", flags=re.IGNORECASE) PORT_PATTERN = re.compile(r":[0-9]*$") +# Whitespace as JavaScript's trim() and \s see it. Python's str.strip() and \s +# also take U+001C to U+001F and U+0085, and miss U+FEFF. +JS_WHITESPACE = ( + "\t\n\v\f\r \xa0\u1680" + + "".join(map(chr, range(0x2000, 0x200B))) + + "\u2028\u2029\u202f\u205f\u3000\ufeff" +) + # Special case for a simple path URL -SIMPLE_PATH_PATTERN = re.compile(r"^(//?(?!/)[^?\s]*)(\?[^\s]*)?$") +SIMPLE_PATH_PATTERN = re.compile( + rf"^(//?(?!/)[^?{JS_WHITESPACE}]*)(\?[^{JS_WHITESPACE}]*)?$" +) # RFC 2396: characters reserved for delimiting URLs. # We actually just auto-escape these. @@ -120,7 +130,7 @@ def parse(self, url: str, slashes_denote_host: bool) -> MutableURL: # trim before proceeding. # This is to support parse stuff like " http://foo.com \n" - rest = rest.strip() + rest = rest.strip(JS_WHITESPACE) if not slashes_denote_host and len(url.split("#")) == 1: # Try fast path regexp diff --git a/tests/test_parse.py b/tests/test_parse.py index aa4ae44..54f527a 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -24,3 +24,17 @@ def is_url_and_dict_equal(url, url_dict): def test_parse(url, expected_dict): parsed = parse(url) assert is_url_and_dict_equal(parsed, expected_dict) + + +@pytest.mark.parametrize( + "url,pathname", + [ + # As in JavaScript trim(): U+001C-U+001F and U+0085 stay, U+FEFF goes + ("/u\x85", "/u\x85"), + ("\x1c/u", "\x1c/u"), + ("http://a.b/c\x1f", "/c\x1f"), + ("/u", "/u"), + ], +) +def test_parse_trims_javascript_whitespace(url, pathname): + assert parse(url).pathname == pathname