From 3ee2faa596871dab1f3fbd0b5aee487cc13ea649 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 09:24:47 +0100
Subject: [PATCH 01/22] Begin adapting for occrp use, python 2 compat.

---
 .gitignore                                        |  1 +
 .travis.yml                                       |  1 +
 README.md                                         | 15 +++++----------
 setup.py                                          |  8 +++++---
 {normalizer => urlcanon}/__init__.py              |  0
 {normalizer => urlcanon}/normalizer.py            |  7 ++++++-
 {normalizer => urlcanon}/tests/__init__.py        |  0
 {normalizer => urlcanon}/tests/test_normalizer.py |  0
 {normalizer => urlcanon}/utils.py                 |  0
 9 files changed, 18 insertions(+), 14 deletions(-)
 rename {normalizer => urlcanon}/__init__.py (100%)
 rename {normalizer => urlcanon}/normalizer.py (99%)
 rename {normalizer => urlcanon}/tests/__init__.py (100%)
 rename {normalizer => urlcanon}/tests/test_normalizer.py (100%)
 rename {normalizer => urlcanon}/utils.py (100%)

diff --git a/.gitignore b/.gitignore
index e814510..2d865d2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,6 +23,7 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
+.vscode/
 MANIFEST
 
 # PyInstaller
diff --git a/.travis.yml b/.travis.yml
index 08a6216..e468c82 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,6 @@
 language: python
 python:
+    - "2.7"
     - "3.3"
     - "3.6"
 script:
diff --git a/README.md b/README.md
index 844279b..ce0e72a 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,4 @@
-URL-NORMALIZER
-==============
+# urlcanon
 
 [![Build Status](https://travis-ci.org/sunu/url-normalizer.svg?branch=master)](https://travis-ci.org/sunu/url-normalizer)
 
@@ -21,11 +20,7 @@ Normalizes URL by doing the following:
 
 Works with `http` and `https` urls only for now.
 
-# Python version
-
-For now, Python 3 only.
-
-# Installation
+## Installation
 
 Install using `pip`
 
@@ -34,7 +29,7 @@ $ pip install git+https://github.com/sunu/url-normalizer
 ```
 or clone and install using `python setup.py install`
 
-# Usage
+## Usage
 
 Pass a url to the `normalize_url` function as a `str` type to normalize it. 
 
@@ -109,10 +104,10 @@ In [18]: repr(normalize_url(1234))
 Out[18]: 'None'
 ```
 
-# Tests
+## Tests
 
 Run tests by using `python setup.py test`
 
-# License
+## License
 
 MIT
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 366e0c4..fffd1bf 100644
--- a/setup.py
+++ b/setup.py
@@ -2,22 +2,24 @@
 from setuptools import setup
 
 setup(
-    name="url-normalizer",
+    name="urlcanon",
     version="0.0.1",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",
     long_description="",
     license="MIT",
-    url="https://github.com/sunu/url-normalizer",
-    packages=['normalizer'],
+    url="https://github.com/alephdata/urlcanon",
+    packages=['urlcanon'],
     classifiers=[
         "Development Status :: 3 - Alpha",
         "Intended Audience :: Developers",
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
+        "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 3"
     ],
+    install_requires=['six'],
     setup_requires=['pytest-runner'],
     tests_require=['pytest'],
 )
diff --git a/normalizer/__init__.py b/urlcanon/__init__.py
similarity index 100%
rename from normalizer/__init__.py
rename to urlcanon/__init__.py
diff --git a/normalizer/normalizer.py b/urlcanon/normalizer.py
similarity index 99%
rename from normalizer/normalizer.py
rename to urlcanon/normalizer.py
index 132c28f..8351928 100644
--- a/normalizer/normalizer.py
+++ b/urlcanon/normalizer.py
@@ -1,6 +1,6 @@
 """This module contains functions to help normalize URLs"""
-from os.path import normpath
 import string
+from os.path import normpath
 from urllib.parse import urlunsplit, unquote, quote, urlencode, urlsplit
 
 from .utils import _parse_qsl, _is_valid_url
@@ -23,6 +23,7 @@
 
 SCHEMES = ("http", "https")
 
+
 def normalize_url(url, extra_query_args=None, drop_fragments=True):
     """Normalize a url to its canonical form.
 
@@ -73,8 +74,10 @@ def normalize_url(url, extra_query_args=None, drop_fragments=True):
     url = urlunsplit((scheme, netloc, path, query, fragment))
     return url
 
+
 __all__ = ["normalize_url"]
 
+
 def _normalize_path(path):
     # If there are any `/` or `?` or `#` in the path encoded as `%2f` or `%3f`
     # or `%23` respectively, we don't want them unquoted. So escape them
@@ -96,6 +99,7 @@ def _normalize_path(path):
         path = "/" + path.lstrip("/")
     return path
 
+
 def _normalize_netloc(scheme, netloc, username, password, port):
     # Leave auth info out before fiddling with netloc
     auth = None
@@ -116,6 +120,7 @@ def _normalize_netloc(scheme, netloc, username, password, port):
         netloc = auth + "@" + netloc
     return netloc
 
+
 def _normalize_query(query, extra_query_args):
     # Percent-encode and sort query arguments.
     queries_list = _parse_qsl(query)
diff --git a/normalizer/tests/__init__.py b/urlcanon/tests/__init__.py
similarity index 100%
rename from normalizer/tests/__init__.py
rename to urlcanon/tests/__init__.py
diff --git a/normalizer/tests/test_normalizer.py b/urlcanon/tests/test_normalizer.py
similarity index 100%
rename from normalizer/tests/test_normalizer.py
rename to urlcanon/tests/test_normalizer.py
diff --git a/normalizer/utils.py b/urlcanon/utils.py
similarity index 100%
rename from normalizer/utils.py
rename to urlcanon/utils.py

From af204c1285454c38602a1c029adce67c5ea14ddb Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 09:52:11 +0100
Subject: [PATCH 02/22] half-way point in porting to python 2

---
 urlcanon/__init__.py              |  5 ++-
 urlcanon/constants.py             | 19 ++++++++
 urlcanon/normalizer.py            | 44 ++++++-------------
 urlcanon/tests/test_normalizer.py | 37 +++++++++++++++-
 urlcanon/utils.py                 | 72 +++++++++++++++----------------
 urlcanon/validator.py             | 40 +++++++++++++++++
 6 files changed, 145 insertions(+), 72 deletions(-)
 create mode 100644 urlcanon/constants.py
 create mode 100644 urlcanon/validator.py

diff --git a/urlcanon/__init__.py b/urlcanon/__init__.py
index 0b6a190..80264ac 100644
--- a/urlcanon/__init__.py
+++ b/urlcanon/__init__.py
@@ -1 +1,4 @@
-from .normalizer import normalize_url
+from urlcanon.normalizer import normalize_url
+from urlcanon.validator import is_valid_url
+
+__all__ = [normalize_url, is_valid_url]
diff --git a/urlcanon/constants.py b/urlcanon/constants.py
new file mode 100644
index 0000000..2da5817
--- /dev/null
+++ b/urlcanon/constants.py
@@ -0,0 +1,19 @@
+import string
+
+# Reserved delimeters from https://tools.ietf.org/html/rfc3986#section-2.2
+GEN_DELIMS = ":/?#[]@"
+SUB_DELIMS = "!$&'()*+,;="
+RESERVED_CHARS = GEN_DELIMS + SUB_DELIMS
+
+# Unreserved characters from https://tools.ietf.org/html/rfc3986#section-2.3
+UNRESERVED_CHARS = string.ascii_letters + string.digits + "-._~"
+
+SAFE_CHARS = RESERVED_CHARS + UNRESERVED_CHARS + '%'
+
+# TODO: Add more schemes may be
+DEFAULT_PORTS = {
+    "http": 80,
+    "https": 443
+}
+
+SCHEMES = ("http", "https")
diff --git a/urlcanon/normalizer.py b/urlcanon/normalizer.py
index 8351928..81ab78d 100644
--- a/urlcanon/normalizer.py
+++ b/urlcanon/normalizer.py
@@ -1,27 +1,13 @@
 """This module contains functions to help normalize URLs"""
-import string
+from __future__ import unicode_literals
+import six
 from os.path import normpath
-from urllib.parse import urlunsplit, unquote, quote, urlencode, urlsplit
+from six.moves.urllib.parse import urlunsplit, unquote, quote
+from six.moves.urllib.parse import urlsplit
 
-from .utils import _parse_qsl, _is_valid_url
-
-# Reserved delimeters from https://tools.ietf.org/html/rfc3986#section-2.2
-GEN_DELIMS = ":/?#[]@"
-SUB_DELIMS = "!$&'()*+,;="
-RESERVED_CHARS = GEN_DELIMS + SUB_DELIMS
-
-# Unreserved characters from https://tools.ietf.org/html/rfc3986#section-2.3
-UNRESERVED_CHARS = string.ascii_letters + string.digits + "-._~"
-
-SAFE_CHARS = RESERVED_CHARS + UNRESERVED_CHARS + '%'
-
-# TODO: Add more schemes may be
-DEFAULT_PORTS = {
-    "http": 80,
-    "https": 443
-}
-
-SCHEMES = ("http", "https")
+from urlcanon.utils import _parse_qsl, _urlencode
+from urlcanon.validator import is_valid_url
+from urlcanon.constants import SCHEMES, DEFAULT_PORTS, SAFE_CHARS
 
 
 def normalize_url(url, extra_query_args=None, drop_fragments=True):
@@ -44,7 +30,7 @@ def normalize_url(url, extra_query_args=None, drop_fragments=True):
     None
         If the passed string doesn't look like a URL, return None
     """
-    if not isinstance(url, str):
+    if not isinstance(url, six.string_types):
         return None
     url = url.strip()
     if not url.lower().startswith(SCHEMES):
@@ -52,7 +38,7 @@ def normalize_url(url, extra_query_args=None, drop_fragments=True):
             url = "http:" + url
         else:
             url = "http://" + url
-    if not _is_valid_url(url):
+    if not is_valid_url(url):
         # Doesn't look like a valid URL
         return None
     parts = urlsplit(url)
@@ -75,9 +61,6 @@ def normalize_url(url, extra_query_args=None, drop_fragments=True):
     return url
 
 
-__all__ = ["normalize_url"]
-
-
 def _normalize_path(path):
     # If there are any `/` or `?` or `#` in the path encoded as `%2f` or `%3f`
     # or `%23` respectively, we don't want them unquoted. So escape them
@@ -126,11 +109,10 @@ def _normalize_query(query, extra_query_args):
     queries_list = _parse_qsl(query)
     # Add the additional query args if any
     if extra_query_args:
-        extra_query_args = [
-            (name.encode("utf-8"), val.encode("utf-8"))
-            for (name, val) in extra_query_args
-        ]
+        for (name, val) in extra_query_args:
+            queries_list.append((name.encode("utf-8"),
+                                 val.encode("utf-8")))
         queries_list.extend(extra_query_args)
     queries_list.sort()
-    query = urlencode(queries_list, safe=SAFE_CHARS)
+    query = _urlencode(queries_list)
     return query
diff --git a/urlcanon/tests/test_normalizer.py b/urlcanon/tests/test_normalizer.py
index d3289e1..12fcee9 100644
--- a/urlcanon/tests/test_normalizer.py
+++ b/urlcanon/tests/test_normalizer.py
@@ -1,41 +1,51 @@
+# coding: utf-8
 """Tests for URL normalization"""
-import pytest
+from __future__ import unicode_literals
+from six import text_type
 
 from ..normalizer import normalize_url
 
 # TODO: parametrize test cases
 
+
 def test_normalized_urls():
     """Already normalized URLs should not change"""
     assert normalize_url("http://example.com/") == "http://example.com/"
 
+
 def test_return_type():
     """Should return string"""
-    assert isinstance(normalize_url("http://example.com/"), str)
+    assert isinstance(normalize_url("http://example.com/"), text_type)
+
 
 def test_append_slash():
     """Append a slash to the end of the URL if it's missing one"""
     assert normalize_url("http://example.com") == "http://example.com/"
 
+
 def test_lower_case():
     """Normalized URL scheme and host are lower case"""
     assert normalize_url("HTTP://examPle.cOm/") == "http://example.com/"
     assert normalize_url("http://example.com/A") == "http://example.com/A"
 
+
 def test_strip_trailing_period():
     assert normalize_url("http://example.com.") == "http://example.com/"
     assert normalize_url("http://example.com./") == "http://example.com/"
 
+
 def test_capitalize_escape_sequence():
     """All letters in percent-encoded triplets should be capitalized"""
     assert (normalize_url("http://www.example.com/a%c2%b1b") ==
             "http://www.example.com/a%C2%B1b")
 
+
 def test_path_percent_encoding():
     """All non-safe characters should be percent-encoded"""
     assert (normalize_url("http://example.com/hello world{}") ==
             "http://example.com/hello%20world%7B%7D")
 
+
 def test_unreserved_percentencoding():
     """Unreserved characters should not be percent encoded. If they are, they
     should be decoded back; except in case of `/`, `?` and `#`"""
@@ -48,12 +58,14 @@ def test_unreserved_percentencoding():
     assert (normalize_url('http://example.com/foo%3fbar') ==
             'http://example.com/foo%3Fbar')
 
+
 def test_remove_dot_segments():
     """Convert the URL path to an absolute path by removing `.` and `..`
     segments"""
     assert (normalize_url("http://www.example.com/../a/b/../c/./d.html") ==
             "http://www.example.com/a/c/d.html")
 
+
 def test_remove_default_port():
     """Remove the default port for the scheme if it's present in the URL"""
     assert (normalize_url("http://www.example.com:80/bar.html") ==
@@ -61,11 +73,13 @@ def test_remove_default_port():
     assert (normalize_url("HTTPS://example.com:443/abc/") ==
             "https://example.com/abc")
 
+
 def test_remove_empty_port():
     """Remove empty port from URL"""
     assert (normalize_url("http://www.example.com:/") ==
             "http://www.example.com/")
 
+
 def test_remove_extra_slash():
     """Remove any extra slashes if present in the URl"""
     # TODO: Should we actually do this?
@@ -75,6 +89,7 @@ def test_remove_extra_slash():
     assert(normalize_url("http://example.com///abc") ==
            "http://example.com/abc")
 
+
 def test_query_string():
     """Query strings should be handled properly"""
     assert (normalize_url("http://example.com/?a=1") ==
@@ -86,11 +101,13 @@ def test_query_string():
     assert (normalize_url("http://example.com/a/?b=1") ==
             "http://example.com/a?b=1")
 
+
 def test_dont_percent_encode_safe_chars_query():
     """Don't percent-encode safe characters in querystring"""
     assert (normalize_url("http://example.com/a/?face=(-.-)") ==
             "http://example.com/a?face=(-.-)")
 
+
 def test_query_sorting():
     """Query strings should be sorted"""
     assert (normalize_url('http://example.com/a?b=1&c=2') ==
@@ -98,6 +115,7 @@ def test_query_sorting():
     assert (normalize_url('http://example.com/a?c=2&b=1') ==
             'http://example.com/a?b=1&c=2')
 
+
 def test_query_string_spaces():
     """Spaces should be handled properly in query strings"""
     assert (normalize_url("http://example.com/search?q=a b&a=1") ==
@@ -107,6 +125,7 @@ def test_query_string_spaces():
     assert (normalize_url("http://example.com/search?q=a%20b&a=1") ==
             "http://example.com/search?a=1&q=a+b")
 
+
 def test_drop_trailing_questionmark():
     """Drop the trailing question mark if no query string present"""
     assert normalize_url("http://example.com/?") == "http://example.com/"
@@ -114,28 +133,33 @@ def test_drop_trailing_questionmark():
     assert normalize_url("http://example.com/a?") == "http://example.com/a"
     assert normalize_url("http://example.com/a/?") == "http://example.com/a"
 
+
 def test_percent_encode_querystring():
     """Non-safe characters in query string should be percent-encoded"""
     assert (normalize_url("http://example.com/?a=hello{}") ==
             "http://example.com/?a=hello%7B%7D")
 
+
 def test_normalize_percent_encoding_in_querystring():
     """Percent-encoded querystring should be uppercased"""
     assert (normalize_url("http://example.com/?a=b%c2") ==
             "http://example.com/?a=b%C2")
 
+
 def test_unicode_query_string():
     """Unicode query strings should be converted to bytes using uft-8 encoding
     and then properly percent-encoded"""
     assert (normalize_url("http://example.com/?file=résumé.pdf") ==
             "http://example.com/?file=r%C3%A9sum%C3%A9.pdf")
 
+
 def test_unicode_path():
     """Unicode path should be converted to bytes using utf-8 encoding and then
     percent-encoded"""
     assert (normalize_url("http://example.com/résumé") ==
             "http://example.com/r%C3%A9sum%C3%A9")
 
+
 def test_idna():
     """International Domain Names should be normalized to safe characters"""
     assert (normalize_url("http://ドメイン.テスト") ==
@@ -143,11 +167,13 @@ def test_idna():
     assert (normalize_url("http://Яндекс.рф") ==
             "http://xn--d1acpjx3f.xn--p1ai/")
 
+
 def test_dont_change_username_password():
     """Username and password shouldn't be lowercased"""
     assert (normalize_url("http://Foo:BAR@exaMPLE.COM/") ==
             "http://Foo:BAR@example.com/")
 
+
 def test_normalize_ipv4():
     """Normalize ipv4 URLs"""
     assert normalize_url("http://192.168.0.1/") == "http://192.168.0.1/"
@@ -157,6 +183,7 @@ def test_normalize_ipv4():
     assert (normalize_url("192.168.0.1:8080/a/b/c") ==
             "http://192.168.0.1:8080/a/b/c")
 
+
 def test_normalize_ipv6():
     """Normalize ipv6 URLs"""
     assert normalize_url("[::1]") == "http://[::1]/"
@@ -164,18 +191,21 @@ def test_normalize_ipv6():
     assert normalize_url("[::1]:8080") == "http://[::1]:8080/"
     assert normalize_url("http://[::1]:8080") == "http://[::1]:8080/"
 
+
 def test_strip_leading_trailing_whitespace():
     """Strip leading and trailing whitespace if any"""
     assert normalize_url("   http://example.com  ") == "http://example.com/"
     assert normalize_url("http://example.com/a  ") == "http://example.com/a"
     assert normalize_url("   http://example.com/") == "http://example.com/"
 
+
 def test_non_ideal_inputs():
     """Not the ideal input; but we should handle it anyway"""
     assert normalize_url("example.com") == "http://example.com/"
     assert normalize_url("example.com/abc") == "http://example.com/abc"
     assert normalize_url("//example.com/abc") == "http://example.com/abc"
 
+
 def test_additional_query_args():
     """Add any additional query arguments to the URL"""
     assert (normalize_url("http://example.com?c=d", [("a", "b")]) ==
@@ -185,6 +215,7 @@ def test_additional_query_args():
     assert (normalize_url("http://example.com", [("résumé", "résumé")]) ==
             "http://example.com/?r%C3%A9sum%C3%A9=r%C3%A9sum%C3%A9")
 
+
 def test_non_urls():
     """If a non-URL string is passed, return None"""
     assert normalize_url("") is None
@@ -195,6 +226,7 @@ def test_non_urls():
     assert normalize_url("http//google.com") is None
     assert normalize_url("http://user@pass:example.com") is None
 
+
 def test_drop_fragments():
     """Drop or keep fragments based on the option passed"""
     assert (normalize_url("http://example.com/a?b=1#frag")
@@ -202,6 +234,7 @@ def test_drop_fragments():
     assert (normalize_url("http://example.com/a?b=1#frag", drop_fragments=False)
             == "http://example.com/a?b=1#frag")
 
+
 def test_non_string_input():
     """Non-string input should produce None as result"""
     assert normalize_url(None) is None
diff --git a/urlcanon/utils.py b/urlcanon/utils.py
index ab9bd53..0d05816 100644
--- a/urlcanon/utils.py
+++ b/urlcanon/utils.py
@@ -1,5 +1,35 @@
-import re
-from urllib.parse import _coerce_args, unquote_to_bytes
+from __future__ import unicode_literals
+from six import text_type
+from six.moves.urllib.parse import unquote_to_bytes, urlencode
+
+
+def _noop(obj):
+    return obj
+
+
+def _encode_result(obj):
+    return obj.encode('utf-8', 'strict')
+
+
+def _decode_args(args):
+    return tuple(x.decode('utf-8', 'strict') if x else '' for x in args)
+
+
+def _coerce_args(*args):
+    # Invokes decode if necessary to create str args
+    # and returns the coerced inputs along with
+    # an appropriate result coercion function
+    #   - noop for str inputs
+    #   - encoding function otherwise
+    str_input = isinstance(args[0], text_type)
+    for arg in args[1:]:
+        # We special-case the empty string to support the
+        # "scheme=''" default argument to some functions
+        if arg and isinstance(arg, text_type) != str_input:
+            raise TypeError("Cannot mix str and non-str arguments")
+    if str_input:
+        return args + (_noop,)
+    return _decode_args(args) + (_encode_result,)
 
 
 def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False):
@@ -39,39 +69,5 @@ def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False):
     return r
 
 
-def _is_valid_url(value):
-    """
-    Does the value look like a URL?
-    From https://github.com/django/django/blob/stable/2.0.x/django/core/validators.py
-    """
-    if value.startswith("//"):
-        value = value[2:]
-    ul = '\u00a1-\uffff'  # unicode letters range (must not be a raw string)
-
-    # IP patterns
-    ipv4_re = r'(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}'
-    ipv6_re = r'\[[0-9a-f:\.]+\]'  # (simple regex, validated later)
-
-    # Host patterns
-    hostname_re = r'[a-z' + ul + r'0-9](?:[a-z' + ul + r'0-9-]{0,61}[a-z' + ul + r'0-9])?'
-    # Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1
-    domain_re = r'(?:\.(?!-)[a-z' + ul + r'0-9-]{1,63}(?<!-))*'
-    tld_re = (
-        r'\.'                                # dot
-        r'(?!-)'                             # can't start with a dash
-        r'(?:[a-z' + ul + '-]{2,63}'         # domain label
-        r'|xn--[a-z0-9]{1,59})'              # or punycode label
-        r'(?<!-)'                            # can't end with a dash
-        r'\.?'                               # may have a trailing dot
-    )
-    host_re = '(' + hostname_re + domain_re + tld_re + '|localhost)'
-
-    pattern = re.compile(
-        r'^(?:[a-z0-9\.\-\+]*)://'  # scheme is validated separately
-        r'(?:\S+(?::\S*)?@)?'  # user:pass authentication
-        r'(?:' + ipv4_re + '|' + ipv6_re + '|' + host_re + ')'
-        r'(?::?(?:\d{2,5})?)?'  # port
-        u"(?:/.*)?" # resource path
-        u"(?:\?.*)?" # query string
-        r'\Z', re.IGNORECASE)
-    return bool(pattern.match(value))
+def _urlencode(queries):
+    return urlencode(queries)
\ No newline at end of file
diff --git a/urlcanon/validator.py b/urlcanon/validator.py
new file mode 100644
index 0000000..029f6d5
--- /dev/null
+++ b/urlcanon/validator.py
@@ -0,0 +1,40 @@
+from __future__ import unicode_literals
+import re
+
+
+def is_valid_url(value):
+    """
+    Does the value look like a URL?
+    From https://github.com/django/django/blob/stable/2.0.x/django/core/validators.py
+    """
+    if value.startswith("//"):
+        value = value[2:]
+    ul = '\u00a1-\uffff'  # unicode letters range (must not be a raw string)
+
+    # IP patterns
+    ipv4_re = r'(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}'
+    ipv6_re = r'\[[0-9a-f:\.]+\]'  # (simple regex, validated later)
+
+    # Host patterns
+    hostname_re = r'[a-z' + ul + r'0-9](?:[a-z' + ul + r'0-9-]{0,61}[a-z' + ul + r'0-9])?'
+    # Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1
+    domain_re = r'(?:\.(?!-)[a-z' + ul + r'0-9-]{1,63}(?<!-))*'
+    tld_re = (
+        r'\.'                                # dot
+        r'(?!-)'                             # can't start with a dash
+        r'(?:[a-z' + ul + '-]{2,63}'         # domain label
+        r'|xn--[a-z0-9]{1,59})'              # or punycode label
+        r'(?<!-)'                            # can't end with a dash
+        r'\.?'                               # may have a trailing dot
+    )
+    host_re = '(' + hostname_re + domain_re + tld_re + '|localhost)'
+
+    pattern = re.compile(
+        r'^(?:[a-z0-9\.\-\+]*)://'  # scheme is validated separately
+        r'(?:\S+(?::\S*)?@)?'  # user:pass authentication
+        r'(?:' + ipv4_re + '|' + ipv6_re + '|' + host_re + ')'
+        r'(?::?(?:\d{2,5})?)?'  # port
+        u"(?:/.*)?"  # resource path
+        u"(?:\?.*)?"  # query string
+        r'\Z', re.IGNORECASE)
+    return bool(pattern.match(value))

From d8ff3455b467bba8886210b64571d3a8feaa5960 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 11:07:10 +0100
Subject: [PATCH 03/22] Python 2 working correctly now.

---
 urlcanon/normalizer.py            |  14 +-
 urlcanon/tests/test_normalizer.py | 449 ++++++++++++++----------------
 urlcanon/utils.py                 |  36 ++-
 3 files changed, 253 insertions(+), 246 deletions(-)

diff --git a/urlcanon/normalizer.py b/urlcanon/normalizer.py
index 81ab78d..cc41352 100644
--- a/urlcanon/normalizer.py
+++ b/urlcanon/normalizer.py
@@ -2,12 +2,12 @@
 from __future__ import unicode_literals
 import six
 from os.path import normpath
-from six.moves.urllib.parse import urlunsplit, unquote, quote
+from six.moves.urllib.parse import urlunsplit
 from six.moves.urllib.parse import urlsplit
 
-from urlcanon.utils import _parse_qsl, _urlencode
+from urlcanon.utils import _parse_qsl, _urlencode, _quote, _unquote
 from urlcanon.validator import is_valid_url
-from urlcanon.constants import SCHEMES, DEFAULT_PORTS, SAFE_CHARS
+from urlcanon.constants import SCHEMES, DEFAULT_PORTS
 
 
 def normalize_url(url, extra_query_args=None, drop_fragments=True):
@@ -69,8 +69,8 @@ def _normalize_path(path):
         path = path.replace('%' + reserved, '%25' + reserved.upper())
     # unquote and quote the path so that any non-safe character is
     # percent-encoded and already percent-encoded triplets are upper cased.
-    unquoted_path = unquote(path)
-    path = quote(unquoted_path, SAFE_CHARS) or "/"
+    unquoted_path = _unquote(path)
+    path = _quote(unquoted_path) or "/"
     # Use `os.path.normpath` to normalize paths i.e. remove duplicate `/` and
     # make the path absolute when `..` or `.` segments are present.
     # TODO: Should we remove duplicate slashes?
@@ -110,9 +110,7 @@ def _normalize_query(query, extra_query_args):
     # Add the additional query args if any
     if extra_query_args:
         for (name, val) in extra_query_args:
-            queries_list.append((name.encode("utf-8"),
-                                 val.encode("utf-8")))
-        queries_list.extend(extra_query_args)
+            queries_list.append((name, val))
     queries_list.sort()
     query = _urlencode(queries_list)
     return query
diff --git a/urlcanon/tests/test_normalizer.py b/urlcanon/tests/test_normalizer.py
index 12fcee9..33927e5 100644
--- a/urlcanon/tests/test_normalizer.py
+++ b/urlcanon/tests/test_normalizer.py
@@ -2,241 +2,222 @@
 """Tests for URL normalization"""
 from __future__ import unicode_literals
 from six import text_type
+from unittest import TestCase
 
 from ..normalizer import normalize_url
 
-# TODO: parametrize test cases
 
-
-def test_normalized_urls():
-    """Already normalized URLs should not change"""
-    assert normalize_url("http://example.com/") == "http://example.com/"
-
-
-def test_return_type():
-    """Should return string"""
-    assert isinstance(normalize_url("http://example.com/"), text_type)
-
-
-def test_append_slash():
-    """Append a slash to the end of the URL if it's missing one"""
-    assert normalize_url("http://example.com") == "http://example.com/"
-
-
-def test_lower_case():
-    """Normalized URL scheme and host are lower case"""
-    assert normalize_url("HTTP://examPle.cOm/") == "http://example.com/"
-    assert normalize_url("http://example.com/A") == "http://example.com/A"
-
-
-def test_strip_trailing_period():
-    assert normalize_url("http://example.com.") == "http://example.com/"
-    assert normalize_url("http://example.com./") == "http://example.com/"
-
-
-def test_capitalize_escape_sequence():
-    """All letters in percent-encoded triplets should be capitalized"""
-    assert (normalize_url("http://www.example.com/a%c2%b1b") ==
-            "http://www.example.com/a%C2%B1b")
-
-
-def test_path_percent_encoding():
-    """All non-safe characters should be percent-encoded"""
-    assert (normalize_url("http://example.com/hello world{}") ==
-            "http://example.com/hello%20world%7B%7D")
-
-
-def test_unreserved_percentencoding():
-    """Unreserved characters should not be percent encoded. If they are, they
-    should be decoded back; except in case of `/`, `?` and `#`"""
-    assert (normalize_url("http://www.example.com/%7Eusername/") ==
-            "http://www.example.com/~username")
-    assert (normalize_url('http://example.com/foo%23bar') ==
-            'http://example.com/foo%23bar')
-    assert (normalize_url('http://example.com/foo%2fbar') ==
-            'http://example.com/foo%2Fbar')
-    assert (normalize_url('http://example.com/foo%3fbar') ==
-            'http://example.com/foo%3Fbar')
-
-
-def test_remove_dot_segments():
-    """Convert the URL path to an absolute path by removing `.` and `..`
-    segments"""
-    assert (normalize_url("http://www.example.com/../a/b/../c/./d.html") ==
-            "http://www.example.com/a/c/d.html")
-
-
-def test_remove_default_port():
-    """Remove the default port for the scheme if it's present in the URL"""
-    assert (normalize_url("http://www.example.com:80/bar.html") ==
-            "http://www.example.com/bar.html")
-    assert (normalize_url("HTTPS://example.com:443/abc/") ==
-            "https://example.com/abc")
-
-
-def test_remove_empty_port():
-    """Remove empty port from URL"""
-    assert (normalize_url("http://www.example.com:/") ==
-            "http://www.example.com/")
-
-
-def test_remove_extra_slash():
-    """Remove any extra slashes if present in the URl"""
-    # TODO: Should we actually do this?
-    # TODO: See https://webmasters.stackexchange.com/questions/8354/what-does-the-double-slash-mean-in-urls/8381#8381
-    assert (normalize_url("http://www.example.com/foo//bar.html") ==
-            "http://www.example.com/foo/bar.html")
-    assert(normalize_url("http://example.com///abc") ==
-           "http://example.com/abc")
-
-
-def test_query_string():
-    """Query strings should be handled properly"""
-    assert (normalize_url("http://example.com/?a=1") ==
-            "http://example.com/?a=1")
-    assert (normalize_url("http://example.com?a=1") ==
-            "http://example.com/?a=1")
-    assert (normalize_url("http://example.com/a?b=1") ==
-            "http://example.com/a?b=1")
-    assert (normalize_url("http://example.com/a/?b=1") ==
-            "http://example.com/a?b=1")
-
-
-def test_dont_percent_encode_safe_chars_query():
-    """Don't percent-encode safe characters in querystring"""
-    assert (normalize_url("http://example.com/a/?face=(-.-)") ==
-            "http://example.com/a?face=(-.-)")
-
-
-def test_query_sorting():
-    """Query strings should be sorted"""
-    assert (normalize_url('http://example.com/a?b=1&c=2') ==
-            'http://example.com/a?b=1&c=2')
-    assert (normalize_url('http://example.com/a?c=2&b=1') ==
-            'http://example.com/a?b=1&c=2')
-
-
-def test_query_string_spaces():
-    """Spaces should be handled properly in query strings"""
-    assert (normalize_url("http://example.com/search?q=a b&a=1") ==
-            "http://example.com/search?a=1&q=a+b")
-    assert (normalize_url("http://example.com/search?q=a+b&a=1") ==
-            "http://example.com/search?a=1&q=a+b")
-    assert (normalize_url("http://example.com/search?q=a%20b&a=1") ==
-            "http://example.com/search?a=1&q=a+b")
-
-
-def test_drop_trailing_questionmark():
-    """Drop the trailing question mark if no query string present"""
-    assert normalize_url("http://example.com/?") == "http://example.com/"
-    assert normalize_url("http://example.com?") == "http://example.com/"
-    assert normalize_url("http://example.com/a?") == "http://example.com/a"
-    assert normalize_url("http://example.com/a/?") == "http://example.com/a"
-
-
-def test_percent_encode_querystring():
-    """Non-safe characters in query string should be percent-encoded"""
-    assert (normalize_url("http://example.com/?a=hello{}") ==
-            "http://example.com/?a=hello%7B%7D")
-
-
-def test_normalize_percent_encoding_in_querystring():
-    """Percent-encoded querystring should be uppercased"""
-    assert (normalize_url("http://example.com/?a=b%c2") ==
-            "http://example.com/?a=b%C2")
-
-
-def test_unicode_query_string():
-    """Unicode query strings should be converted to bytes using uft-8 encoding
-    and then properly percent-encoded"""
-    assert (normalize_url("http://example.com/?file=résumé.pdf") ==
-            "http://example.com/?file=r%C3%A9sum%C3%A9.pdf")
-
-
-def test_unicode_path():
-    """Unicode path should be converted to bytes using utf-8 encoding and then
-    percent-encoded"""
-    assert (normalize_url("http://example.com/résumé") ==
-            "http://example.com/r%C3%A9sum%C3%A9")
-
-
-def test_idna():
-    """International Domain Names should be normalized to safe characters"""
-    assert (normalize_url("http://ドメイン.テスト") ==
-            "http://xn--eckwd4c7c.xn--zckzah/")
-    assert (normalize_url("http://Яндекс.рф") ==
-            "http://xn--d1acpjx3f.xn--p1ai/")
-
-
-def test_dont_change_username_password():
-    """Username and password shouldn't be lowercased"""
-    assert (normalize_url("http://Foo:BAR@exaMPLE.COM/") ==
-            "http://Foo:BAR@example.com/")
-
-
-def test_normalize_ipv4():
-    """Normalize ipv4 URLs"""
-    assert normalize_url("http://192.168.0.1/") == "http://192.168.0.1/"
-    assert (normalize_url("http://192.168.0.1:8080/a?b=1") ==
-            "http://192.168.0.1:8080/a?b=1")
-    assert normalize_url("192.168.0.1") == "http://192.168.0.1/"
-    assert (normalize_url("192.168.0.1:8080/a/b/c") ==
-            "http://192.168.0.1:8080/a/b/c")
-
-
-def test_normalize_ipv6():
-    """Normalize ipv6 URLs"""
-    assert normalize_url("[::1]") == "http://[::1]/"
-    assert normalize_url("http://[::1]") == "http://[::1]/"
-    assert normalize_url("[::1]:8080") == "http://[::1]:8080/"
-    assert normalize_url("http://[::1]:8080") == "http://[::1]:8080/"
-
-
-def test_strip_leading_trailing_whitespace():
-    """Strip leading and trailing whitespace if any"""
-    assert normalize_url("   http://example.com  ") == "http://example.com/"
-    assert normalize_url("http://example.com/a  ") == "http://example.com/a"
-    assert normalize_url("   http://example.com/") == "http://example.com/"
-
-
-def test_non_ideal_inputs():
-    """Not the ideal input; but we should handle it anyway"""
-    assert normalize_url("example.com") == "http://example.com/"
-    assert normalize_url("example.com/abc") == "http://example.com/abc"
-    assert normalize_url("//example.com/abc") == "http://example.com/abc"
-
-
-def test_additional_query_args():
-    """Add any additional query arguments to the URL"""
-    assert (normalize_url("http://example.com?c=d", [("a", "b")]) ==
-            "http://example.com/?a=b&c=d")
-    assert (normalize_url("http://example.com", [("a", "b")]) ==
-            "http://example.com/?a=b")
-    assert (normalize_url("http://example.com", [("résumé", "résumé")]) ==
-            "http://example.com/?r%C3%A9sum%C3%A9=r%C3%A9sum%C3%A9")
-
-
-def test_non_urls():
-    """If a non-URL string is passed, return None"""
-    assert normalize_url("") is None
-    assert normalize_url("abc xyz") is None
-    assert normalize_url("asb#abc") is None
-    assert normalize_url("Яндекс.рф") is not None
-    assert normalize_url("google.blog") is not None
-    assert normalize_url("http//google.com") is None
-    assert normalize_url("http://user@pass:example.com") is None
-
-
-def test_drop_fragments():
-    """Drop or keep fragments based on the option passed"""
-    assert (normalize_url("http://example.com/a?b=1#frag")
-            == "http://example.com/a?b=1")
-    assert (normalize_url("http://example.com/a?b=1#frag", drop_fragments=False)
-            == "http://example.com/a?b=1#frag")
-
-
-def test_non_string_input():
-    """Non-string input should produce None as result"""
-    assert normalize_url(None) is None
-    assert normalize_url([]) is None
-    assert normalize_url(123) is None
+class UrlTestCase(TestCase):
+
+    def test_normalized_urls(self):
+        """Already normalized URLs should not change"""
+        self.assertEqual(normalize_url("http://example.com/"),
+                         "http://example.com/")
+
+    def test_return_type(self):
+        """Should return string"""
+        assert isinstance(normalize_url("http://example.com/"), text_type)
+
+    def test_append_slash(self):
+        """Append a slash to the end of the URL if it's missing one"""
+        self.assertEqual(normalize_url("http://example.com"),
+                         "http://example.com/")
+
+    def test_lower_case(self):
+        """Normalized URL scheme and host are lower case"""
+        self.assertEqual(normalize_url("HTTP://examPle.cOm/"),
+                         "http://example.com/")
+        self.assertEqual(normalize_url("http://example.com/A"),
+                         "http://example.com/A")
+
+    def test_strip_trailing_period(self):
+        self.assertEqual(normalize_url("http://example.com."),
+                         "http://example.com/")
+        self.assertEqual(normalize_url("http://example.com./"),
+                         "http://example.com/")
+
+    def test_capitalize_escape_sequence(self):
+        """All letters in percent-encoded triplets should be capitalized"""
+        self.assertEqual(normalize_url("http://www.example.com/a%7b%7db"),
+                         "http://www.example.com/a%7B%7Db")
+
+    def test_path_percent_encoding(self):
+        """All non-safe characters should be percent-encoded"""
+        self.assertEqual(normalize_url("http://example.com/hello world{}"),
+                         "http://example.com/hello%20world%7B%7D")
+
+    def test_unreserved_percentencoding(self):
+        """Unreserved characters should not be percent encoded. If they are, they
+        should be decoded back; except in case of `/`, `?` and `#`"""
+        self.assertEqual(normalize_url("http://www.example.com/%7Eusername/"),
+                         "http://www.example.com/~username")
+        self.assertEqual(normalize_url('http://example.com/foo%23bar'),
+                         "http://example.com/foo%23bar")
+        self.assertEqual(normalize_url('http://example.com/foo%2fbar'),
+                         'http://example.com/foo%2Fbar')
+        self.assertEqual(normalize_url('http://example.com/foo%3fbar'),
+                         'http://example.com/foo%3Fbar')
+
+    def test_remove_dot_segments(self):
+        """Convert the URL path to an absolute path by removing `.` and `..`
+        segments"""
+        self.assertEqual(normalize_url("http://www.example.com/../a/b/../c/./d.html"),
+                         "http://www.example.com/a/c/d.html")
+
+    def test_remove_default_port(self):
+        """Remove the default port for the scheme if it's present in the URL"""
+        self.assertEqual(normalize_url("http://www.example.com:80/bar.html"),
+                         "http://www.example.com/bar.html")
+        self.assertEqual(normalize_url("HTTPS://example.com:443/abc/"),
+                         "https://example.com/abc")
+
+    def test_remove_empty_port(self):
+        """Remove empty port from URL"""
+        self.assertEqual(normalize_url("http://www.example.com:/"),
+                         "http://www.example.com/")
+
+    def test_remove_extra_slash(self):
+        """Remove any extra slashes if present in the URl"""
+        # TODO: Should we actually do this?
+        # TODO: See https://webmasters.stackexchange.com/questions/8354/what-does-the-double-slash-mean-in-urls/8381#8381
+        self.assertEqual(normalize_url("http://www.example.com/foo//bar.html"),
+                         "http://www.example.com/foo/bar.html")
+        self.assertEqual(normalize_url("http://example.com///abc"),
+                         "http://example.com/abc")
+
+    def test_query_string(self):
+        """Query strings should be handled properly"""
+        self.assertEqual(normalize_url("http://example.com/?a=1"),
+                         "http://example.com/?a=1")
+        self.assertEqual(normalize_url("http://example.com?a=1"),
+                         "http://example.com/?a=1")
+        self.assertEqual(normalize_url("http://example.com/a?b=1"),
+                         "http://example.com/a?b=1")
+        self.assertEqual(normalize_url("http://example.com/a/?b=1"),
+                         "http://example.com/a?b=1")
+
+    def test_dont_percent_encode_safe_chars_query(self):
+        """Don't percent-encode safe characters in querystring"""
+        self.assertEqual(normalize_url("http://example.com/a/?face=(-.-)"),
+                         "http://example.com/a?face=(-.-)")
+
+    def test_query_sorting(self):
+        """Query strings should be sorted"""
+        self.assertEqual(normalize_url('http://example.com/a?b=1&c=2'),
+                         'http://example.com/a?b=1&c=2')
+        self.assertEqual(normalize_url('http://example.com/a?c=2&b=1'),
+                         'http://example.com/a?b=1&c=2')
+
+    def test_query_string_spaces(self):
+        """Spaces should be handled properly in query strings"""
+        self.assertEqual(normalize_url("http://example.com/search?q=a b&a=1"),
+                         "http://example.com/search?a=1&q=a+b")
+        self.assertEqual(normalize_url("http://example.com/search?q=a+b&a=1"),
+                         "http://example.com/search?a=1&q=a+b")
+        self.assertEqual(normalize_url("http://example.com/search?q=a%20b&a=1"),  # noqa
+                         "http://example.com/search?a=1&q=a+b")
+
+    def test_drop_trailing_questionmark(self):
+        """Drop the trailing question mark if no query string present"""
+        self.assertEqual(normalize_url("http://example.com/?"),
+                         "http://example.com/")
+        self.assertEqual(normalize_url("http://example.com?"),
+                         "http://example.com/")
+        self.assertEqual(normalize_url("http://example.com/a?"),
+                         "http://example.com/a")
+        self.assertEqual(normalize_url("http://example.com/a/?"),
+                         "http://example.com/a")
+
+    def test_percent_encode_querystring(self):
+        """Non-safe characters in query string should be percent-encoded"""
+        self.assertEqual(normalize_url("http://example.com/?a=hello{}"),
+                         "http://example.com/?a=hello%7B%7D")
+
+    def test_normalize_percent_encoding_in_querystring(self):
+        """Percent-encoded querystring should be uppercased"""
+        self.assertEqual(normalize_url("http://example.com/?a=b%7b%7d"),
+                         "http://example.com/?a=b%7B%7D")
+
+    def test_unicode_query_string(self):
+        """Unicode query strings should be converted to bytes using uft-8 encoding
+        and then properly percent-encoded"""
+        self.assertEqual(normalize_url("http://example.com/?file=résumé.pdf"),
+                         "http://example.com/?file=r%C3%A9sum%C3%A9.pdf")
+
+    def test_unicode_path(self):
+        """Unicode path should be converted to bytes using utf-8 encoding and then
+        percent-encoded"""
+        self.assertEqual(normalize_url("http://example.com/résumé"),
+                         "http://example.com/r%C3%A9sum%C3%A9")
+
+    def test_idna(self):
+        """International Domain Names should be normalized to safe characters"""
+        self.assertEqual(normalize_url("http://ドメイン.テスト"),
+                         "http://xn--eckwd4c7c.xn--zckzah/")
+        self.assertEqual(normalize_url("http://Яндекс.рф"),
+                         "http://xn--d1acpjx3f.xn--p1ai/")
+
+    def test_dont_change_username_password(self):
+        """Username and password shouldn't be lowercased"""
+        self.assertEqual(normalize_url("http://Foo:BAR@exaMPLE.COM/"),
+                         "http://Foo:BAR@example.com/")
+
+    def test_normalize_ipv4(self):
+        """Normalize ipv4 URLs"""
+        assert normalize_url("http://192.168.0.1/") == "http://192.168.0.1/"
+        assert (normalize_url("http://192.168.0.1:8080/a?b=1") ==
+                "http://192.168.0.1:8080/a?b=1")
+        assert normalize_url("192.168.0.1") == "http://192.168.0.1/"
+        assert (normalize_url("192.168.0.1:8080/a/b/c") ==
+                "http://192.168.0.1:8080/a/b/c")
+
+    def test_normalize_ipv6(self):
+        """Normalize ipv6 URLs"""
+        assert normalize_url("[::1]") == "http://[::1]/"
+        assert normalize_url("http://[::1]") == "http://[::1]/"
+        assert normalize_url("[::1]:8080") == "http://[::1]:8080/"
+        assert normalize_url("http://[::1]:8080") == "http://[::1]:8080/"
+
+    def test_strip_leading_trailing_whitespace(self):
+        """Strip leading and trailing whitespace if any"""
+        assert normalize_url("   http://example.com  ") == "http://example.com/"
+        assert normalize_url("http://example.com/a  ") == "http://example.com/a"
+        assert normalize_url("   http://example.com/") == "http://example.com/"
+
+    def test_non_ideal_inputs(self):
+        """Not the ideal input; but we should handle it anyway"""
+        assert normalize_url("example.com") == "http://example.com/"
+        assert normalize_url("example.com/abc") == "http://example.com/abc"
+        assert normalize_url("//example.com/abc") == "http://example.com/abc"
+
+    def test_additional_query_args(self):
+        """Add any additional query arguments to the URL"""
+        self.assertEqual(normalize_url("http://example.com?c=d", [("a", "b")]),
+                         "http://example.com/?a=b&c=d")
+        self.assertEqual(normalize_url("http://example.com", [("a", "b")]),
+                         "http://example.com/?a=b")
+        self.assertEqual(normalize_url("http://example.com", [("résumé", "résumé")]),
+                         "http://example.com/?r%C3%A9sum%C3%A9=r%C3%A9sum%C3%A9")
+
+    def test_non_urls(self):
+        """If a non-URL string is passed, return None"""
+        assert normalize_url("") is None
+        assert normalize_url("abc xyz") is None
+        assert normalize_url("asb#abc") is None
+        assert normalize_url("Яндекс.рф") is not None
+        assert normalize_url("google.blog") is not None
+        assert normalize_url("http//google.com") is None
+        assert normalize_url("http://user@pass:example.com") is None
+
+    def test_drop_fragments(self):
+        """Drop or keep fragments based on the option passed"""
+        assert (normalize_url("http://example.com/a?b=1#frag")
+                == "http://example.com/a?b=1")
+        assert (normalize_url("http://example.com/a?b=1#frag", drop_fragments=False)
+                == "http://example.com/a?b=1#frag")
+
+    def test_non_string_input(self):
+        """Non-string input should produce None as result"""
+        assert normalize_url(None) is None
+        assert normalize_url([]) is None
+        assert normalize_url(123) is None
diff --git a/urlcanon/utils.py b/urlcanon/utils.py
index 0d05816..099bfe9 100644
--- a/urlcanon/utils.py
+++ b/urlcanon/utils.py
@@ -1,6 +1,11 @@
 from __future__ import unicode_literals
 from six import text_type
-from six.moves.urllib.parse import unquote_to_bytes, urlencode
+from six.moves.urllib.parse import unquote, quote, quote_plus
+from six.moves.urllib.parse import unquote_to_bytes
+
+from urlcanon.constants import SAFE_CHARS
+
+_enc = 'utf-8'
 
 
 def _noop(obj):
@@ -8,11 +13,11 @@ def _noop(obj):
 
 
 def _encode_result(obj):
-    return obj.encode('utf-8', 'strict')
+    return obj.encode(_enc)
 
 
 def _decode_args(args):
-    return tuple(x.decode('utf-8', 'strict') if x else '' for x in args)
+    return tuple(x.decode(_enc) if x else '' for x in args)
 
 
 def _coerce_args(*args):
@@ -69,5 +74,28 @@ def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False):
     return r
 
 
+def _quote(text):
+    if isinstance(text, text_type):
+        text = text.encode(_enc)
+    return quote(text, safe=SAFE_CHARS)
+
+
+def _quote_plus(text):
+    if isinstance(text, text_type):
+        text = text.encode(_enc)
+    return quote_plus(text, safe=SAFE_CHARS)
+
+
+def _unquote(text):
+    text = unquote(text)
+    if not isinstance(text, text_type):
+        text = text.decode(_enc)
+    return text
+
+
 def _urlencode(queries):
-    return urlencode(queries)
\ No newline at end of file
+    parts = []
+    for k, v in sorted(set(queries)):
+        part = _quote_plus(k), _quote_plus(v)
+        parts.append('='.join(part))
+    return '&'.join(parts)

From 78729a201157a605256b5b4dbb73f999f8e4a820 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 11:15:28 +0100
Subject: [PATCH 04/22] try and patch up unquote_to_bytes

---
 urlcanon/utils.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/urlcanon/utils.py b/urlcanon/utils.py
index 099bfe9..108e8ec 100644
--- a/urlcanon/utils.py
+++ b/urlcanon/utils.py
@@ -1,10 +1,13 @@
 from __future__ import unicode_literals
-from six import text_type
-from six.moves.urllib.parse import unquote, quote, quote_plus
-from six.moves.urllib.parse import unquote_to_bytes
-
+from six import text_type, PY3
+from six.moves.urllib.parse import quote, quote_plus
 from urlcanon.constants import SAFE_CHARS
 
+if PY3:
+    from urllib.parse import unquote_to_bytes
+else:
+    from urllib import unquote as unquote_to_bytes
+
 _enc = 'utf-8'
 
 
@@ -87,9 +90,10 @@ def _quote_plus(text):
 
 
 def _unquote(text):
-    text = unquote(text)
-    if not isinstance(text, text_type):
-        text = text.decode(_enc)
+    if isinstance(text, text_type):
+        text = text.encode(_enc)
+    text = unquote_to_bytes(text)
+    text = text.decode(_enc)
     return text
 
 

From b421dd14d70e192316fb5274cd1b0a9c3e55642d Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 11:17:18 +0100
Subject: [PATCH 05/22] argh python 3 bytes str sort

---
 urlcanon/normalizer.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/urlcanon/normalizer.py b/urlcanon/normalizer.py
index cc41352..4a396ef 100644
--- a/urlcanon/normalizer.py
+++ b/urlcanon/normalizer.py
@@ -111,6 +111,4 @@ def _normalize_query(query, extra_query_args):
     if extra_query_args:
         for (name, val) in extra_query_args:
             queries_list.append((name, val))
-    queries_list.sort()
-    query = _urlencode(queries_list)
-    return query
+    return _urlencode(queries_list)

From 0552cb2158bbdde3a5f33b7c57952e57365ea076 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 11:19:36 +0100
Subject: [PATCH 06/22] fuck my life

---
 urlcanon/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/urlcanon/utils.py b/urlcanon/utils.py
index 108e8ec..df293c5 100644
--- a/urlcanon/utils.py
+++ b/urlcanon/utils.py
@@ -99,7 +99,8 @@ def _unquote(text):
 
 def _urlencode(queries):
     parts = []
-    for k, v in sorted(set(queries)):
+    for k, v in queries:
         part = _quote_plus(k), _quote_plus(v)
         parts.append('='.join(part))
+    parts = sorted(set(parts))
     return '&'.join(parts)

From a7e097bc6be228be924136a9d3198ff316bb6656 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 11:24:51 +0100
Subject: [PATCH 07/22] release stuff

---
 Makefile | 17 +++++++++++++++++
 setup.py |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 Makefile

diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..40c48b9
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,17 @@
+all: clean test dists
+
+test:
+	python setup.py test
+
+dists:
+	python setup.py sdist bdist_wheel
+
+release: clean dists
+	twine upload dist/*
+
+clean:
+	rm -rf dist build .eggs
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -f {} +
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
\ No newline at end of file
diff --git a/setup.py b/setup.py
index fffd1bf..11d9184 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="urlcanon",
-    version="0.0.1",
+    version="1.0.0",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",

From cef15902f070361bdc518f81db19659b9e5bef82 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 11:30:42 +0100
Subject: [PATCH 08/22] Rename once more

---
 README.md                                            | 8 ++++----
 setup.py                                             | 6 +++---
 {urlcanon => urlnormalizer}/__init__.py              | 0
 {urlcanon => urlnormalizer}/constants.py             | 0
 {urlcanon => urlnormalizer}/normalizer.py            | 0
 {urlcanon => urlnormalizer}/tests/__init__.py        | 0
 {urlcanon => urlnormalizer}/tests/test_normalizer.py | 0
 {urlcanon => urlnormalizer}/utils.py                 | 0
 {urlcanon => urlnormalizer}/validator.py             | 0
 9 files changed, 7 insertions(+), 7 deletions(-)
 rename {urlcanon => urlnormalizer}/__init__.py (100%)
 rename {urlcanon => urlnormalizer}/constants.py (100%)
 rename {urlcanon => urlnormalizer}/normalizer.py (100%)
 rename {urlcanon => urlnormalizer}/tests/__init__.py (100%)
 rename {urlcanon => urlnormalizer}/tests/test_normalizer.py (100%)
 rename {urlcanon => urlnormalizer}/utils.py (100%)
 rename {urlcanon => urlnormalizer}/validator.py (100%)

diff --git a/README.md b/README.md
index ce0e72a..8580f0d 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
-# urlcanon
+# urlnormalizer
 
-[![Build Status](https://travis-ci.org/sunu/url-normalizer.svg?branch=master)](https://travis-ci.org/sunu/url-normalizer)
+[![Build Status](https://travis-ci.org/alephdata/urlnormalizer.svg?branch=master)](https://travis-ci.org/alephdata/urlnormalizer)
 
 Normalizes URL by doing the following:
 
@@ -25,7 +25,7 @@ Works with `http` and `https` urls only for now.
 Install using `pip`
 
 ```console
-$ pip install git+https://github.com/sunu/url-normalizer
+$ pip install urlnormalizer
 ```
 or clone and install using `python setup.py install`
 
@@ -34,7 +34,7 @@ or clone and install using `python setup.py install`
 Pass a url to the `normalize_url` function as a `str` type to normalize it. 
 
 ```pycon
-In [1]: from normalizer import normalize_url
+In [1]: from urlnormalizer import normalize_url
 
 In [2]: normalize_url("hello.com")
 Out[2]: 'http://hello.com/'
diff --git a/setup.py b/setup.py
index 11d9184..2b06a15 100644
--- a/setup.py
+++ b/setup.py
@@ -2,15 +2,15 @@
 from setuptools import setup
 
 setup(
-    name="urlcanon",
+    name="urlnormalizer",
     version="1.0.0",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",
     long_description="",
     license="MIT",
-    url="https://github.com/alephdata/urlcanon",
-    packages=['urlcanon'],
+    url="https://github.com/alephdata/urlnormalizer",
+    packages=['urlnormalizer'],
     classifiers=[
         "Development Status :: 3 - Alpha",
         "Intended Audience :: Developers",
diff --git a/urlcanon/__init__.py b/urlnormalizer/__init__.py
similarity index 100%
rename from urlcanon/__init__.py
rename to urlnormalizer/__init__.py
diff --git a/urlcanon/constants.py b/urlnormalizer/constants.py
similarity index 100%
rename from urlcanon/constants.py
rename to urlnormalizer/constants.py
diff --git a/urlcanon/normalizer.py b/urlnormalizer/normalizer.py
similarity index 100%
rename from urlcanon/normalizer.py
rename to urlnormalizer/normalizer.py
diff --git a/urlcanon/tests/__init__.py b/urlnormalizer/tests/__init__.py
similarity index 100%
rename from urlcanon/tests/__init__.py
rename to urlnormalizer/tests/__init__.py
diff --git a/urlcanon/tests/test_normalizer.py b/urlnormalizer/tests/test_normalizer.py
similarity index 100%
rename from urlcanon/tests/test_normalizer.py
rename to urlnormalizer/tests/test_normalizer.py
diff --git a/urlcanon/utils.py b/urlnormalizer/utils.py
similarity index 100%
rename from urlcanon/utils.py
rename to urlnormalizer/utils.py
diff --git a/urlcanon/validator.py b/urlnormalizer/validator.py
similarity index 100%
rename from urlcanon/validator.py
rename to urlnormalizer/validator.py

From 0b7faf411d65eed078f20e561753bd0b583be7bd Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 11:31:01 +0100
Subject: [PATCH 09/22] up version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2b06a15..c33300c 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="urlnormalizer",
-    version="1.0.0",
+    version="1.0.1",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",

From 3642ae21e53fb25a650a0b590e4c77ca1d8654cf Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 11:34:13 +0100
Subject: [PATCH 10/22] fix imports

---
 setup.py                               | 2 +-
 urlnormalizer/__init__.py              | 4 ++--
 urlnormalizer/normalizer.py            | 6 +++---
 urlnormalizer/tests/test_normalizer.py | 2 +-
 urlnormalizer/utils.py                 | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/setup.py b/setup.py
index c33300c..a2bc0ad 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="urlnormalizer",
-    version="1.0.1",
+    version="1.0.2",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",
diff --git a/urlnormalizer/__init__.py b/urlnormalizer/__init__.py
index 80264ac..8d17d50 100644
--- a/urlnormalizer/__init__.py
+++ b/urlnormalizer/__init__.py
@@ -1,4 +1,4 @@
-from urlcanon.normalizer import normalize_url
-from urlcanon.validator import is_valid_url
+from urlnormalizer.normalizer import normalize_url
+from urlnormalizer.validator import is_valid_url
 
 __all__ = [normalize_url, is_valid_url]
diff --git a/urlnormalizer/normalizer.py b/urlnormalizer/normalizer.py
index 4a396ef..f29235a 100644
--- a/urlnormalizer/normalizer.py
+++ b/urlnormalizer/normalizer.py
@@ -5,9 +5,9 @@
 from six.moves.urllib.parse import urlunsplit
 from six.moves.urllib.parse import urlsplit
 
-from urlcanon.utils import _parse_qsl, _urlencode, _quote, _unquote
-from urlcanon.validator import is_valid_url
-from urlcanon.constants import SCHEMES, DEFAULT_PORTS
+from urlnormalizer.utils import _parse_qsl, _urlencode, _quote, _unquote
+from urlnormalizer.validator import is_valid_url
+from urlnormalizer.constants import SCHEMES, DEFAULT_PORTS
 
 
 def normalize_url(url, extra_query_args=None, drop_fragments=True):
diff --git a/urlnormalizer/tests/test_normalizer.py b/urlnormalizer/tests/test_normalizer.py
index 33927e5..47a7b3f 100644
--- a/urlnormalizer/tests/test_normalizer.py
+++ b/urlnormalizer/tests/test_normalizer.py
@@ -4,7 +4,7 @@
 from six import text_type
 from unittest import TestCase
 
-from ..normalizer import normalize_url
+from urlnormalizer import normalize_url
 
 
 class UrlTestCase(TestCase):
diff --git a/urlnormalizer/utils.py b/urlnormalizer/utils.py
index df293c5..60dea98 100644
--- a/urlnormalizer/utils.py
+++ b/urlnormalizer/utils.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 from six import text_type, PY3
 from six.moves.urllib.parse import quote, quote_plus
-from urlcanon.constants import SAFE_CHARS
+from urlnormalizer.constants import SAFE_CHARS
 
 if PY3:
     from urllib.parse import unquote_to_bytes

From c21020d94492d957a36019a4fc743d6e33fcec72 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Tue, 31 Oct 2017 11:47:03 +0100
Subject: [PATCH 11/22] handle non-string inputs for is_valid_url

---
 setup.py                   | 2 +-
 urlnormalizer/validator.py | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index a2bc0ad..786d306 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="urlnormalizer",
-    version="1.0.2",
+    version="1.0.3",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",
diff --git a/urlnormalizer/validator.py b/urlnormalizer/validator.py
index 029f6d5..09ef3c6 100644
--- a/urlnormalizer/validator.py
+++ b/urlnormalizer/validator.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 import re
+import six
 
 
 def is_valid_url(value):
@@ -7,6 +8,8 @@ def is_valid_url(value):
     Does the value look like a URL?
     From https://github.com/django/django/blob/stable/2.0.x/django/core/validators.py
     """
+    if not isinstance(value, six.string_types):
+        return False
     if value.startswith("//"):
         value = value[2:]
     ul = '\u00a1-\uffff'  # unicode letters range (must not be a raw string)

From 2b5d153ecca8ed7ee6fddf3fd7a73429a7074b4e Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Wed, 1 Nov 2017 07:49:00 +0100
Subject: [PATCH 12/22] convert extra query args to string

---
 setup.py                               |  2 +-
 urlnormalizer/tests/test_normalizer.py | 11 ++++++++++
 urlnormalizer/utils.py                 | 28 +++++++++++++-------------
 3 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/setup.py b/setup.py
index 786d306..11e5624 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="urlnormalizer",
-    version="1.0.3",
+    version="1.0.4",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",
diff --git a/urlnormalizer/tests/test_normalizer.py b/urlnormalizer/tests/test_normalizer.py
index 47a7b3f..6ce704d 100644
--- a/urlnormalizer/tests/test_normalizer.py
+++ b/urlnormalizer/tests/test_normalizer.py
@@ -76,6 +76,17 @@ def test_remove_empty_port(self):
         self.assertEqual(normalize_url("http://www.example.com:/"),
                          "http://www.example.com/")
 
+    def test_extra_query_args(self):
+        """Extra query args"""
+        args = (('a', 4), )
+        self.assertEqual(normalize_url("http://www.example.com:/",
+                                       extra_query_args=args),
+                         "http://www.example.com/?a=4")
+        args = (('a', None), )
+        self.assertEqual(normalize_url("http://www.example.com:/",
+                                       extra_query_args=args),
+                         "http://www.example.com/?a=")
+
     def test_remove_extra_slash(self):
         """Remove any extra slashes if present in the URl"""
         # TODO: Should we actually do this?
diff --git a/urlnormalizer/utils.py b/urlnormalizer/utils.py
index 60dea98..698d6cc 100644
--- a/urlnormalizer/utils.py
+++ b/urlnormalizer/utils.py
@@ -1,9 +1,9 @@
 from __future__ import unicode_literals
-from six import text_type, PY3
+import six
 from six.moves.urllib.parse import quote, quote_plus
 from urlnormalizer.constants import SAFE_CHARS
 
-if PY3:
+if six.PY3:
     from urllib.parse import unquote_to_bytes
 else:
     from urllib import unquote as unquote_to_bytes
@@ -29,11 +29,11 @@ def _coerce_args(*args):
     # an appropriate result coercion function
     #   - noop for str inputs
     #   - encoding function otherwise
-    str_input = isinstance(args[0], text_type)
+    str_input = isinstance(args[0], six.text_type)
     for arg in args[1:]:
         # We special-case the empty string to support the
         # "scheme=''" default argument to some functions
-        if arg and isinstance(arg, text_type) != str_input:
+        if arg and isinstance(arg, six.text_type) != str_input:
             raise TypeError("Cannot mix str and non-str arguments")
     if str_input:
         return args + (_noop,)
@@ -77,20 +77,20 @@ def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False):
     return r
 
 
-def _quote(text):
-    if isinstance(text, text_type):
+def _quote(text, plus=False):
+    if text is None:
+        return ''
+    if not isinstance(text, six.string_types):
+        text = six.text_type(text)
+    if isinstance(text, six.text_type):
         text = text.encode(_enc)
+    if plus:
+        return quote_plus(text, safe=SAFE_CHARS)
     return quote(text, safe=SAFE_CHARS)
 
 
-def _quote_plus(text):
-    if isinstance(text, text_type):
-        text = text.encode(_enc)
-    return quote_plus(text, safe=SAFE_CHARS)
-
-
 def _unquote(text):
-    if isinstance(text, text_type):
+    if isinstance(text, six.text_type):
         text = text.encode(_enc)
     text = unquote_to_bytes(text)
     text = text.decode(_enc)
@@ -100,7 +100,7 @@ def _unquote(text):
 def _urlencode(queries):
     parts = []
     for k, v in queries:
-        part = _quote_plus(k), _quote_plus(v)
+        part = _quote(k, plus=True), _quote(v, plus=True)
         parts.append('='.join(part))
     parts = sorted(set(parts))
     return '&'.join(parts)

From 0c0d1105e15551dae6d27922b839cd132aa4d0e8 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Wed, 1 Nov 2017 07:59:02 +0100
Subject: [PATCH 13/22] excludes bytes from conversion

---
 urlnormalizer/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/urlnormalizer/utils.py b/urlnormalizer/utils.py
index 698d6cc..e4aefc6 100644
--- a/urlnormalizer/utils.py
+++ b/urlnormalizer/utils.py
@@ -80,7 +80,7 @@ def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False):
 def _quote(text, plus=False):
     if text is None:
         return ''
-    if not isinstance(text, six.string_types):
+    if not isinstance(text, (six.text_type, six.binary_type)):
         text = six.text_type(text)
     if isinstance(text, six.text_type):
         text = text.encode(_enc)

From 4e7ae57b912e10f0d6aff878c98547ac6a1e4802 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Sun, 5 Nov 2017 15:33:57 +0100
Subject: [PATCH 14/22] extract query string function from aleph.

---
 setup.py                               | 2 +-
 urlnormalizer/__init__.py              | 3 ++-
 urlnormalizer/query.py                 | 9 +++++++++
 urlnormalizer/tests/test_normalizer.py | 9 ++++++++-
 4 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100644 urlnormalizer/query.py

diff --git a/setup.py b/setup.py
index 11e5624..67bbfd5 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="urlnormalizer",
-    version="1.0.4",
+    version="1.1.0",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",
diff --git a/urlnormalizer/__init__.py b/urlnormalizer/__init__.py
index 8d17d50..1f2671f 100644
--- a/urlnormalizer/__init__.py
+++ b/urlnormalizer/__init__.py
@@ -1,4 +1,5 @@
 from urlnormalizer.normalizer import normalize_url
 from urlnormalizer.validator import is_valid_url
+from urlnormalizer.query import query_string
 
-__all__ = [normalize_url, is_valid_url]
+__all__ = [normalize_url, is_valid_url, query_string]
diff --git a/urlnormalizer/query.py b/urlnormalizer/query.py
new file mode 100644
index 0000000..82e82ec
--- /dev/null
+++ b/urlnormalizer/query.py
@@ -0,0 +1,9 @@
+from urlnormalizer.utils import _urlencode
+
+
+def query_string(items):
+    """Given a list of tuples, returns a query string for URL building."""
+    query = [(k, v) for (k, v) in items if v is not None]
+    if not len(query):
+        return ''
+    return '?' + _urlencode(query)
diff --git a/urlnormalizer/tests/test_normalizer.py b/urlnormalizer/tests/test_normalizer.py
index 6ce704d..a24d0e0 100644
--- a/urlnormalizer/tests/test_normalizer.py
+++ b/urlnormalizer/tests/test_normalizer.py
@@ -4,7 +4,7 @@
 from six import text_type
 from unittest import TestCase
 
-from urlnormalizer import normalize_url
+from urlnormalizer import normalize_url, query_string
 
 
 class UrlTestCase(TestCase):
@@ -232,3 +232,10 @@ def test_non_string_input(self):
         assert normalize_url(None) is None
         assert normalize_url([]) is None
         assert normalize_url(123) is None
+
+    def test_query_string_empty(self):
+        self.assertEqual(query_string((('foo', None), )), '')
+
+    def test_query_string_item(self):
+        query = (('b', 5), ('a', '1'))
+        self.assertEqual(query_string(query), '?a=1&b=5')

From ff29d4681979d6e6bb5cfe85aa1360e5443948a0 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Thu, 16 Nov 2017 15:19:53 +0100
Subject: [PATCH 15/22] universal packages

---
 setup.cfg | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 9af7e6f..945205e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,5 @@
 [aliases]
-test=pytest
\ No newline at end of file
+test=pytest
+
+[bdist_wheel]
+universal=1
\ No newline at end of file

From f824d2321747eaed70477d407c56413e65487228 Mon Sep 17 00:00:00 2001
From: rhiaro <amy@rhiaro.co.uk>
Date: Mon, 27 Nov 2017 16:52:16 +0100
Subject: [PATCH 16/22] Don't strip trailing slashes

---
 urlnormalizer/normalizer.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/urlnormalizer/normalizer.py b/urlnormalizer/normalizer.py
index f29235a..c754289 100644
--- a/urlnormalizer/normalizer.py
+++ b/urlnormalizer/normalizer.py
@@ -71,11 +71,16 @@ def _normalize_path(path):
     # percent-encoded and already percent-encoded triplets are upper cased.
     unquoted_path = _unquote(path)
     path = _quote(unquoted_path) or "/"
+    trailing_slash = "/" == path[-1]
     # Use `os.path.normpath` to normalize paths i.e. remove duplicate `/` and
     # make the path absolute when `..` or `.` segments are present.
     # TODO: Should we remove duplicate slashes?
     # TODO: See https://webmasters.stackexchange.com/questions/8354/what-does-the-double-slash-mean-in-urls/8381#8381
     path = normpath(path)
+    # normpath strips trailing slash. Add it back if it was there because
+    # this might make a difference for URLs.
+    if trailing_slash:
+        path = path + "/"
     # POSIX allows one or two initial slashes, but treats three or more
     # as single slash.So if there are two initial slashes, make them one.
     if path.startswith("//"):
@@ -97,7 +102,7 @@ def _normalize_netloc(scheme, netloc, username, password, port):
     netloc = netloc.lower().rstrip(":").rstrip(".")
     # strip default port
     if port and DEFAULT_PORTS.get(scheme) == port:
-        netloc = netloc.rstrip(":"+str(port))
+        netloc = netloc.rstrip(":" + str(port))
     # Put auth info back in
     if auth:
         netloc = auth + "@" + netloc

From 394cba2c27004b9992dfc14a873af2a629b00d9a Mon Sep 17 00:00:00 2001
From: rhiaro <amy@rhiaro.co.uk>
Date: Mon, 27 Nov 2017 16:52:42 +0100
Subject: [PATCH 17/22] up

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 67bbfd5..a762d95 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="urlnormalizer",
-    version="1.1.0",
+    version="1.1.1",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",

From 4d48a290f68c5beb6f350db742aca362f6552d74 Mon Sep 17 00:00:00 2001
From: rhiaro <amy@rhiaro.co.uk>
Date: Mon, 27 Nov 2017 17:00:03 +0100
Subject: [PATCH 18/22] Fix tests

---
 urlnormalizer/tests/test_normalizer.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/urlnormalizer/tests/test_normalizer.py b/urlnormalizer/tests/test_normalizer.py
index a24d0e0..11391ef 100644
--- a/urlnormalizer/tests/test_normalizer.py
+++ b/urlnormalizer/tests/test_normalizer.py
@@ -50,7 +50,7 @@ def test_unreserved_percentencoding(self):
         """Unreserved characters should not be percent encoded. If they are, they
         should be decoded back; except in case of `/`, `?` and `#`"""
         self.assertEqual(normalize_url("http://www.example.com/%7Eusername/"),
-                         "http://www.example.com/~username")
+                         "http://www.example.com/~username/")
         self.assertEqual(normalize_url('http://example.com/foo%23bar'),
                          "http://example.com/foo%23bar")
         self.assertEqual(normalize_url('http://example.com/foo%2fbar'),
@@ -69,7 +69,7 @@ def test_remove_default_port(self):
         self.assertEqual(normalize_url("http://www.example.com:80/bar.html"),
                          "http://www.example.com/bar.html")
         self.assertEqual(normalize_url("HTTPS://example.com:443/abc/"),
-                         "https://example.com/abc")
+                         "https://example.com/abc/")
 
     def test_remove_empty_port(self):
         """Remove empty port from URL"""
@@ -105,12 +105,12 @@ def test_query_string(self):
         self.assertEqual(normalize_url("http://example.com/a?b=1"),
                          "http://example.com/a?b=1")
         self.assertEqual(normalize_url("http://example.com/a/?b=1"),
-                         "http://example.com/a?b=1")
+                         "http://example.com/a/?b=1")
 
     def test_dont_percent_encode_safe_chars_query(self):
         """Don't percent-encode safe characters in querystring"""
         self.assertEqual(normalize_url("http://example.com/a/?face=(-.-)"),
-                         "http://example.com/a?face=(-.-)")
+                         "http://example.com/a/?face=(-.-)")
 
     def test_query_sorting(self):
         """Query strings should be sorted"""
@@ -137,7 +137,7 @@ def test_drop_trailing_questionmark(self):
         self.assertEqual(normalize_url("http://example.com/a?"),
                          "http://example.com/a")
         self.assertEqual(normalize_url("http://example.com/a/?"),
-                         "http://example.com/a")
+                         "http://example.com/a/")
 
     def test_percent_encode_querystring(self):
         """Non-safe characters in query string should be percent-encoded"""

From 4825fa63d6145d0504d315d2743cc2cde53e6417 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Sun, 4 Feb 2018 22:54:26 +0100
Subject: [PATCH 19/22] Change behaviour: keep trailing slashes on URLs

---
 setup.py                    | 2 +-
 urlnormalizer/normalizer.py | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/setup.py b/setup.py
index a762d95..e946936 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="urlnormalizer",
-    version="1.1.1",
+    version="1.2.0",
     author="Tarashish Mishra",
     author_email="sunu@sunu.in",
     description="Normalize URLs. Mostly useful for deduplicating HTTP URLs.",
diff --git a/urlnormalizer/normalizer.py b/urlnormalizer/normalizer.py
index c754289..78acab1 100644
--- a/urlnormalizer/normalizer.py
+++ b/urlnormalizer/normalizer.py
@@ -79,12 +79,12 @@ def _normalize_path(path):
     path = normpath(path)
     # normpath strips trailing slash. Add it back if it was there because
     # this might make a difference for URLs.
-    if trailing_slash:
+    if trailing_slash and not path.endswith('/'):
         path = path + "/"
     # POSIX allows one or two initial slashes, but treats three or more
     # as single slash.So if there are two initial slashes, make them one.
-    if path.startswith("//"):
-        path = "/" + path.lstrip("/")
+    if path.startswith('//'):
+        path = '/' + path.lstrip('/')
     return path
 
 

From dab31c7b3f59eb2cc9e7ed358e9f0b3f2a79a0d9 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Sun, 4 Feb 2018 22:59:40 +0100
Subject: [PATCH 20/22] Being OCD.

---
 urlnormalizer/normalizer.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/urlnormalizer/normalizer.py b/urlnormalizer/normalizer.py
index 78acab1..41a0c59 100644
--- a/urlnormalizer/normalizer.py
+++ b/urlnormalizer/normalizer.py
@@ -70,8 +70,8 @@ def _normalize_path(path):
     # unquote and quote the path so that any non-safe character is
     # percent-encoded and already percent-encoded triplets are upper cased.
     unquoted_path = _unquote(path)
-    path = _quote(unquoted_path) or "/"
-    trailing_slash = "/" == path[-1]
+    path = _quote(unquoted_path) or '/'
+    trailing_slash = path.endswith('/')
     # Use `os.path.normpath` to normalize paths i.e. remove duplicate `/` and
     # make the path absolute when `..` or `.` segments are present.
     # TODO: Should we remove duplicate slashes?
@@ -80,7 +80,7 @@ def _normalize_path(path):
     # normpath strips trailing slash. Add it back if it was there because
     # this might make a difference for URLs.
     if trailing_slash and not path.endswith('/'):
-        path = path + "/"
+        path = path + '/'
     # POSIX allows one or two initial slashes, but treats three or more
     # as single slash.So if there are two initial slashes, make them one.
     if path.startswith('//'):

From d1e438590dfbb682627062d50d7a7eb035805ccd Mon Sep 17 00:00:00 2001
From: jen <jen@occrp.org>
Date: Sun, 3 Mar 2019 23:18:56 +0100
Subject: [PATCH 21/22] Attempt to fix decoding from bytes by detecting proper
 coding

---
 setup.py                               | 5 ++++-
 urlnormalizer/tests/test_normalizer.py | 3 +++
 urlnormalizer/utils.py                 | 8 +++++++-
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index e946936..00364bb 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,10 @@
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 3"
     ],
-    install_requires=['six'],
+    install_requires=[
+        'six',
+        'chardet'
+    ],
     setup_requires=['pytest-runner'],
     tests_require=['pytest'],
 )
diff --git a/urlnormalizer/tests/test_normalizer.py b/urlnormalizer/tests/test_normalizer.py
index 11391ef..cb90510 100644
--- a/urlnormalizer/tests/test_normalizer.py
+++ b/urlnormalizer/tests/test_normalizer.py
@@ -160,6 +160,9 @@ def test_unicode_path(self):
         percent-encoded"""
         self.assertEqual(normalize_url("http://example.com/résumé"),
                          "http://example.com/r%C3%A9sum%C3%A9")
+        self.assertEqual(normalize_url("https://ru.wikipedia.org/wiki/%C4%E0%F3%EA%E5%E5%E2%2c_%D1%E5%F0%E8%EA%E1%E5%EA_%C6%F3%F1%F3%EF%E1%E5%EA%EE%E2%E8%F7"),
+                                       "https://ru.wikipedia.org/wiki/%D0%94%D0%B0%D1%83%D0%BA%D0%B5%D0%B5%D0%B2,_%D0%A1%D0%B5%D1%80%D0%B8%D0%BA%D0%B1%D0%B5%D0%BA_%D0%96%D1%83%D1%81%D1%83%D0%BF%D0%B1%D0%B5%D0%BA%D0%BE%D0%B2%D0%B8%D1%87"
+                                       )
 
     def test_idna(self):
         """International Domain Names should be normalized to safe characters"""
diff --git a/urlnormalizer/utils.py b/urlnormalizer/utils.py
index e4aefc6..676886e 100644
--- a/urlnormalizer/utils.py
+++ b/urlnormalizer/utils.py
@@ -2,6 +2,7 @@
 import six
 from six.moves.urllib.parse import quote, quote_plus
 from urlnormalizer.constants import SAFE_CHARS
+from chardet import detect
 
 if six.PY3:
     from urllib.parse import unquote_to_bytes
@@ -9,6 +10,7 @@
     from urllib import unquote as unquote_to_bytes
 
 _enc = 'utf-8'
+_enc_fallback = 'raw_unicode_escape'
 
 
 def _noop(obj):
@@ -93,7 +95,11 @@ def _unquote(text):
     if isinstance(text, six.text_type):
         text = text.encode(_enc)
     text = unquote_to_bytes(text)
-    text = text.decode(_enc)
+    try:
+        text = text.decode(_enc)
+    except UnicodeDecodeError:
+        encoding = detect(text).get('encoding', _enc_fallback)
+        text = text.decode(encoding)
     return text
 
 

From 7003326875624c713bc1fd632d141d9ed0da7056 Mon Sep 17 00:00:00 2001
From: Friedrich Lindenberg <friedrich@pudo.org>
Date: Mon, 4 Mar 2019 21:27:01 +0100
Subject: [PATCH 22/22] Hope to make tests pass

---
 .gitignore                  |   2 +
 .travis.yml                 |   1 -
 Pipfile                     |  20 -----
 Pipfile.lock                | 156 ------------------------------------
 urlnormalizer/normalizer.py |   2 +-
 urlnormalizer/utils.py      |   6 +-
 6 files changed, 6 insertions(+), 181 deletions(-)
 delete mode 100644 Pipfile
 delete mode 100644 Pipfile.lock

diff --git a/.gitignore b/.gitignore
index 2d865d2..899ce90 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,8 @@ __pycache__/
 
 # Distribution / packaging
 .Python
+.vscode/
+.pytest_cache/
 build/
 develop-eggs/
 dist/
diff --git a/.travis.yml b/.travis.yml
index e468c82..60743eb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,6 @@
 language: python
 python:
     - "2.7"
-    - "3.3"
     - "3.6"
 script:
     - "python setup.py test"
diff --git a/Pipfile b/Pipfile
deleted file mode 100644
index ecb0d1c..0000000
--- a/Pipfile
+++ /dev/null
@@ -1,20 +0,0 @@
-[[source]]
-
-url = "https://pypi.python.org/simple"
-verify_ssl = true
-name = "pypi"
-
-
-[dev-packages]
-
-pytest = "*"
-pylint = "*"
-
-
-[packages]
-
-
-
-[requires]
-
-python_version = "3.6"
\ No newline at end of file
diff --git a/Pipfile.lock b/Pipfile.lock
deleted file mode 100644
index e6ac80a..0000000
--- a/Pipfile.lock
+++ /dev/null
@@ -1,156 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "1bd673160561b20e61306edf380454208fae2d1949d4aa10f53d1029a124acba"
-        },
-        "host-environment-markers": {
-            "implementation_name": "cpython",
-            "implementation_version": "3.6.0",
-            "os_name": "posix",
-            "platform_machine": "x86_64",
-            "platform_python_implementation": "CPython",
-            "platform_release": "16.7.0",
-            "platform_system": "Darwin",
-            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
-            "python_full_version": "3.6.0",
-            "python_version": "3.6",
-            "sys_platform": "darwin"
-        },
-        "pipfile-spec": 6,
-        "requires": {
-            "python_version": "3.6"
-        },
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.python.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {},
-    "develop": {
-        "astroid": {
-            "hashes": [
-                "sha256:39a21dd2b5d81a6731dc0ac2884fa419532dffd465cdd43ea6c168d36b76efb3",
-                "sha256:492c2a2044adbf6a84a671b7522e9295ad2f6a7c781b899014308db25312dd35"
-            ],
-            "version": "==1.5.3"
-        },
-        "backports.functools-lru-cache": {
-            "hashes": [
-                "sha256:4ba998e881f285c1d1b73f5b6e3766539b4e162320f9589334400c5ddc35198c",
-                "sha256:31f235852f88edc1558d428d890663c49eb4514ffec9f3650e7f3c9e4a12e36f"
-            ],
-            "markers": "python_version == '2.7'",
-            "version": "==1.4"
-        },
-        "configparser": {
-            "hashes": [
-                "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a"
-            ],
-            "markers": "python_version == '2.7'",
-            "version": "==3.5.0"
-        },
-        "enum34": {
-            "hashes": [
-                "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79",
-                "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a",
-                "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1",
-                "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850"
-            ],
-            "markers": "python_version < '3.4'",
-            "version": "==1.1.6"
-        },
-        "isort": {
-            "hashes": [
-                "sha256:cd5d3fc2c16006b567a17193edf4ed9830d9454cbeb5a42ac80b36ea00c23db4",
-                "sha256:79f46172d3a4e2e53e7016e663cc7a8b538bec525c36675fcfd2767df30b3983"
-            ],
-            "version": "==4.2.15"
-        },
-        "lazy-object-proxy": {
-            "hashes": [
-                "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019",
-                "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39",
-                "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c",
-                "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e",
-                "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b",
-                "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6",
-                "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b",
-                "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d",
-                "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff",
-                "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd",
-                "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f",
-                "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514",
-                "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92",
-                "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35",
-                "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff",
-                "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252",
-                "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7",
-                "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b",
-                "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f",
-                "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4",
-                "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577",
-                "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d",
-                "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109",
-                "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2",
-                "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33",
-                "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d",
-                "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5",
-                "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088",
-                "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a"
-            ],
-            "version": "==1.3.1"
-        },
-        "mccabe": {
-            "hashes": [
-                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
-                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
-            ],
-            "version": "==0.6.1"
-        },
-        "py": {
-            "hashes": [
-                "sha256:2ccb79b01769d99115aa600d7eed99f524bf752bba8f041dc1c184853514655a",
-                "sha256:0f2d585d22050e90c7d293b6451c83db097df77871974d90efd5a30dc12fcde3"
-            ],
-            "version": "==1.4.34"
-        },
-        "pylint": {
-            "hashes": [
-                "sha256:948679535a28afc54afb9210dabc6973305409042ece8e5768ca1409910c1ed8",
-                "sha256:1f65b3815c3bf7524b845711d54c4242e4057dd93826586620239ecdfe591fb1"
-            ],
-            "version": "==1.7.4"
-        },
-        "pytest": {
-            "hashes": [
-                "sha256:81a25f36a97da3313e1125fce9e7bbbba565bc7fec3c5beb14c262ddab238ac1",
-                "sha256:27fa6617efc2869d3e969a3e75ec060375bfb28831ade8b5cdd68da3a741dc3c"
-            ],
-            "version": "==3.2.3"
-        },
-        "singledispatch": {
-            "hashes": [
-                "sha256:833b46966687b3de7f438c761ac475213e53b306740f1abfaa86e1d1aae56aa8",
-                "sha256:5b06af87df13818d14f08a028e42f566640aef80805c3b50c5056b086e3c2b9c"
-            ],
-            "markers": "python_version < '3.4'",
-            "version": "==3.4.0.3"
-        },
-        "six": {
-            "hashes": [
-                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb",
-                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9"
-            ],
-            "version": "==1.11.0"
-        },
-        "wrapt": {
-            "hashes": [
-                "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6"
-            ],
-            "version": "==1.10.11"
-        }
-    }
-}
diff --git a/urlnormalizer/normalizer.py b/urlnormalizer/normalizer.py
index 41a0c59..d47efd3 100644
--- a/urlnormalizer/normalizer.py
+++ b/urlnormalizer/normalizer.py
@@ -75,7 +75,7 @@ def _normalize_path(path):
     # Use `os.path.normpath` to normalize paths i.e. remove duplicate `/` and
     # make the path absolute when `..` or `.` segments are present.
     # TODO: Should we remove duplicate slashes?
-    # TODO: See https://webmasters.stackexchange.com/questions/8354/what-does-the-double-slash-mean-in-urls/8381#8381
+    # TODO: See https://webmasters.stackexchange.com/questions/8354/what-does-the-double-slash-mean-in-urls/8381#8381  # noqa
     path = normpath(path)
     # normpath strips trailing slash. Add it back if it was there because
     # this might make a difference for URLs.
diff --git a/urlnormalizer/utils.py b/urlnormalizer/utils.py
index 676886e..53b856f 100644
--- a/urlnormalizer/utils.py
+++ b/urlnormalizer/utils.py
@@ -46,11 +46,11 @@ def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False):
     """Modify `urllib.parse.parse_qsl` to handle percent-encoded characters
     properly. `parse_qsl` replaces percent-encoded characters with
     replacement character (U+FFFD) (if errors = "replace") or drops them (if 
-    errors = "ignore") (See https://docs.python.org/3/howto/unicode.html#the-string-type).
+    errors = "ignore") (See https://docs.python.org/3/howto/unicode.html#the-string-type).  # noqa
     Instead we want to keep the raw bytes. And later we can percent-encode them
     directly when we need to.
 
-    Code from https://github.com/python/cpython/blob/73c4708630f99b94c35476529748629fff1fc63e/Lib/urllib/parse.py#L658
+    Code from https://github.com/python/cpython/blob/73c4708630f99b94c35476529748629fff1fc63e/Lib/urllib/parse.py#L658  # noqa
     with `unquote` replaced with `unquote_to_bytes`
     """
     qs, _coerce_result = _coerce_args(qs)
@@ -99,7 +99,7 @@ def _unquote(text):
         text = text.decode(_enc)
     except UnicodeDecodeError:
         encoding = detect(text).get('encoding', _enc_fallback)
-        text = text.decode(encoding)
+        text = text.decode(encoding, 'ignore')
     return text