diff --git a/.gitignore b/.gitignore index e814510..899ce90 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ __pycache__/ # Distribution / packaging .Python +.vscode/ +.pytest_cache/ build/ develop-eggs/ dist/ @@ -23,6 +25,7 @@ wheels/ *.egg-info/ .installed.cfg *.egg +.vscode/ MANIFEST # PyInstaller diff --git a/.travis.yml b/.travis.yml index 08a6216..60743eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: - - "3.3" + - "2.7" - "3.6" script: - "python setup.py test" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..40c48b9 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +all: clean test dists + +test: + python setup.py test + +dists: + python setup.py sdist bdist_wheel + +release: clean dists + twine upload dist/* + +clean: + rm -rf dist build .eggs + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + \ No newline at end of file diff --git a/Pipfile b/Pipfile deleted file mode 100644 index ecb0d1c..0000000 --- a/Pipfile +++ /dev/null @@ -1,20 +0,0 @@ -[[source]] - -url = "https://pypi.python.org/simple" -verify_ssl = true -name = "pypi" - - -[dev-packages] - -pytest = "*" -pylint = "*" - - -[packages] - - - -[requires] - -python_version = "3.6" \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index e6ac80a..0000000 --- a/Pipfile.lock +++ /dev/null @@ -1,156 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "1bd673160561b20e61306edf380454208fae2d1949d4aa10f53d1029a124acba" - }, - "host-environment-markers": { - "implementation_name": "cpython", - "implementation_version": "3.6.0", - "os_name": "posix", - "platform_machine": "x86_64", - "platform_python_implementation": "CPython", - "platform_release": "16.7.0", - "platform_system": "Darwin", - "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64", - "python_full_version": "3.6.0", - "python_version": "3.6", - "sys_platform": "darwin" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.6" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.python.org/simple", - "verify_ssl": true - } - ] - }, - "default": {}, - "develop": { - "astroid": { - "hashes": [ - "sha256:39a21dd2b5d81a6731dc0ac2884fa419532dffd465cdd43ea6c168d36b76efb3", - "sha256:492c2a2044adbf6a84a671b7522e9295ad2f6a7c781b899014308db25312dd35" - ], - "version": "==1.5.3" - }, - "backports.functools-lru-cache": { - "hashes": [ - "sha256:4ba998e881f285c1d1b73f5b6e3766539b4e162320f9589334400c5ddc35198c", - "sha256:31f235852f88edc1558d428d890663c49eb4514ffec9f3650e7f3c9e4a12e36f" - ], - "markers": "python_version == '2.7'", - "version": "==1.4" - }, - "configparser": { - "hashes": [ - "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" - ], - "markers": "python_version == '2.7'", - "version": "==3.5.0" - }, - "enum34": { - "hashes": [ - "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", - "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", - "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1", - "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850" - ], - "markers": "python_version < '3.4'", - "version": "==1.1.6" - }, - "isort": { - "hashes": [ - "sha256:cd5d3fc2c16006b567a17193edf4ed9830d9454cbeb5a42ac80b36ea00c23db4", - "sha256:79f46172d3a4e2e53e7016e663cc7a8b538bec525c36675fcfd2767df30b3983" - ], - "version": "==4.2.15" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019", - "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39", - "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c", - "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e", - "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b", - "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6", - "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b", - "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d", - "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff", - "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd", - "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f", - "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514", - "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92", - "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35", - "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff", - "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252", - "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7", - "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b", - "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f", - "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4", - "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577", - "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d", - "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109", - "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2", - "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33", - "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d", - "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5", - "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088", - "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a" - ], - "version": "==1.3.1" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "py": { - "hashes": [ - "sha256:2ccb79b01769d99115aa600d7eed99f524bf752bba8f041dc1c184853514655a", - "sha256:0f2d585d22050e90c7d293b6451c83db097df77871974d90efd5a30dc12fcde3" - ], - "version": "==1.4.34" - }, - "pylint": { - "hashes": [ - "sha256:948679535a28afc54afb9210dabc6973305409042ece8e5768ca1409910c1ed8", - "sha256:1f65b3815c3bf7524b845711d54c4242e4057dd93826586620239ecdfe591fb1" - ], - "version": "==1.7.4" - }, - "pytest": { - "hashes": [ - "sha256:81a25f36a97da3313e1125fce9e7bbbba565bc7fec3c5beb14c262ddab238ac1", - "sha256:27fa6617efc2869d3e969a3e75ec060375bfb28831ade8b5cdd68da3a741dc3c" - ], - "version": "==3.2.3" - }, - "singledispatch": { - "hashes": [ - "sha256:833b46966687b3de7f438c761ac475213e53b306740f1abfaa86e1d1aae56aa8", - "sha256:5b06af87df13818d14f08a028e42f566640aef80805c3b50c5056b086e3c2b9c" - ], - "markers": "python_version < '3.4'", - "version": "==3.4.0.3" - }, - "six": { - "hashes": [ - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" - ], - "version": "==1.11.0" - }, - "wrapt": { - "hashes": [ - "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6" - ], - "version": "==1.10.11" - } - } -} diff --git a/README.md b/README.md index 844279b..8580f0d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ -URL-NORMALIZER -============== +# urlnormalizer -[![Build Status](https://travis-ci.org/sunu/url-normalizer.svg?branch=master)](https://travis-ci.org/sunu/url-normalizer) +[![Build Status](https://travis-ci.org/alephdata/urlnormalizer.svg?branch=master)](https://travis-ci.org/alephdata/urlnormalizer) Normalizes URL by doing the following: @@ -21,25 +20,21 @@ Normalizes URL by doing the following: Works with `http` and `https` urls only for now. -# Python version - -For now, Python 3 only. - -# Installation +## Installation Install using `pip` ```console -$ pip install git+https://github.com/sunu/url-normalizer +$ pip install urlnormalizer ``` or clone and install using `python setup.py install` -# Usage +## Usage Pass a url to the `normalize_url` function as a `str` type to normalize it. ```pycon -In [1]: from normalizer import normalize_url +In [1]: from urlnormalizer import normalize_url In [2]: normalize_url("hello.com") Out[2]: 'http://hello.com/' @@ -109,10 +104,10 @@ In [18]: repr(normalize_url(1234)) Out[18]: 'None' ``` -# Tests +## Tests Run tests by using `python setup.py test` -# License +## License MIT \ No newline at end of file diff --git a/normalizer/__init__.py b/normalizer/__init__.py deleted file mode 100644 index 0b6a190..0000000 --- a/normalizer/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .normalizer import normalize_url diff --git a/normalizer/tests/test_normalizer.py b/normalizer/tests/test_normalizer.py deleted file mode 100644 index d3289e1..0000000 --- a/normalizer/tests/test_normalizer.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Tests for URL normalization""" -import pytest - -from ..normalizer import normalize_url - -# TODO: parametrize test cases - -def test_normalized_urls(): - """Already normalized URLs should not change""" - assert normalize_url("http://example.com/") == "http://example.com/" - -def test_return_type(): - """Should return string""" - assert isinstance(normalize_url("http://example.com/"), str) - -def test_append_slash(): - """Append a slash to the end of the URL if it's missing one""" - assert normalize_url("http://example.com") == "http://example.com/" - -def test_lower_case(): - """Normalized URL scheme and host are lower case""" - assert normalize_url("HTTP://examPle.cOm/") == "http://example.com/" - assert normalize_url("http://example.com/A") == "http://example.com/A" - -def test_strip_trailing_period(): - assert normalize_url("http://example.com.") == "http://example.com/" - assert normalize_url("http://example.com./") == "http://example.com/" - -def test_capitalize_escape_sequence(): - """All letters in percent-encoded triplets should be capitalized""" - assert (normalize_url("http://www.example.com/a%c2%b1b") == - "http://www.example.com/a%C2%B1b") - -def test_path_percent_encoding(): - """All non-safe characters should be percent-encoded""" - assert (normalize_url("http://example.com/hello world{}") == - "http://example.com/hello%20world%7B%7D") - -def test_unreserved_percentencoding(): - """Unreserved characters should not be percent encoded. If they are, they - should be decoded back; except in case of `/`, `?` and `#`""" - assert (normalize_url("http://www.example.com/%7Eusername/") == - "http://www.example.com/~username") - assert (normalize_url('http://example.com/foo%23bar') == - 'http://example.com/foo%23bar') - assert (normalize_url('http://example.com/foo%2fbar') == - 'http://example.com/foo%2Fbar') - assert (normalize_url('http://example.com/foo%3fbar') == - 'http://example.com/foo%3Fbar') - -def test_remove_dot_segments(): - """Convert the URL path to an absolute path by removing `.` and `..` - segments""" - assert (normalize_url("http://www.example.com/../a/b/../c/./d.html") == - "http://www.example.com/a/c/d.html") - -def test_remove_default_port(): - """Remove the default port for the scheme if it's present in the URL""" - assert (normalize_url("http://www.example.com:80/bar.html") == - "http://www.example.com/bar.html") - assert (normalize_url("HTTPS://example.com:443/abc/") == - "https://example.com/abc") - -def test_remove_empty_port(): - """Remove empty port from URL""" - assert (normalize_url("http://www.example.com:/") == - "http://www.example.com/") - -def test_remove_extra_slash(): - """Remove any extra slashes if present in the URl""" - # TODO: Should we actually do this? - # TODO: See https://webmasters.stackexchange.com/questions/8354/what-does-the-double-slash-mean-in-urls/8381#8381 - assert (normalize_url("http://www.example.com/foo//bar.html") == - "http://www.example.com/foo/bar.html") - assert(normalize_url("http://example.com///abc") == - "http://example.com/abc") - -def test_query_string(): - """Query strings should be handled properly""" - assert (normalize_url("http://example.com/?a=1") == - "http://example.com/?a=1") - assert (normalize_url("http://example.com?a=1") == - "http://example.com/?a=1") - assert (normalize_url("http://example.com/a?b=1") == - "http://example.com/a?b=1") - assert (normalize_url("http://example.com/a/?b=1") == - "http://example.com/a?b=1") - -def test_dont_percent_encode_safe_chars_query(): - """Don't percent-encode safe characters in querystring""" - assert (normalize_url("http://example.com/a/?face=(-.-)") == - "http://example.com/a?face=(-.-)") - -def test_query_sorting(): - """Query strings should be sorted""" - assert (normalize_url('http://example.com/a?b=1&c=2') == - 'http://example.com/a?b=1&c=2') - assert (normalize_url('http://example.com/a?c=2&b=1') == - 'http://example.com/a?b=1&c=2') - -def test_query_string_spaces(): - """Spaces should be handled properly in query strings""" - assert (normalize_url("http://example.com/search?q=a b&a=1") == - "http://example.com/search?a=1&q=a+b") - assert (normalize_url("http://example.com/search?q=a+b&a=1") == - "http://example.com/search?a=1&q=a+b") - assert (normalize_url("http://example.com/search?q=a%20b&a=1") == - "http://example.com/search?a=1&q=a+b") - -def test_drop_trailing_questionmark(): - """Drop the trailing question mark if no query string present""" - assert normalize_url("http://example.com/?") == "http://example.com/" - assert normalize_url("http://example.com?") == "http://example.com/" - assert normalize_url("http://example.com/a?") == "http://example.com/a" - assert normalize_url("http://example.com/a/?") == "http://example.com/a" - -def test_percent_encode_querystring(): - """Non-safe characters in query string should be percent-encoded""" - assert (normalize_url("http://example.com/?a=hello{}") == - "http://example.com/?a=hello%7B%7D") - -def test_normalize_percent_encoding_in_querystring(): - """Percent-encoded querystring should be uppercased""" - assert (normalize_url("http://example.com/?a=b%c2") == - "http://example.com/?a=b%C2") - -def test_unicode_query_string(): - """Unicode query strings should be converted to bytes using uft-8 encoding - and then properly percent-encoded""" - assert (normalize_url("http://example.com/?file=résumé.pdf") == - "http://example.com/?file=r%C3%A9sum%C3%A9.pdf") - -def test_unicode_path(): - """Unicode path should be converted to bytes using utf-8 encoding and then - percent-encoded""" - assert (normalize_url("http://example.com/résumé") == - "http://example.com/r%C3%A9sum%C3%A9") - -def test_idna(): - """International Domain Names should be normalized to safe characters""" - assert (normalize_url("http://ドメイン.テスト") == - "http://xn--eckwd4c7c.xn--zckzah/") - assert (normalize_url("http://Яндекс.рф") == - "http://xn--d1acpjx3f.xn--p1ai/") - -def test_dont_change_username_password(): - """Username and password shouldn't be lowercased""" - assert (normalize_url("http://Foo:BAR@exaMPLE.COM/") == - "http://Foo:BAR@example.com/") - -def test_normalize_ipv4(): - """Normalize ipv4 URLs""" - assert normalize_url("http://192.168.0.1/") == "http://192.168.0.1/" - assert (normalize_url("http://192.168.0.1:8080/a?b=1") == - "http://192.168.0.1:8080/a?b=1") - assert normalize_url("192.168.0.1") == "http://192.168.0.1/" - assert (normalize_url("192.168.0.1:8080/a/b/c") == - "http://192.168.0.1:8080/a/b/c") - -def test_normalize_ipv6(): - """Normalize ipv6 URLs""" - assert normalize_url("[::1]") == "http://[::1]/" - assert normalize_url("http://[::1]") == "http://[::1]/" - assert normalize_url("[::1]:8080") == "http://[::1]:8080/" - assert normalize_url("http://[::1]:8080") == "http://[::1]:8080/" - -def test_strip_leading_trailing_whitespace(): - """Strip leading and trailing whitespace if any""" - assert normalize_url(" http://example.com ") == "http://example.com/" - assert normalize_url("http://example.com/a ") == "http://example.com/a" - assert normalize_url(" http://example.com/") == "http://example.com/" - -def test_non_ideal_inputs(): - """Not the ideal input; but we should handle it anyway""" - assert normalize_url("example.com") == "http://example.com/" - assert normalize_url("example.com/abc") == "http://example.com/abc" - assert normalize_url("//example.com/abc") == "http://example.com/abc" - -def test_additional_query_args(): - """Add any additional query arguments to the URL""" - assert (normalize_url("http://example.com?c=d", [("a", "b")]) == - "http://example.com/?a=b&c=d") - assert (normalize_url("http://example.com", [("a", "b")]) == - "http://example.com/?a=b") - assert (normalize_url("http://example.com", [("résumé", "résumé")]) == - "http://example.com/?r%C3%A9sum%C3%A9=r%C3%A9sum%C3%A9") - -def test_non_urls(): - """If a non-URL string is passed, return None""" - assert normalize_url("") is None - assert normalize_url("abc xyz") is None - assert normalize_url("asb#abc") is None - assert normalize_url("Яндекс.рф") is not None - assert normalize_url("google.blog") is not None - assert normalize_url("http//google.com") is None - assert normalize_url("http://user@pass:example.com") is None - -def test_drop_fragments(): - """Drop or keep fragments based on the option passed""" - assert (normalize_url("http://example.com/a?b=1#frag") - == "http://example.com/a?b=1") - assert (normalize_url("http://example.com/a?b=1#frag", drop_fragments=False) - == "http://example.com/a?b=1#frag") - -def test_non_string_input(): - """Non-string input should produce None as result""" - assert normalize_url(None) is None - assert normalize_url([]) is None - assert normalize_url(123) is None diff --git a/normalizer/utils.py b/normalizer/utils.py deleted file mode 100644 index ab9bd53..0000000 --- a/normalizer/utils.py +++ /dev/null @@ -1,77 +0,0 @@ -import re -from urllib.parse import _coerce_args, unquote_to_bytes - - -def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False): - """Modify `urllib.parse.parse_qsl` to handle percent-encoded characters - properly. `parse_qsl` replaces percent-encoded characters with - replacement character (U+FFFD) (if errors = "replace") or drops them (if - errors = "ignore") (See https://docs.python.org/3/howto/unicode.html#the-string-type). - Instead we want to keep the raw bytes. And later we can percent-encode them - directly when we need to. - - Code from https://github.com/python/cpython/blob/73c4708630f99b94c35476529748629fff1fc63e/Lib/urllib/parse.py#L658 - with `unquote` replaced with `unquote_to_bytes` - """ - qs, _coerce_result = _coerce_args(qs) - pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] - r = [] - for name_value in pairs: - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: - raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = nv[0].replace('+', ' ') - name = unquote_to_bytes(name) - name = _coerce_result(name) - value = nv[1].replace('+', ' ') - value = unquote_to_bytes(value) - value = _coerce_result(value) - r.append((name, value)) - return r - - -def _is_valid_url(value): - """ - Does the value look like a URL? - From https://github.com/django/django/blob/stable/2.0.x/django/core/validators.py - """ - if value.startswith("//"): - value = value[2:] - ul = '\u00a1-\uffff' # unicode letters range (must not be a raw string) - - # IP patterns - ipv4_re = r'(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}' - ipv6_re = r'\[[0-9a-f:\.]+\]' # (simple regex, validated later) - - # Host patterns - hostname_re = r'[a-z' + ul + r'0-9](?:[a-z' + ul + r'0-9-]{0,61}[a-z' + ul + r'0-9])?' - # Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1 - domain_re = r'(?:\.(?!-)[a-z' + ul + r'0-9-]{1,63}(?