arno/isso
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

use SSL for https://

Browse Source
This commit is contained in:
Martin Zimmermann 2013-09-13 18:08:15 +02:00
parent 9d7c049b32
commit b744e2fe39
1 changed files with 9 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
isso/utils.py
View File

@ -33,7 +33,11 @@ class IssoEncoder(json.JSONEncoder):
def urlexists(host, path): def urlexists(host, path):
with closing(httplib.HTTPConnection(normalize(host), timeout=3)) as con:
rv = urlparse(host)
http = httplib.HTTPSConnection if rv.scheme == 'https' else httplib.HTTPConnection
with closing(http(rv.netloc, rv.port, timeout=3)) as con:
try: try:
con.request('HEAD', path) con.request('HEAD', path)
except (httplib.HTTPException, socket.error): except (httplib.HTTPException, socket.error):
@ -45,7 +49,10 @@ def heading(host, path):
"""Connect to `host`, GET path and start from #isso-thread to search for """Connect to `host`, GET path and start from #isso-thread to search for
a possible heading (h1). Returns `None` if nothing found.""" a possible heading (h1). Returns `None` if nothing found."""
with closing(httplib.HTTPConnection(normalize(host), timeout=15)) as con: rv = urlparse(host)
http = httplib.HTTPSConnection if rv.scheme == 'https' else httplib.HTTPConnection
with closing(http(rv.netloc, rv.port, timeout=15)) as con:
con.request('GET', path) con.request('GET', path)
html = html5lib.parse(con.getresponse().read(), treebuilder="dom") html = html5lib.parse(con.getresponse().read(), treebuilder="dom")
@ -92,15 +99,6 @@ def heading(host, path):
return None return None
def normalize(host):
"""Make `host` compatible with :py:mod:`httplib`."""
if not host.startswith(('http://', 'https://')):
host = 'http://' + host
rv = urlparse(host)
return (rv.netloc + ':443') if rv.scheme == 'https' else rv.netloc
def anonymize(remote_addr): def anonymize(remote_addr):
ip = ipaddress.ip_address(remote_addr) ip = ipaddress.ip_address(remote_addr)
if ip.version == "4": if ip.version == "4":