Blame | Last modification | View Log | Download
# Written by John Hoffman# see LICENSE.txt for license informationfrom httplib import HTTPConnection, HTTPSConnection, HTTPExceptionfrom urlparse import urlparsefrom bencode import bdecodeimport socketfrom gzip import GzipFilefrom StringIO import StringIOfrom urllib import quote, unquotefrom __init__ import product_name, version_shortVERSION = product_name+'/'+version_shortMAX_REDIRECTS = 10class btHTTPcon(HTTPConnection): # attempt to add automatic connection timeoutdef connect(self):HTTPConnection.connect(self)try:self.sock.settimeout(30)except:passclass btHTTPScon(HTTPSConnection): # attempt to add automatic connection timeoutdef connect(self):HTTPSConnection.connect(self)try:self.sock.settimeout(30)except:passclass urlopen:def __init__(self, url):self.tries = 0self._open(url.strip())self.error_return = Nonedef _open(self, url):self.tries += 1if self.tries > MAX_REDIRECTS:raise IOError, ('http error', 500,"Internal Server Error: Redirect Recursion")(scheme, netloc, path, pars, query, fragment) = urlparse(url)if scheme != 'http' and scheme != 'https':raise IOError, ('url error', 'unknown url type', scheme, url)url = pathif pars:url += ';'+parsif query:url += '?'+query# if fragment:try:if scheme == 'http':self.connection = btHTTPcon(netloc)else:self.connection = btHTTPScon(netloc)self.connection.request('GET', url, None,{ 'User-Agent': VERSION,'Accept-Encoding': 'gzip' } )self.response = self.connection.getresponse()except HTTPException, e:raise IOError, ('http error', str(e))status = self.response.statusif status in (301,302):try:self.connection.close()except:passself._open(self.response.getheader('Location'))returnif status != 200:try:data = self._read()d = bdecode(data)if d.has_key('failure reason'):self.error_return = datareturnexcept:passraise IOError, ('http error', status, self.response.reason)def read(self):if self.error_return:return self.error_returnreturn self._read()def _read(self):data = self.response.read()if self.response.getheader('Content-Encoding','').find('gzip') >= 0:try:compressed = StringIO(data)f = GzipFile(fileobj = compressed)data = f.read()except:raise IOError, ('http error', 'got corrupt response')return datadef close(self):self.connection.close()