| 36 |
kaklik |
1 |
# Written by John Hoffman
|
|
|
2 |
# see LICENSE.txt for license information
|
|
|
3 |
|
|
|
4 |
from BitTornado.CurrentRateMeasure import Measure
|
|
|
5 |
from random import randint
|
|
|
6 |
from urlparse import urlparse
|
|
|
7 |
from httplib import HTTPConnection
|
|
|
8 |
from urllib import quote
|
|
|
9 |
from threading import Thread
|
|
|
10 |
from BitTornado.__init__ import product_name,version_short
|
|
|
11 |
try:
|
|
|
12 |
True
|
|
|
13 |
except:
|
|
|
14 |
True = 1
|
|
|
15 |
False = 0
|
|
|
16 |
|
|
|
17 |
EXPIRE_TIME = 60 * 60
|
|
|
18 |
|
|
|
19 |
VERSION = product_name+'/'+version_short
|
|
|
20 |
|
|
|
21 |
class haveComplete:
|
|
|
22 |
def complete(self):
|
|
|
23 |
return True
|
|
|
24 |
def __getitem__(self, x):
|
|
|
25 |
return True
|
|
|
26 |
haveall = haveComplete()
|
|
|
27 |
|
|
|
28 |
class SingleDownload:
|
|
|
29 |
def __init__(self, downloader, url):
|
|
|
30 |
self.downloader = downloader
|
|
|
31 |
self.baseurl = url
|
|
|
32 |
try:
|
|
|
33 |
(scheme, self.netloc, path, pars, query, fragment) = urlparse(url)
|
|
|
34 |
except:
|
|
|
35 |
self.downloader.errorfunc('cannot parse http seed address: '+url)
|
|
|
36 |
return
|
|
|
37 |
if scheme != 'http':
|
|
|
38 |
self.downloader.errorfunc('http seed url not http: '+url)
|
|
|
39 |
return
|
|
|
40 |
try:
|
|
|
41 |
self.connection = HTTPConnection(self.netloc)
|
|
|
42 |
except:
|
|
|
43 |
self.downloader.errorfunc('cannot connect to http seed: '+url)
|
|
|
44 |
return
|
|
|
45 |
self.seedurl = path
|
|
|
46 |
if pars:
|
|
|
47 |
self.seedurl += ';'+pars
|
|
|
48 |
self.seedurl += '?'
|
|
|
49 |
if query:
|
|
|
50 |
self.seedurl += query+'&'
|
|
|
51 |
self.seedurl += 'info_hash='+quote(self.downloader.infohash)
|
|
|
52 |
|
|
|
53 |
self.measure = Measure(downloader.max_rate_period)
|
|
|
54 |
self.index = None
|
|
|
55 |
self.url = ''
|
|
|
56 |
self.requests = []
|
|
|
57 |
self.request_size = 0
|
|
|
58 |
self.endflag = False
|
|
|
59 |
self.error = None
|
|
|
60 |
self.retry_period = 30
|
|
|
61 |
self._retry_period = None
|
|
|
62 |
self.errorcount = 0
|
|
|
63 |
self.goodseed = False
|
|
|
64 |
self.active = False
|
|
|
65 |
self.cancelled = False
|
|
|
66 |
self.resched(randint(2,10))
|
|
|
67 |
|
|
|
68 |
def resched(self, len = None):
|
|
|
69 |
if len is None:
|
|
|
70 |
len = self.retry_period
|
|
|
71 |
if self.errorcount > 3:
|
|
|
72 |
len = len * (self.errorcount - 2)
|
|
|
73 |
self.downloader.rawserver.add_task(self.download, len)
|
|
|
74 |
|
|
|
75 |
def _want(self, index):
|
|
|
76 |
if self.endflag:
|
|
|
77 |
return self.downloader.storage.do_I_have_requests(index)
|
|
|
78 |
else:
|
|
|
79 |
return self.downloader.storage.is_unstarted(index)
|
|
|
80 |
|
|
|
81 |
def download(self):
|
|
|
82 |
self.cancelled = False
|
|
|
83 |
if self.downloader.picker.am_I_complete():
|
|
|
84 |
self.downloader.downloads.remove(self)
|
|
|
85 |
return
|
|
|
86 |
self.index = self.downloader.picker.next(haveall, self._want)
|
|
|
87 |
if ( self.index is None and not self.endflag
|
|
|
88 |
and not self.downloader.peerdownloader.has_downloaders() ):
|
|
|
89 |
self.endflag = True
|
|
|
90 |
self.index = self.downloader.picker.next(haveall, self._want)
|
|
|
91 |
if self.index is None:
|
|
|
92 |
self.endflag = True
|
|
|
93 |
self.resched()
|
|
|
94 |
else:
|
|
|
95 |
self.url = ( self.seedurl+'&piece='+str(self.index) )
|
|
|
96 |
self._get_requests()
|
|
|
97 |
if self.request_size < self.downloader.storage._piecelen(self.index):
|
|
|
98 |
self.url += '&ranges='+self._request_ranges()
|
|
|
99 |
rq = Thread(target = self._request)
|
|
|
100 |
rq.setDaemon(False)
|
|
|
101 |
rq.start()
|
|
|
102 |
self.active = True
|
|
|
103 |
|
|
|
104 |
def _request(self):
|
|
|
105 |
import encodings.ascii
|
|
|
106 |
import encodings.punycode
|
|
|
107 |
import encodings.idna
|
|
|
108 |
|
|
|
109 |
self.error = None
|
|
|
110 |
self.received_data = None
|
|
|
111 |
try:
|
|
|
112 |
self.connection.request('GET',self.url, None,
|
|
|
113 |
{'User-Agent': VERSION})
|
|
|
114 |
r = self.connection.getresponse()
|
|
|
115 |
self.connection_status = r.status
|
|
|
116 |
self.received_data = r.read()
|
|
|
117 |
except Exception, e:
|
|
|
118 |
self.error = 'error accessing http seed: '+str(e)
|
|
|
119 |
try:
|
|
|
120 |
self.connection.close()
|
|
|
121 |
except:
|
|
|
122 |
pass
|
|
|
123 |
try:
|
|
|
124 |
self.connection = HTTPConnection(self.netloc)
|
|
|
125 |
except:
|
|
|
126 |
self.connection = None # will cause an exception and retry next cycle
|
|
|
127 |
self.downloader.rawserver.add_task(self.request_finished)
|
|
|
128 |
|
|
|
129 |
def request_finished(self):
|
|
|
130 |
self.active = False
|
|
|
131 |
if self.error is not None:
|
|
|
132 |
if self.goodseed:
|
|
|
133 |
self.downloader.errorfunc(self.error)
|
|
|
134 |
self.errorcount += 1
|
|
|
135 |
if self.received_data:
|
|
|
136 |
self.errorcount = 0
|
|
|
137 |
if not self._got_data():
|
|
|
138 |
self.received_data = None
|
|
|
139 |
if not self.received_data:
|
|
|
140 |
self._release_requests()
|
|
|
141 |
self.downloader.peerdownloader.piece_flunked(self.index)
|
|
|
142 |
if self._retry_period:
|
|
|
143 |
self.resched(self._retry_period)
|
|
|
144 |
self._retry_period = None
|
|
|
145 |
return
|
|
|
146 |
self.resched()
|
|
|
147 |
|
|
|
148 |
def _got_data(self):
|
|
|
149 |
if self.connection_status == 503: # seed is busy
|
|
|
150 |
try:
|
|
|
151 |
self.retry_period = max(int(self.received_data),5)
|
|
|
152 |
except:
|
|
|
153 |
pass
|
|
|
154 |
return False
|
|
|
155 |
if self.connection_status != 200:
|
|
|
156 |
self.errorcount += 1
|
|
|
157 |
return False
|
|
|
158 |
self._retry_period = 1
|
|
|
159 |
if len(self.received_data) != self.request_size:
|
|
|
160 |
if self.goodseed:
|
|
|
161 |
self.downloader.errorfunc('corrupt data from http seed - redownloading')
|
|
|
162 |
return False
|
|
|
163 |
self.measure.update_rate(len(self.received_data))
|
|
|
164 |
self.downloader.measurefunc(len(self.received_data))
|
|
|
165 |
if self.cancelled:
|
|
|
166 |
return False
|
|
|
167 |
if not self._fulfill_requests():
|
|
|
168 |
return False
|
|
|
169 |
if not self.goodseed:
|
|
|
170 |
self.goodseed = True
|
|
|
171 |
self.downloader.seedsfound += 1
|
|
|
172 |
if self.downloader.storage.do_I_have(self.index):
|
|
|
173 |
self.downloader.picker.complete(self.index)
|
|
|
174 |
self.downloader.peerdownloader.check_complete(self.index)
|
|
|
175 |
self.downloader.gotpiecefunc(self.index)
|
|
|
176 |
return True
|
|
|
177 |
|
|
|
178 |
def _get_requests(self):
|
|
|
179 |
self.requests = []
|
|
|
180 |
self.request_size = 0L
|
|
|
181 |
while self.downloader.storage.do_I_have_requests(self.index):
|
|
|
182 |
r = self.downloader.storage.new_request(self.index)
|
|
|
183 |
self.requests.append(r)
|
|
|
184 |
self.request_size += r[1]
|
|
|
185 |
self.requests.sort()
|
|
|
186 |
|
|
|
187 |
def _fulfill_requests(self):
|
|
|
188 |
start = 0L
|
|
|
189 |
success = True
|
|
|
190 |
while self.requests:
|
|
|
191 |
begin, length = self.requests.pop(0)
|
|
|
192 |
if not self.downloader.storage.piece_came_in(self.index, begin,
|
|
|
193 |
self.received_data[start:start+length]):
|
|
|
194 |
success = False
|
|
|
195 |
break
|
|
|
196 |
start += length
|
|
|
197 |
return success
|
|
|
198 |
|
|
|
199 |
def _release_requests(self):
|
|
|
200 |
for begin, length in self.requests:
|
|
|
201 |
self.downloader.storage.request_lost(self.index, begin, length)
|
|
|
202 |
self.requests = []
|
|
|
203 |
|
|
|
204 |
def _request_ranges(self):
|
|
|
205 |
s = ''
|
|
|
206 |
begin, length = self.requests[0]
|
|
|
207 |
for begin1, length1 in self.requests[1:]:
|
|
|
208 |
if begin + length == begin1:
|
|
|
209 |
length += length1
|
|
|
210 |
continue
|
|
|
211 |
else:
|
|
|
212 |
if s:
|
|
|
213 |
s += ','
|
|
|
214 |
s += str(begin)+'-'+str(begin+length-1)
|
|
|
215 |
begin, length = begin1, length1
|
|
|
216 |
if s:
|
|
|
217 |
s += ','
|
|
|
218 |
s += str(begin)+'-'+str(begin+length-1)
|
|
|
219 |
return s
|
|
|
220 |
|
|
|
221 |
|
|
|
222 |
class HTTPDownloader:
|
|
|
223 |
def __init__(self, storage, picker, rawserver,
|
|
|
224 |
finflag, errorfunc, peerdownloader,
|
|
|
225 |
max_rate_period, infohash, measurefunc, gotpiecefunc):
|
|
|
226 |
self.storage = storage
|
|
|
227 |
self.picker = picker
|
|
|
228 |
self.rawserver = rawserver
|
|
|
229 |
self.finflag = finflag
|
|
|
230 |
self.errorfunc = errorfunc
|
|
|
231 |
self.peerdownloader = peerdownloader
|
|
|
232 |
self.infohash = infohash
|
|
|
233 |
self.max_rate_period = max_rate_period
|
|
|
234 |
self.gotpiecefunc = gotpiecefunc
|
|
|
235 |
self.measurefunc = measurefunc
|
|
|
236 |
self.downloads = []
|
|
|
237 |
self.seedsfound = 0
|
|
|
238 |
|
|
|
239 |
def make_download(self, url):
|
|
|
240 |
self.downloads.append(SingleDownload(self, url))
|
|
|
241 |
return self.downloads[-1]
|
|
|
242 |
|
|
|
243 |
def get_downloads(self):
|
|
|
244 |
if self.finflag.isSet():
|
|
|
245 |
return []
|
|
|
246 |
return self.downloads
|
|
|
247 |
|
|
|
248 |
def cancel_piece_download(self, pieces):
|
|
|
249 |
for d in self.downloads:
|
|
|
250 |
if d.active and d.index in pieces:
|
|
|
251 |
d.cancelled = True
|