diff options
Diffstat (limited to 'lib/glider/download.py')
-rw-r--r-- | lib/glider/download.py | 127 |
1 files changed, 0 insertions, 127 deletions
diff --git a/lib/glider/download.py b/lib/glider/download.py deleted file mode 100644 index 3135e39..0000000 --- a/lib/glider/download.py +++ /dev/null @@ -1,127 +0,0 @@ - - -import urllib2 -import httplib -import random - -import threading, Queue - -import glider.util - -class Downloads: - def __init__(self, n_threads=2): - self._lock = threading.RLock() - self.downloads = {} - self.haveDownloaded = {} - self.downloadQueue = Queue.Queue() - self.threads = [ threading.Thread(target=self._thread) ] - for t in self.threads: - t.setDaemon(True) - - def start(self): - for t in self.threads: - t.start() - - def isCurrentlyDownloading(self, relPath): - self._lock.acquire() - try: - return self.downloads.has_key(relPath) - finally: - self._lock.release() - - def isRedundant(self, relPath): - self._lock.acquire() - try: - return (self.downloads.has_key(relPath) or - self.haveDownloaded.has_key(relPath)) - finally: - self._lock.release() - - def addDownloadJob(self, job): - rp = job.getRelativePath() - self._lock.acquire() - self.downloads[rp] = job - self._lock.release() - self.downloadQueue.put(job) - - def _thread(self): - while True: - job = self.downloadQueue.get() - job.download() - rp = job.getRelativePath() - self._lock.acquire() - try: - del self.downloads[rp] - self.haveDownloaded[rp] = True - finally: - self._lock.release() - -class DownloadJob: - def __init__(self, relPath, destPath, mirrorlist=None, - wantHash=None, canStall=False): - self._relPath = relPath - self._wantHash = wantHash - self._mirrorList = mirrorlist - self._destPath = destPath - - tmppath = glider.util.userFilename("tmp") - if relPath.startswith("/"): - relPath = relPath[1:] - self._tmppath = os.path.join(tmppath, relPath) - - d = os.path.dirname(self._tmppath) - if not os.path.exists(d): - os.makedirs(d, 0700) - - def getRelativePath(self): - return self._relPath - - def haveStalledFile(self): - return os.path.exists(self._tmppath) - - def getURL(self, mirrorlist=None): - if mirrorlist is None: - mirrorlist = self._mirrorList - weightSoFar = 0 - usable = [] - - for m in mirrorlist['mirrors']: - for c in m['contents']: - # CHECK FOR URL SUITABILITY XXXXX - - if glider.formats.rolePathMatches(c, self._relPath): - weightSoFar += m['weight'] - usable.append( (weightSoFar, m) ) - break - - wTarget = random.randint(0, weightSoFar) - mirror = None - # Could use bisect here instead - for w, m in mirrorlist: - if w >= wTarget: - mirror = m - break - - return m['urlbase'] + self._relPath - - def download(self): - # XXXX RESUME - - f_in = urllib2.urlopen(self.getURL()) - f_out = open(self._tmpPath, 'w') - while True: - c = f_in.read(1024) - if not c: - break - f_out.write(c) - f_in.close() - f_out.close() - # XXXXX retry on failure - - if self._wantHash: - gotHash = glider.formats.getFileDigest(self._tmpPath) - if gotHash != self._wantHash: - # XXXX Corrupt file. - pass - - glider.utils.moveFile(self._tmpPath, self._destPath) |