diff options
| -rw-r--r-- | doc/interface.txt | 6 | ||||
| -rw-r--r-- | lib/thandy/ClientCLI.py | 49 | ||||
| -rw-r--r-- | lib/thandy/bt_compat.py | 56 | ||||
| -rw-r--r-- | lib/thandy/download.py | 47 | ||||
| -rw-r--r-- | lib/thandy/repository.py | 50 |
5 files changed, 188 insertions, 20 deletions
diff --git a/doc/interface.txt b/doc/interface.txt index 22a35ed..b50442b 100644 --- a/doc/interface.txt +++ b/doc/interface.txt @@ -39,6 +39,12 @@ Recognized options for thandy-client update are: Output log messages in a format designed to be easy for controllers to read. (See below.) + --download-method=<method> + Choose this download method. Can be used in conjunction + with the socks-port option. Supported methods are: + - direct: downloads the file directly + - bittorrent: use bittorrent for downloading + THE CONTROLLER LOG FORMAT: diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 73c96df..bcbdfc7 100644 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -68,7 +68,8 @@ def update(args): options, args = getopt.getopt(args, "", [ "repo=", "no-download", "loop", "no-packagesys", "install", "socks-port=", "debug", "info", - "warn", "force-check", "controller-log-format" + "warn", "force-check", "controller-log-format", + "download-method=" ]) download = True keep_looping = False @@ -76,6 +77,7 @@ def update(args): install = False socksPort = None forceCheck = False + downloadMethod = "direct" for o, v in options: if o == '--repo': @@ -92,12 +94,20 @@ def update(args): socksPort = int(v) elif o == '--force-check': forceCheck = True + elif o == '--download-method': + downloadMethod = v configureLogs(options) if socksPort: thandy.socksurls.setSocksProxy("127.0.0.1", socksPort) + if downloadMethod == "bittorrent": + thandy.bt_compat.BtCompat.setUseBt(True) + elif downloadMethod != "direct": + usage() + sys.exit() + repo = thandy.repository.LocalRepository(repoRoot) downloader = thandy.download.DownloadManager() downloader.start() @@ -109,11 +119,15 @@ def update(args): hashes = {} lengths = {} installable = {} + btMetadata = {} logging.info("Checking for files to update.") - files = repo.getFilesToUpdate(trackingBundles=args, hashDict=hashes, - lengthDict=lengths, - usePackageSystem=use_packagesys, - installableDict=installable) + files, downloadingFiles = repo.getFilesToUpdate( + trackingBundles=args, + hashDict=hashes, + lengthDict=lengths, + usePackageSystem=use_packagesys, + installableDict=installable, + btMetadataDict=btMetadata) if forceCheck: files.add("/meta/timestamp.txt") @@ -183,13 +197,23 @@ def update(args): logging.info("Waiting a while before we fetch %s", f) continue - dj = thandy.download.ThandyDownloadJob( - f, repo.getFilename(f), - mirrorlist, - wantHash=hashes.get(f), - wantLength=lengths.get(f), - repoFile=repo.getRequestedFile(f), - useTor=(socksPort!=None)) + dj = None + if thandy.bt_compat.BtCompat.shouldUseBt() and downloadingFiles: + dj = thandy.download.ThandyBittorrentDownloadJob( + repo.getFilename(btMetadata[f]), f, + repo.getFilename(f), + wantHash=hashes.get(f), + wantLength=lengths.get(f), + repoFile=repo.getRequestedFile(f)) + + else: + dj = thandy.download.ThandyDownloadJob( + f, repo.getFilename(f), + mirrorlist, + wantHash=hashes.get(f), + wantLength=lengths.get(f), + repoFile=repo.getRequestedFile(f), + useTor=(socksPort!=None)) def successCb(rp=f): rf = repo.getRequestedFile(rp) @@ -219,6 +243,7 @@ def usage(): print " [--no-packagesys] [--install] [--socks-port=port]" print " [--debug|--info|--warn] [--force-check]" print " [--controller-log-format]" + print " [--download-method=direct|bittorrent]" print " bundle1, bundle2, ..." print " json2xml file" sys.exit(1) diff --git a/lib/thandy/bt_compat.py b/lib/thandy/bt_compat.py index 43cecad..e9298a6 100644 --- a/lib/thandy/bt_compat.py +++ b/lib/thandy/bt_compat.py @@ -2,6 +2,7 @@ import os.path import time +import threading import thandy.master_keys @@ -74,3 +75,58 @@ class BtCompat: 'creation date': long(time.time())} return BitTorrent.bencode.bencode(data) + def getFileLength(self, file): + """Parse the .torrent metainfo file and return the length of the + file it refers to. + """ + f = open(file, 'rb') + metainfo = BitTorrent.bencode.bdecode(f.read())['info'] + f.close() + assert(metainfo['length']) + return metainfo['length'] + + def getFileHash(self, file): + """Parse the .torrent metainfo file and return the hash of the + file it refers to. + """ + f = open(file, 'rb') + metainfo = BitTorrent.bencode.bdecode(f.read())['info'] + f.close() + return sha(BitTorrent.bencode.bencode(metainfo)).hexdigest() + + def download(self, metaFile, saveTo ): + """Initiate a download via bittorrent.""" + + event = threading.Event() + + params = ['--responsefile', metaFile, '--saveas', saveTo] + + def filefunc(default, size, saveas, dir): + return saveas + + def statusfunc(dict): + # XXX we should see how fast we upload/download here. + # If we don't get a connection for quite a while, or we are + # _very_ slow, we should cancel bt, disable it, and start fetching + # via http. + pass + + def finfunc(): + # XXX here we can set a timer for how long to seed, or + # wait for statusfunc to have shared some data, or something. + # Not the real solution, though, because installation will be + # delayed by the time we sleep... + # time.sleep(60) + event.set() + pass + + def errorfunc(msg): + # XXX Not really sure how to encounter an error here. Our best bet + # is to cancel the download, stop bittorrent, and move on. + BtCompat.setUseBt(False) + event.set() + + + BitTorrent.download.download(params, filefunc, statusfunc, finfunc, + errorfunc, event, 80) + diff --git a/lib/thandy/download.py b/lib/thandy/download.py index fb1b9f3..8e774ec 100644 --- a/lib/thandy/download.py +++ b/lib/thandy/download.py @@ -634,6 +634,53 @@ class ThandyDownloadJob(DownloadJob): def getMirror(self): return self._usingMirror +class ThandyBittorrentDownloadJob(DownloadJob): + """Thandy's subtype of DownloadJob with BitTorrent support. Makes sure the + file downloaded via BitTorrent is the file we wanted, and moves + it into the right place. + """ + def __init__(self, metaFile, relPath, destPath, wantHash=None, + supportedURLTypes=None, useTor=None, repoFile=None, + downloadStatusLog=None, wantLength=None): + + DownloadJob.__init__(self, destPath, None, wantHash=wantHash, + wantLength=wantLength, + useTor=useTor, repoFile=repoFile) + self._relPath = relPath + self._metaFile = metaFile + + tmppath = thandy.util.userFilename("tmp") + if relPath.startswith("/"): + relPath = relPath[1:] + self._tmpPath = os.path.join(tmppath, relPath) + + d = os.path.dirname(self._tmpPath) + if not os.path.exists(d): + os.makedirs(d, 0700) + + self._downloadStatusLog = downloadStatusLog + + def setDownloadStatusLog(self, log): + self._downloadStatusLog = log + + def getRelativePath(self): + return self._relPath + + def _download(self): + + btcomp = thandy.bt_compat.BtCompat() + btcomp.download(self._metaFile, self._tmpPath) + + try: + self._checkTmpFile() + except (thandy.FormatException, thandy.DownloadError), err: + self._removeTmpFile() + if haveStalled: + raise BadCompoundData(err) + else: + raise + thandy.util.ensureParentDir(self._destPath) + thandy.util.moveFile(self._tmpPath, self._destPath) _socks_opener = thandy.socksurls.build_socks_opener() diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index af20904..f5a4de2 100644 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -3,12 +3,14 @@ import thandy.formats import thandy.util import thandy.packagesys.PackageSystem +import thandy.bt_compat json = thandy.util.importJSON() import logging import os import time +import sys MAX_TIMESTAMP_AGE = 3*60*60 @@ -285,9 +287,10 @@ class LocalRepository: def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None, lengthDict=None, usePackageSystem=True, - installableDict=None): + installableDict=None, btMetadataDict=None): """Return a set of relative paths for all files that we need - to fetch. Assumes that we care about the bundles + to fetch, and True if we're fetching actual files to install + instead of metadata. Assumes that we care about the bundles 'trackingBundles'. DOCDOC installableDict, hashDict, usePackageSystem """ @@ -305,6 +308,9 @@ class LocalRepository: if lengthDict == None: lengthDict = {} + if btMetadataDict == None: + btMetadataDict = {} + pkgItems = None need = set() @@ -341,7 +347,7 @@ class LocalRepository: need.add(self._keylistFile.getRelativePath()) if need: - return need + return need, False # Import the keys from the keylist. self._keyDB.addFromKeylist(self._keylistFile.get()) @@ -354,7 +360,7 @@ class LocalRepository: "timestamp file and keylist.") need.add(self._keylistFile.getRelativePath()) need.add(self._timestampFile.getRelativePath()) - return need + return need, False # FINALLY, we know we have an up-to-date, signed timestamp # file. Check whether the keys and mirrors file are as @@ -375,7 +381,7 @@ class LocalRepository: need.add(self._keylistFile.getRelativePath()) if need: - return need + return need, False s = self._mirrorlistFile.checkSignatures() if not s.isValid(): @@ -389,7 +395,7 @@ class LocalRepository: need.add(self._mirrorlistFile.getRelativePath()) if need: - return need + return need, False # Okay; that's it for the metadata. Do we have the right # bundles? @@ -459,11 +465,35 @@ class LocalRepository: s = pfile.checkSignatures() if not s.isValid(): logging.warn("Package hash was as expected, but signature " - "did nto match") + "did not match") # Can't use it. continue packages[rp] = pfile + # We have the packages. If we're downloading via bittorrent, we need + # the .torrent metafiles, as well. + if thandy.bt_compat.BtCompat.shouldUseBt(): + btcomp = thandy.bt_compat.BtCompat() + for pfile in packages.values(): + package = pfile.get() + for f in package['files']: + rp = btcomp.getBtMetadataLocation(pfile.getRelativePath(),f[:1][0]) + try: + l = btcomp.getFileLength(self.getFilename(rp)) + except IOError: + need.add(rp) + continue + # XXX The following sanity check is a weak hack. + # In reality, we want to check a signature here. + if l != f[3:4][0]: + # We got a bad .torrent file. Disable BitTorrent. + logging.warn("Disable BitTorrent, bad metadata file!") + thandy.bt_compat.BtCompat.setUseBt(False) + btMetadataDict[f[:1][0]] = rp + + if need: + return need, False + # Finally, we have some packages. Do we have their underlying # files? for pfile in packages.values(): @@ -514,8 +544,12 @@ class LocalRepository: logging.info("Hash for %s not as expected; must load.", rp) need.add(rp) else: + # XXX What if not? Maybe this should always be true. + # if that works, we can get rid of the second return + # value and just use installableDict from the caller. if pkgItems.has_key(rp): installableDict.setdefault(pkg_rp, {})[rp] = pkgItems[rp] + # Okay; these are the files we need. - return need + return need, True |