Add the ability to download files via BitTorrent

The Client has learned a new option, --download-method, too specify
whether we're downloading directly or via BitTorrent.

This implementation has a few remaining issues, the biggest one is
that seeding isn't implemented at all (when the download stops,
Thandy stops sharing). Failure to download due to no available
peers also doesn't work.

4 files changed, 182 insertions, 20 deletions

diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py
index 73c96df..bcbdfc7 100644
--- a/lib/thandy/ClientCLI.py
+++ b/lib/thandy/ClientCLI.py
@@ -68,7 +68,8 @@ def update(args):
     options, args = getopt.getopt(args, "",
         [ "repo=", "no-download", "loop", "no-packagesys",
           "install", "socks-port=", "debug", "info",
-          "warn", "force-check", "controller-log-format"
+          "warn", "force-check", "controller-log-format",
+          "download-method="
           ])
     download = True
     keep_looping = False
@@ -76,6 +77,7 @@ def update(args):
     install = False
     socksPort = None
     forceCheck = False
+    downloadMethod = "direct"
 
     for o, v in options:
         if o == '--repo':
@@ -92,12 +94,20 @@ def update(args):
             socksPort = int(v)
         elif o == '--force-check':
             forceCheck = True
+        elif o == '--download-method':
+            downloadMethod = v
 
     configureLogs(options)
 
     if socksPort:
         thandy.socksurls.setSocksProxy("127.0.0.1", socksPort)
 
+    if downloadMethod == "bittorrent":
+        thandy.bt_compat.BtCompat.setUseBt(True)
+    elif downloadMethod != "direct":
+        usage()
+        sys.exit()
+
     repo = thandy.repository.LocalRepository(repoRoot)
     downloader = thandy.download.DownloadManager()
     downloader.start()
@@ -109,11 +119,15 @@ def update(args):
         hashes = {}
         lengths = {}
         installable = {}
+        btMetadata = {}
         logging.info("Checking for files to update.")
-        files = repo.getFilesToUpdate(trackingBundles=args, hashDict=hashes,
-                                      lengthDict=lengths,
-                                      usePackageSystem=use_packagesys,
-                                      installableDict=installable)
+        files, downloadingFiles = repo.getFilesToUpdate(
+              trackingBundles=args,
+              hashDict=hashes,
+              lengthDict=lengths,
+              usePackageSystem=use_packagesys,
+              installableDict=installable,
+              btMetadataDict=btMetadata)
 
         if forceCheck:
             files.add("/meta/timestamp.txt")
@@ -183,13 +197,23 @@ def update(args):
                 logging.info("Waiting a while before we fetch %s", f)
                 continue
 
-            dj = thandy.download.ThandyDownloadJob(
-                f, repo.getFilename(f),
-                mirrorlist,
-                wantHash=hashes.get(f),
-                wantLength=lengths.get(f),
-                repoFile=repo.getRequestedFile(f),
-                useTor=(socksPort!=None))
+            dj = None
+            if thandy.bt_compat.BtCompat.shouldUseBt() and downloadingFiles:
+                dj = thandy.download.ThandyBittorrentDownloadJob(
+                    repo.getFilename(btMetadata[f]), f,
+                    repo.getFilename(f),
+                    wantHash=hashes.get(f),
+                    wantLength=lengths.get(f),
+                    repoFile=repo.getRequestedFile(f))
+
+            else:
+                dj = thandy.download.ThandyDownloadJob(
+                    f, repo.getFilename(f),
+                    mirrorlist,
+                    wantHash=hashes.get(f),
+                    wantLength=lengths.get(f),
+                    repoFile=repo.getRequestedFile(f),
+                    useTor=(socksPort!=None))
 
             def successCb(rp=f):
                 rf = repo.getRequestedFile(rp)
@@ -219,6 +243,7 @@ def usage():
     print "         [--no-packagesys] [--install] [--socks-port=port]"
     print "         [--debug|--info|--warn] [--force-check]"
     print "         [--controller-log-format]"
+    print "         [--download-method=direct|bittorrent]"
     print "         bundle1, bundle2, ..."
     print "  json2xml file"
     sys.exit(1)
diff --git a/lib/thandy/bt_compat.py b/lib/thandy/bt_compat.py
index 43cecad..e9298a6 100644
--- a/lib/thandy/bt_compat.py
+++ b/lib/thandy/bt_compat.py
@@ -2,6 +2,7 @@
 
 import os.path
 import time
+import threading
 
 import thandy.master_keys
 
@@ -74,3 +75,58 @@ class BtCompat:
                 'creation date': long(time.time())}
         return BitTorrent.bencode.bencode(data)
 
+    def getFileLength(self, file):
+        """Parse the .torrent metainfo file and return the length of the
+           file it refers to.
+        """
+        f = open(file, 'rb')
+        metainfo = BitTorrent.bencode.bdecode(f.read())['info']
+        f.close()
+        assert(metainfo['length'])
+        return metainfo['length']
+
+    def getFileHash(self, file):
+        """Parse the .torrent metainfo file and return the hash of the
+           file it refers to.
+        """
+        f = open(file, 'rb')
+        metainfo = BitTorrent.bencode.bdecode(f.read())['info']
+        f.close()
+        return sha(BitTorrent.bencode.bencode(metainfo)).hexdigest()
+
+    def download(self, metaFile, saveTo ):
+        """Initiate a download via bittorrent."""
+
+        event = threading.Event()
+
+        params = ['--responsefile', metaFile, '--saveas', saveTo]
+
+        def filefunc(default, size, saveas, dir):
+            return saveas
+
+        def statusfunc(dict):
+            # XXX we should see how fast we upload/download here.
+            # If we don't get a connection for quite a while, or we are
+            # _very_ slow, we should cancel bt, disable it, and start fetching
+            # via http.
+            pass
+
+        def finfunc():
+            # XXX here we can set a timer for how long to seed, or
+            # wait for statusfunc to have shared some data, or something.
+            # Not the real solution, though, because installation will be
+            # delayed by the time we sleep...
+            # time.sleep(60)
+            event.set()
+            pass
+
+        def errorfunc(msg):
+            # XXX Not really sure how to encounter an error here. Our best bet
+            # is to cancel the download, stop bittorrent, and move on.
+            BtCompat.setUseBt(False)
+            event.set()
+
+
+        BitTorrent.download.download(params, filefunc, statusfunc, finfunc,
+                                     errorfunc, event, 80)
+
diff --git a/lib/thandy/download.py b/lib/thandy/download.py
index fb1b9f3..8e774ec 100644
--- a/lib/thandy/download.py
+++ b/lib/thandy/download.py
@@ -634,6 +634,53 @@ class ThandyDownloadJob(DownloadJob):
     def getMirror(self):
         return self._usingMirror
 
+class ThandyBittorrentDownloadJob(DownloadJob):
+    """Thandy's subtype of DownloadJob with BitTorrent support. Makes sure the
+       file downloaded via BitTorrent is the file we wanted, and moves
+       it into the right place.
+    """
+    def __init__(self, metaFile, relPath, destPath, wantHash=None,
+                 supportedURLTypes=None, useTor=None, repoFile=None,
+                 downloadStatusLog=None, wantLength=None):
+
+        DownloadJob.__init__(self, destPath, None, wantHash=wantHash,
+                             wantLength=wantLength,
+                             useTor=useTor, repoFile=repoFile)
+        self._relPath = relPath
+        self._metaFile = metaFile
+
+        tmppath = thandy.util.userFilename("tmp")
+        if relPath.startswith("/"):
+            relPath = relPath[1:]
+        self._tmpPath = os.path.join(tmppath, relPath)
+
+        d = os.path.dirname(self._tmpPath)
+        if not os.path.exists(d):
+            os.makedirs(d, 0700)
+
+        self._downloadStatusLog = downloadStatusLog
+
+    def setDownloadStatusLog(self, log):
+        self._downloadStatusLog = log
+
+    def getRelativePath(self):
+        return self._relPath
+
+    def _download(self):
+
+        btcomp = thandy.bt_compat.BtCompat()
+        btcomp.download(self._metaFile, self._tmpPath)
+
+        try:
+            self._checkTmpFile()
+        except (thandy.FormatException, thandy.DownloadError), err:
+            self._removeTmpFile()
+            if haveStalled:
+                raise BadCompoundData(err)
+            else:
+                raise
+        thandy.util.ensureParentDir(self._destPath)
+        thandy.util.moveFile(self._tmpPath, self._destPath)
 
 _socks_opener = thandy.socksurls.build_socks_opener()
 
diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py
index af20904..f5a4de2 100644
--- a/lib/thandy/repository.py
+++ b/lib/thandy/repository.py
@@ -3,12 +3,14 @@
 import thandy.formats
 import thandy.util
 import thandy.packagesys.PackageSystem
+import thandy.bt_compat
 
 json = thandy.util.importJSON()
 
 import logging
 import os
 import time
+import sys
 
 MAX_TIMESTAMP_AGE = 3*60*60
 
@@ -285,9 +287,10 @@ class LocalRepository:
 
     def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None,
                          lengthDict=None, usePackageSystem=True,
-                         installableDict=None):
+                         installableDict=None, btMetadataDict=None):
         """Return a set of relative paths for all files that we need
-           to fetch.  Assumes that we care about the bundles
+           to fetch, and True if we're fetching actual files to install
+           instead of metadata.  Assumes that we care about the bundles
            'trackingBundles'.
            DOCDOC installableDict, hashDict, usePackageSystem
         """
@@ -305,6 +308,9 @@ class LocalRepository:
         if lengthDict == None:
             lengthDict = {}
 
+        if btMetadataDict == None:
+            btMetadataDict = {}
+
         pkgItems = None
 
         need = set()
@@ -341,7 +347,7 @@ class LocalRepository:
                 need.add(self._keylistFile.getRelativePath())
 
         if need:
-            return need
+            return need, False
 
         # Import the keys from the keylist.
         self._keyDB.addFromKeylist(self._keylistFile.get())
@@ -354,7 +360,7 @@ class LocalRepository:
                          "timestamp file and keylist.")
             need.add(self._keylistFile.getRelativePath())
             need.add(self._timestampFile.getRelativePath())
-            return need
+            return need, False
 
         # FINALLY, we know we have an up-to-date, signed timestamp
         # file.  Check whether the keys and mirrors file are as
@@ -375,7 +381,7 @@ class LocalRepository:
             need.add(self._keylistFile.getRelativePath())
 
         if need:
-            return need
+            return need, False
 
         s = self._mirrorlistFile.checkSignatures()
         if not s.isValid():
@@ -389,7 +395,7 @@ class LocalRepository:
             need.add(self._mirrorlistFile.getRelativePath())
 
         if need:
-            return need
+            return need, False
 
         # Okay; that's it for the metadata.  Do we have the right
         # bundles?
@@ -459,11 +465,35 @@ class LocalRepository:
                 s = pfile.checkSignatures()
                 if not s.isValid():
                     logging.warn("Package hash was as expected, but signature "
-                                 "did nto match")
+                                 "did not match")
                     # Can't use it.
                     continue
                 packages[rp] = pfile
 
+        # We have the packages. If we're downloading via bittorrent, we need
+        # the .torrent metafiles, as well.
+        if thandy.bt_compat.BtCompat.shouldUseBt():
+            btcomp = thandy.bt_compat.BtCompat()
+            for pfile in packages.values():
+                package = pfile.get()
+                for f in package['files']:
+                    rp = btcomp.getBtMetadataLocation(pfile.getRelativePath(),f[:1][0])
+                    try:
+                        l = btcomp.getFileLength(self.getFilename(rp))
+                    except IOError:
+                        need.add(rp)
+                        continue
+                    # XXX The following sanity check is a weak hack.
+                    # In reality, we want to check a signature here.
+                    if l != f[3:4][0]:
+                        # We got a bad .torrent file. Disable BitTorrent.
+                        logging.warn("Disable BitTorrent, bad metadata file!")
+                        thandy.bt_compat.BtCompat.setUseBt(False)
+                    btMetadataDict[f[:1][0]] = rp
+
+        if need:
+            return need, False
+
         # Finally, we have some packages.  Do we have their underlying
         # files?
         for pfile in packages.values():
@@ -514,8 +544,12 @@ class LocalRepository:
                     logging.info("Hash for %s not as expected; must load.", rp)
                     need.add(rp)
                 else:
+                    # XXX What if not? Maybe this should always be true.
+                    # if that works, we can get rid of the second return
+                    # value and just use installableDict from the caller.
                     if pkgItems.has_key(rp):
                         installableDict.setdefault(pkg_rp, {})[rp] = pkgItems[rp]
 
+
         # Okay; these are the files we need.
-        return need
+        return need, True