have some more thandy. This update includes a working downloader with tor support, a package system framework, and more. Most of what's left is glue code.

git-svn-id: file:///home/or/svnrepo/updater/trunk@17288 55e972cd-5a19-0410-ae62-a4d7a52db4cd

1 files changed, 147 insertions, 6 deletions

diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py
index 1385dd6..af2fa4f 100644
--- a/lib/thandy/repository.py
+++ b/lib/thandy/repository.py
@@ -16,8 +16,18 @@ import time
 MAX_TIMESTAMP_AGE = 24*60*60
 
 class RepositoryFile:
+    """Represents information about a file stored in our local repository
+       cache.  Used to validate and load files.
+    """
     def __init__(self, repository, relativePath, schema,
                  needRole=None, signedFormat=True, needSigs=1):
+        """Allocate a new RepositoryFile for a file to be stored under
+           the LocalRepository 'repository' in relativePath.  Make
+           sure the file validates with 'schema' (or its signed form,
+           if 'signedFormat').  When checking signatures, this file needs
+           at least 'needSigs' signatures with role 'needRole'.
+        """
+        # These fields are as in the arguments.
         self._repository = repository
         self._relativePath = relativePath
         self._schema = schema
@@ -25,17 +35,37 @@ class RepositoryFile:
         self._signedFormat = signedFormat
         self._needSigs = needSigs
 
-        self._signed_obj = self._main_obj = None
+        # The contents of the file, parsed.  None if we haven't loaded
+        # the file.
+        self._main_obj = None
+
+        # The contents of the file along with their signatures.  May
+        # be aliased by _main_obj.  None if we haven't loaded the
+        # file.
+        self._signed_obj = None
+
+        # A SignatureStatus object, if we have checked signatures.
+        self._sigStatus = None
+        # The mtime of the file on disk, if we know it.
+        self._mtime = None
+
+    def clear(self):
+        """DOCDOC"""
+        self._main_obj = self._signed_obj = None
         self._sigStatus = None
         self._mtime = None
 
     def getRelativePath(self):
+        """Return the filename for this item relative to the top of the
+           repository."""
         return self._relativePath
 
     def getPath(self):
+        """Return the actual filename for this item."""
         return self._repository.getFilename(self._relativePath)
 
     def _load(self):
+        """Helper: load and parse this item's contents."""
         fname = self.getPath()
 
         # Propagate OSError
@@ -59,6 +89,7 @@ class RepositoryFile:
         self._mtime = mtime
 
     def _save(self, content=None):
+        """Helper: Flush this object's contents to disk."""
         if content == None:
             content = sexpr.encode
 
@@ -69,9 +100,13 @@ class RepositoryFile:
 
         self._signed_obj = signed_obj
         self._main_obj = main_obj
-        self._mtime = mtime
+        self._mtime = time.time()
 
     def _checkContent(self, content):
+        """Helper.  Check whether 'content' matches SIGNED_SCHEMA, and
+           self._schema (as appropraite).  Return a tuple of the
+           signed_schema match, and the schema match, or raise
+           FormatException."""
 
         try:
             obj = json.loads(content)
@@ -94,20 +129,26 @@ class RepositoryFile:
         return signed_obj, main_obj
 
     def load(self):
+        """Load this object from disk if it hasn't already been loaded."""
         if self._main_obj == None:
             self._load()
 
     def get(self):
+        """Return the object, or None if it isn't loaded."""
         return self._main_obj
 
     def isLoaded(self):
+        """Return true iff this object is loaded."""
         return self._main_obj != None
 
     def getContent(self):
+        """Load this object as needed and return its content."""
         self.load()
         return self._main_obj
 
     def _checkSignatures(self):
+        """Helper: Try to verify all the signatures on this object, and
+           cache the SignatureStatus object."""
         self.load()
         sigStatus = thandy.formats.checkSignatures(self._signed_obj,
                                      self._repository._keyDB,
@@ -115,15 +156,47 @@ class RepositoryFile:
         self._sigStatus = sigStatus
 
     def checkSignatures(self):
+        """Try to verify all the signatures on this object if we
+           haven't already done so, and return a SignatureStatus
+           object."""
         if self._sigStatus is None:
             self._checkSignatures()
         return self._sigStatus
 
+class PkgFile:
+    def __init__(self, repository, relativePath, needHash):
+        self._repository = repository
+        self._relativePath = relativePath
+        self._needHash = needHash
+
+        self._mtime = None
+
+    def clear(self):
+        self._mtime = None
+
+    def getRelativePath(self):
+        return self._relativePath
+
+    def getPath(self):
+        return self._repository.getFilename(self._relativePath)
+
+    def getExpectedHash(self):
+        return self._needHash
+
+    def checkFile(self):
+        return self._needHash == self._repository.getFileDigest()
+
 class LocalRepository:
+    """Represents a client's partial copy of a remote mirrored repository."""
     def __init__(self, root):
+        """Create a new local repository that stores its files under 'root'"""
+        # Top of our mirror.
         self._root = root
+
+        # A base keylist of master keys; we'll add others later.
         self._keyDB = thandy.util.getKeylist(None)
 
+        # Entries for the three invariant metafiles.
         self._keylistFile = RepositoryFile(
             self, "/meta/keys.txt", thandy.formats.KEYLIST_SCHEMA,
             needRole="master")
@@ -133,28 +206,38 @@ class LocalRepository:
         self._mirrorlistFile = RepositoryFile(
             self, "/meta/mirrors.txt", thandy.formats.MIRRORLIST_SCHEMA,
             needRole="mirrors")
+
         self._metaFiles = [ self._keylistFile,
                             self._timestampFile,
                             self._mirrorlistFile ]
 
+        # Map from relative path to a RepositoryFile for packages.
         self._packageFiles = {}
+
+        # Map from relative path to a RepositoryFile for bundles.
         self._bundleFiles = {}
 
     def getFilename(self, relativePath):
+        """Return the file on disk that caches 'relativePath'."""
         if relativePath.startswith("/"):
             relativePath = relativePath[1:]
         return os.path.join(self._root, relativePath)
 
     def getKeylistFile(self):
+        """Return a RepositoryFile for our keylist."""
         return self._keylistFile
 
     def getTimestampFile(self):
+        """Return a RepositoryFile for our timestamp file."""
         return self._timestampFile
 
     def getMirrorlistFile(self):
+        """Return a RepositoryFile for our mirrorlist."""
         return self._mirrorlistFile
 
     def getPackageFile(self, relPath):
+        """Return a RepositoryFile for a package stored at relative path
+           'relPath'."""
         try:
             return self._packageFiles[relPath]
         except KeyError:
@@ -164,6 +247,8 @@ class LocalRepository:
             return pkg
 
     def getBundleFile(self, relPath):
+        """Return a RepositoryFile for a bundle stored at relative path
+           'relPath'."""
         try:
             return self._bundleFiles[relPath]
         except KeyError:
@@ -172,10 +257,38 @@ class LocalRepository:
                 needRole='bundle')
             return pkg
 
-    def getFilesToUpdate(self, now=None, trackingBundles=()):
+    def getRequestedFile(self, relPath):
+        """ """
+        for f in self._metafiles:
+            if f.getRelativePath() == relPath:
+                return f
+        for f in self._bundleFiles.itervalues():
+            if f.getRelativePath() == relPath:
+                return f
+        for f in self._packageFiles.itervalues():
+            if f.getRelativePath() == relPath:
+                return f
+            f.load()
+            for item in f.get()['files']:
+                rp, h = item[:2]
+                if rp == relPath:
+                    return PkgFile(self, rp, thandy.formats.parseHash(h))
+
+    def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None):
+        """Return a set of relative paths for all files that we need
+           to fetch.  Assumes that we care about the bundles
+           'trackingBundles'.  If hashDict is provided, add mappings to it
+           from the relative paths we want to fecth to the hashes that we
+           want those items to have, when we know those hashes.
+        """
+
         if now == None:
             now = time.time()
 
+        if hashDict == None:
+            # Use a dummy hashdict.
+            hashDict = {}
+
         need = set()
 
         # Fetch missing metafiles.
@@ -196,6 +309,8 @@ class LocalRepository:
             age = now - thandy.formats.parseTime(ts['at'])
             ts = thandy.formats.TimestampFile.fromJSon(ts)
             if age > MAX_TIMESTAMP_AGE:
+                logging.info("Timestamp file from %s is out of "
+                             "date; must fetch it.", ts['at'])
                 need.add(self._timestampFile.getRelativePath())
 
         # If the keylist isn't signed right, we can't check the
@@ -203,6 +318,8 @@ class LocalRepository:
         if self._keylistFile.get():
             s = self._keylistFile.checkSignatures()
             if not s.isValid(): # For now only require one master key.
+                logging.info("Key list is not properly signed; must get a "
+                             "new one.")
                 need.add(self._keylistFile.getRelativePath())
 
         if need:
@@ -215,6 +332,8 @@ class LocalRepository:
         # new keylist.
         s = self._timestampFile.checkSignatures()
         if not s.isValid():
+            logging.info("Timestamp file is not properly signed; fetching new "
+                         "timestamp file and keylist.")
             need.add(self._keylistFile.getRelativePath())
             need.add(self._timestampFile.getRelativePath())
             return need
@@ -222,9 +341,15 @@ class LocalRepository:
         # FINALLY, we know we have an up-to-date, signed timestamp
         # file.  Check whether the keys and mirrors file are as
         # authenticated.
+        hashDict[self._keylistFile.getRelativePath()] = \
+            ts.getKeylistInfo().getHash()
+        hashDict[self._mirrorlistFile.getRelativePath()] = \
+            ts.getMirrorlistInfo().getHash()
+
         h_kf = thandy.formats.getDigest(self._keylistFile.get())
         h_expected = ts.getKeylistInfo().getHash()
         if h_kf != h_expected:
+            logging.info("Keylist file hash did not match.  Must fetch it.")
             need.add(self._keylistFile.getRelativePath())
 
         if need:
@@ -232,11 +357,13 @@ class LocalRepository:
 
         s = self._mirrorlistFile.checkSignatures()
         if not s.isValid():
+            logging.info("Mirrorlist file signatures not valid. Must fetch.")
             need.add(self._mirrorlistFile.getRelativePath())
 
         h_mf = thandy.formats.getDigest(self._mirrorlistFile.get())
         h_expected = ts.getMirrorlistInfo().getHash()
         if h_mf != h_expected:
+            logging.info("Mirrorlist file hash did not match. Must fetch.")
             need.add(self._mirrorlistFile.getRelativePath())
 
         if need:
@@ -249,26 +376,30 @@ class LocalRepository:
             try:
                 binfo = ts.getBundleInfo(b)
             except KeyError:
-                logging.warn("Unrecognized bundle %s"%b)
+                logging.warn("Bundle %s not listed in timestamp file."%b)
                 continue
 
             rp = binfo.getRelativePath()
+            hashDict[rp] = h_expected = binfo.getHash()
             bfile = self.getBundleFile(rp)
             try:
                 bfile.load()
             except OSError:
+                logging.info("Can't find bundle %s on disk; must fetch.", rp)
                 need.add(rp)
                 continue
 
             h_b = thandy.formats.getDigest(bfile.get())
-            h_expected = binfo.getHash()
             if h_b != h_expected:
+                logging.info("Bundle hash not as expected; must fetch.", rp)
                 need.add(rp)
                 continue
 
             s = bfile.checkSignatures()
             if not s.isValid():
                 # Can't actually use it.
+                logging.warn("Bundle hash was as expected, but signatures did "
+                             "not match.")
                 continue
 
             bundles[rp] = bfile
@@ -280,20 +411,26 @@ class LocalRepository:
             for pkginfo in bundle['packages']:
                 rp = pkginfo['path']
                 pfile = self.getPackageFile(rp)
+                h_expected = thandy.formats.parseHash(pkginfo['hash'])
+                hashDict[rp] = h_expected
                 try:
                     pfile.load()
                 except OSError:
+                    logging.info("Can't find package %s on disk; must fetch.",
+                                 rp)
                     need.add(rp)
                     continue
 
                 h_p = thandy.formats.getDigest(pfile.get())
-                h_expected = thandy.formats.parseHash(pkginfo['hash'])
                 if h_p != h_expected:
+                    logging.info("Wrong hash for package %s; must fetch.", rp)
                     need.add(rp)
                     continue
 
                 s = pfile.checkSignatures()
                 if not s.isValid():
+                    logging.warn("Package hash was as expected, but signature "
+                                 "did nto match")
                     # Can't use it.
                     continue
                 packages[rp] = pfile
@@ -305,13 +442,17 @@ class LocalRepository:
             for f in package['files']:
                 rp, h = f[:2]
                 h_expected = thandy.formats.parseHash(h)
+                hashDict[rp] = h_expected
                 fn = self.getFilename(rp)
                 try:
                     h_got = thandy.formats.getFileDigest(fn)
                 except OSError:
+                    logging.info("Installable file %s not found on disk; "
+                                 "must load", rp)
                     need.add(rp)
                     continue
                 if h_got != h_expected:
+                    logging.info("Hash for %s not as expected; must load.", rp)
                     need.add(rp)
 
         # Okay; these are the files we need.