diff options
Diffstat (limited to 'lib/thandy/repository.py')
-rw-r--r-- | lib/thandy/repository.py | 153 |
1 files changed, 147 insertions, 6 deletions
diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index 1385dd6..af2fa4f 100644 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -16,8 +16,18 @@ import time MAX_TIMESTAMP_AGE = 24*60*60 class RepositoryFile: + """Represents information about a file stored in our local repository + cache. Used to validate and load files. + """ def __init__(self, repository, relativePath, schema, needRole=None, signedFormat=True, needSigs=1): + """Allocate a new RepositoryFile for a file to be stored under + the LocalRepository 'repository' in relativePath. Make + sure the file validates with 'schema' (or its signed form, + if 'signedFormat'). When checking signatures, this file needs + at least 'needSigs' signatures with role 'needRole'. + """ + # These fields are as in the arguments. self._repository = repository self._relativePath = relativePath self._schema = schema @@ -25,17 +35,37 @@ class RepositoryFile: self._signedFormat = signedFormat self._needSigs = needSigs - self._signed_obj = self._main_obj = None + # The contents of the file, parsed. None if we haven't loaded + # the file. + self._main_obj = None + + # The contents of the file along with their signatures. May + # be aliased by _main_obj. None if we haven't loaded the + # file. + self._signed_obj = None + + # A SignatureStatus object, if we have checked signatures. + self._sigStatus = None + # The mtime of the file on disk, if we know it. + self._mtime = None + + def clear(self): + """DOCDOC""" + self._main_obj = self._signed_obj = None self._sigStatus = None self._mtime = None def getRelativePath(self): + """Return the filename for this item relative to the top of the + repository.""" return self._relativePath def getPath(self): + """Return the actual filename for this item.""" return self._repository.getFilename(self._relativePath) def _load(self): + """Helper: load and parse this item's contents.""" fname = self.getPath() # Propagate OSError @@ -59,6 +89,7 @@ class RepositoryFile: self._mtime = mtime def _save(self, content=None): + """Helper: Flush this object's contents to disk.""" if content == None: content = sexpr.encode @@ -69,9 +100,13 @@ class RepositoryFile: self._signed_obj = signed_obj self._main_obj = main_obj - self._mtime = mtime + self._mtime = time.time() def _checkContent(self, content): + """Helper. Check whether 'content' matches SIGNED_SCHEMA, and + self._schema (as appropraite). Return a tuple of the + signed_schema match, and the schema match, or raise + FormatException.""" try: obj = json.loads(content) @@ -94,20 +129,26 @@ class RepositoryFile: return signed_obj, main_obj def load(self): + """Load this object from disk if it hasn't already been loaded.""" if self._main_obj == None: self._load() def get(self): + """Return the object, or None if it isn't loaded.""" return self._main_obj def isLoaded(self): + """Return true iff this object is loaded.""" return self._main_obj != None def getContent(self): + """Load this object as needed and return its content.""" self.load() return self._main_obj def _checkSignatures(self): + """Helper: Try to verify all the signatures on this object, and + cache the SignatureStatus object.""" self.load() sigStatus = thandy.formats.checkSignatures(self._signed_obj, self._repository._keyDB, @@ -115,15 +156,47 @@ class RepositoryFile: self._sigStatus = sigStatus def checkSignatures(self): + """Try to verify all the signatures on this object if we + haven't already done so, and return a SignatureStatus + object.""" if self._sigStatus is None: self._checkSignatures() return self._sigStatus +class PkgFile: + def __init__(self, repository, relativePath, needHash): + self._repository = repository + self._relativePath = relativePath + self._needHash = needHash + + self._mtime = None + + def clear(self): + self._mtime = None + + def getRelativePath(self): + return self._relativePath + + def getPath(self): + return self._repository.getFilename(self._relativePath) + + def getExpectedHash(self): + return self._needHash + + def checkFile(self): + return self._needHash == self._repository.getFileDigest() + class LocalRepository: + """Represents a client's partial copy of a remote mirrored repository.""" def __init__(self, root): + """Create a new local repository that stores its files under 'root'""" + # Top of our mirror. self._root = root + + # A base keylist of master keys; we'll add others later. self._keyDB = thandy.util.getKeylist(None) + # Entries for the three invariant metafiles. self._keylistFile = RepositoryFile( self, "/meta/keys.txt", thandy.formats.KEYLIST_SCHEMA, needRole="master") @@ -133,28 +206,38 @@ class LocalRepository: self._mirrorlistFile = RepositoryFile( self, "/meta/mirrors.txt", thandy.formats.MIRRORLIST_SCHEMA, needRole="mirrors") + self._metaFiles = [ self._keylistFile, self._timestampFile, self._mirrorlistFile ] + # Map from relative path to a RepositoryFile for packages. self._packageFiles = {} + + # Map from relative path to a RepositoryFile for bundles. self._bundleFiles = {} def getFilename(self, relativePath): + """Return the file on disk that caches 'relativePath'.""" if relativePath.startswith("/"): relativePath = relativePath[1:] return os.path.join(self._root, relativePath) def getKeylistFile(self): + """Return a RepositoryFile for our keylist.""" return self._keylistFile def getTimestampFile(self): + """Return a RepositoryFile for our timestamp file.""" return self._timestampFile def getMirrorlistFile(self): + """Return a RepositoryFile for our mirrorlist.""" return self._mirrorlistFile def getPackageFile(self, relPath): + """Return a RepositoryFile for a package stored at relative path + 'relPath'.""" try: return self._packageFiles[relPath] except KeyError: @@ -164,6 +247,8 @@ class LocalRepository: return pkg def getBundleFile(self, relPath): + """Return a RepositoryFile for a bundle stored at relative path + 'relPath'.""" try: return self._bundleFiles[relPath] except KeyError: @@ -172,10 +257,38 @@ class LocalRepository: needRole='bundle') return pkg - def getFilesToUpdate(self, now=None, trackingBundles=()): + def getRequestedFile(self, relPath): + """ """ + for f in self._metafiles: + if f.getRelativePath() == relPath: + return f + for f in self._bundleFiles.itervalues(): + if f.getRelativePath() == relPath: + return f + for f in self._packageFiles.itervalues(): + if f.getRelativePath() == relPath: + return f + f.load() + for item in f.get()['files']: + rp, h = item[:2] + if rp == relPath: + return PkgFile(self, rp, thandy.formats.parseHash(h)) + + def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None): + """Return a set of relative paths for all files that we need + to fetch. Assumes that we care about the bundles + 'trackingBundles'. If hashDict is provided, add mappings to it + from the relative paths we want to fecth to the hashes that we + want those items to have, when we know those hashes. + """ + if now == None: now = time.time() + if hashDict == None: + # Use a dummy hashdict. + hashDict = {} + need = set() # Fetch missing metafiles. @@ -196,6 +309,8 @@ class LocalRepository: age = now - thandy.formats.parseTime(ts['at']) ts = thandy.formats.TimestampFile.fromJSon(ts) if age > MAX_TIMESTAMP_AGE: + logging.info("Timestamp file from %s is out of " + "date; must fetch it.", ts['at']) need.add(self._timestampFile.getRelativePath()) # If the keylist isn't signed right, we can't check the @@ -203,6 +318,8 @@ class LocalRepository: if self._keylistFile.get(): s = self._keylistFile.checkSignatures() if not s.isValid(): # For now only require one master key. + logging.info("Key list is not properly signed; must get a " + "new one.") need.add(self._keylistFile.getRelativePath()) if need: @@ -215,6 +332,8 @@ class LocalRepository: # new keylist. s = self._timestampFile.checkSignatures() if not s.isValid(): + logging.info("Timestamp file is not properly signed; fetching new " + "timestamp file and keylist.") need.add(self._keylistFile.getRelativePath()) need.add(self._timestampFile.getRelativePath()) return need @@ -222,9 +341,15 @@ class LocalRepository: # FINALLY, we know we have an up-to-date, signed timestamp # file. Check whether the keys and mirrors file are as # authenticated. + hashDict[self._keylistFile.getRelativePath()] = \ + ts.getKeylistInfo().getHash() + hashDict[self._mirrorlistFile.getRelativePath()] = \ + ts.getMirrorlistInfo().getHash() + h_kf = thandy.formats.getDigest(self._keylistFile.get()) h_expected = ts.getKeylistInfo().getHash() if h_kf != h_expected: + logging.info("Keylist file hash did not match. Must fetch it.") need.add(self._keylistFile.getRelativePath()) if need: @@ -232,11 +357,13 @@ class LocalRepository: s = self._mirrorlistFile.checkSignatures() if not s.isValid(): + logging.info("Mirrorlist file signatures not valid. Must fetch.") need.add(self._mirrorlistFile.getRelativePath()) h_mf = thandy.formats.getDigest(self._mirrorlistFile.get()) h_expected = ts.getMirrorlistInfo().getHash() if h_mf != h_expected: + logging.info("Mirrorlist file hash did not match. Must fetch.") need.add(self._mirrorlistFile.getRelativePath()) if need: @@ -249,26 +376,30 @@ class LocalRepository: try: binfo = ts.getBundleInfo(b) except KeyError: - logging.warn("Unrecognized bundle %s"%b) + logging.warn("Bundle %s not listed in timestamp file."%b) continue rp = binfo.getRelativePath() + hashDict[rp] = h_expected = binfo.getHash() bfile = self.getBundleFile(rp) try: bfile.load() except OSError: + logging.info("Can't find bundle %s on disk; must fetch.", rp) need.add(rp) continue h_b = thandy.formats.getDigest(bfile.get()) - h_expected = binfo.getHash() if h_b != h_expected: + logging.info("Bundle hash not as expected; must fetch.", rp) need.add(rp) continue s = bfile.checkSignatures() if not s.isValid(): # Can't actually use it. + logging.warn("Bundle hash was as expected, but signatures did " + "not match.") continue bundles[rp] = bfile @@ -280,20 +411,26 @@ class LocalRepository: for pkginfo in bundle['packages']: rp = pkginfo['path'] pfile = self.getPackageFile(rp) + h_expected = thandy.formats.parseHash(pkginfo['hash']) + hashDict[rp] = h_expected try: pfile.load() except OSError: + logging.info("Can't find package %s on disk; must fetch.", + rp) need.add(rp) continue h_p = thandy.formats.getDigest(pfile.get()) - h_expected = thandy.formats.parseHash(pkginfo['hash']) if h_p != h_expected: + logging.info("Wrong hash for package %s; must fetch.", rp) need.add(rp) continue s = pfile.checkSignatures() if not s.isValid(): + logging.warn("Package hash was as expected, but signature " + "did nto match") # Can't use it. continue packages[rp] = pfile @@ -305,13 +442,17 @@ class LocalRepository: for f in package['files']: rp, h = f[:2] h_expected = thandy.formats.parseHash(h) + hashDict[rp] = h_expected fn = self.getFilename(rp) try: h_got = thandy.formats.getFileDigest(fn) except OSError: + logging.info("Installable file %s not found on disk; " + "must load", rp) need.add(rp) continue if h_got != h_expected: + logging.info("Hash for %s not as expected; must load.", rp) need.add(rp) # Okay; these are the files we need. |