diff options
Diffstat (limited to 'lib/glider/repository.py')
| -rw-r--r-- | lib/glider/repository.py | 294 |
1 files changed, 243 insertions, 51 deletions
diff --git a/lib/glider/repository.py b/lib/glider/repository.py index 9ec1dc6..b956993 100644 --- a/lib/glider/repository.py +++ b/lib/glider/repository.py @@ -1,10 +1,14 @@ -import sexp.parse -import sexp.access import glider.formats +import glider.util +import simplejson +import logging import os import threading +import time + +MAX_TIMESTAMP_AGE = 24*60*60 class RepositoryFile: def __init__(self, repository, relativePath, schema, @@ -16,12 +20,15 @@ class RepositoryFile: self._signedFormat = signedFormat self._needSigs = needSigs - self._signed_sexpr = None - self._main_sexpr = None + self._signed_obj = self._main_obj = None + self._sigStatus = None self._mtime = None + def getRelativePath(self): + return self._relativePath + def getPath(self): - return os.path.join(self._repository._root, self._relativePath) + return self._repository.getFilename(self._relativePath) def _load(self): fname = self.getPath() @@ -40,82 +47,267 @@ class RepositoryFile: finally: f.close() - signed_sexpr,main_sexpr = self._checkContent(content) + signed_obj,main_obj = self._checkContent(content) - self._signed_sexpr = signed_sexpr - self._main_sexpr = main_sexpr + self._signed_obj = signed_obj + self._main_obj = main_obj self._mtime = mtime def _save(self, content=None): if content == None: content = sexpr.encode - signed_sexpr,main_sexpr = self._checkContent(content) + signed_obj,main_obj = self._checkContent(content) fname = self.getPath() - fname_tmp = fname+"_tmp" + glider.util.replaceFile(fname, contents) - fd = os.open(fname_tmp, os.WRONLY|os.O_CREAT|os.O_TRUNC, 0644) - try: - os.write(fd, contents) - finally: - os.close(fd) - if sys.platform in ('cygwin', 'win32'): - # Win32 doesn't let rename replace an existing file. - try: - os.unlink(fname) - except OSError: - pass - os.rename(fname_tmp, fname) - - self._signed_sexpr = signed_sexpr - self._main_sexpr = main_sexpr + self._signed_obj = signed_obj + self._main_obj = main_obj self._mtime = mtime def _checkContent(self, content): - sexpr = sexp.parse.parse(content) - if not sexpr: - raise ParseError() + + try: + obj = simplejson.loads(content) + except ValueError, e: + raise glider.FormatException("Couldn't decode content: %s"%e) if self._signedFormat: - if not glider.formats.SIGNED_SCHEMA.matches(sexpr): - raise FormatError() - - sigs = checkSignatures(sexpr, self._repository._keyDB, - self._needRole, self._relativePath) - good = sigs[0] - # XXXX If good is too low but unknown is high, we may need - # a new key file. - if len(good) < 1: - raise SignatureError() - - main_sexpr = sexpr[1] - signed_sexpr = sexpr + # This is supposed to be signed. + glider.formats.SIGNED_SCHEMA.checkMatch(obj) + + main_obj = obj['signed'] + signed_obj = obj else: - signed_sexpr = None - main_sexpr = sexpr + signed_obj = None + main_obj = obj - if self._schema != None and not self._schema.matches(main_sexpr): - raise FormatError() + if self._schema != None: + self._schema.checkMatch(main_obj) - return signed_sexpr, main_sexpr + return signed_obj, main_obj def load(self): - if self._main_sexpr == None: + if self._main_obj == None: self._load() + def get(self): + return self._main_obj + + def isLoaded(self): + return self._main_obj != None + + def getContent(self): + self.load() + return self._main_obj + + def _checkSignatures(self): + self.load() + sigStatus = glider.formats.checkSignatures(self._signed_obj, + self._repository._keyDB, + self._needRole, self._relativePath) + self._sigStatus = sigStatus + + def checkSignatures(self): + if self._sigStatus is None: + self._checkSignatures() + return self._sigStatus + class LocalRepository: def __init__(self, root): self._root = root - self._keyDB = None + self._keyDB = glider.util.getKeylist(None) self._keylistFile = RepositoryFile( - self, "meta/keys.txt", glider.formats.KEYLIST_SCHEMA, + self, "/meta/keys.txt", glider.formats.KEYLIST_SCHEMA, needRole="master") self._timestampFile = RepositoryFile( - self, "meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA, + self, "/meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA, needRole="timestamp") self._mirrorlistFile = RepositoryFile( - self, "meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA, + self, "/meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA, needRole="mirrors") + self._metaFiles = [ self._keylistFile, + self._timestampFile, + self._mirrorlistFile ] + + self._packageFiles = {} + self._bundleFiles = {} + + def getFilename(self, relativePath): + if relativePath.startswith("/"): + relativePath = relativePath[1:] + return os.path.join(self._root, relativePath) + + def getKeylistFile(self): + return self._keylistFile + + def getTimestampFile(self): + return self._timestampFile + + def getMirrorlistFile(self): + return self._mirrorlistFile + + def getPackageFile(self, relPath): + try: + return self._packageFiles[relPath] + except KeyError: + self._packageFiles[relPath] = pkg = RepositoryFile( + self, relPath, glider.formats.PACKAGE_SCHEMA, + needRole='package') + return pkg + + def getBundleFile(self, relPath): + try: + return self._bundleFiles[relPath] + except KeyError: + self._bundleFiles[relPath] = pkg = RepositoryFile( + self, relPath, glider.formats.BUNDLE_SCHEMA, + needRole='bundle') + return pkg + + def getFilesToUpdate(self, now=None, trackingBundles=()): + if now == None: + now = time.time() + + need = set() + + # Fetch missing metafiles. + for f in self._metaFiles: + try: + f.load() + except OSError, e: + print "need", f.getPath() + logging.info("Couldn't load %s: %s. Must fetch it.", + f.getPath(), e) + need.add(f.getRelativePath()) + + # If the timestamp file is out of date, we need to fetch it no + # matter what. (Even if it is isn't signed, it can't possibly + # be good.) + ts = self._timestampFile.get() + if ts: + age = now - glider.formats.parseTime(ts['at']) + ts = glider.formats.TimestampFile.fromJSon(ts) + if age > MAX_TIMESTAMP_AGE: + need.add(self._timestampFile.getRelativePath()) + + # If the keylist isn't signed right, we can't check the + # signatures on anything else. + if self._keylistFile.get(): + s = self._keylistFile.checkSignatures() + if not s.isValid(): # For now only require one master key. + need.add(self._keylistFile.getRelativePath()) + + if need: + return need + + # Import the keys from the keylist. + self._keyDB.addFromKeylist(self._keylistFile.get()) + + # If the timestamp isn't signed right, get a new timestamp and a + # new keylist. + s = self._timestampFile.checkSignatures() + if not s.isValid(): + need.add(self._keylistFile.getRelativePath()) + need.add(self._timestampFile.getRelativePath()) + return need + + # FINALLY, we know we have an up-to-date, signed timestamp + # file. Check whether the keys and mirrors file are as + # authenticated. + h_kf = glider.formats.getDigest(self._keylistFile.get()) + h_expected = ts.getKeylistInfo().getHash() + if h_kf != h_expected: + need.add(self._keylistFile.getRelativePath()) + + if need: + return need + + s = self._mirrorlistFile.checkSignatures() + if not s.isValid(): + need.add(self._mirrorlistFile.getRelativePath()) + + h_mf = glider.formats.getDigest(self._mirrorlistFile.get()) + h_expected = ts.getMirrorlistInfo().getHash() + if h_mf != h_expected: + need.add(self._mirrorlistFile.getRelativePath()) + + if need: + return need + + # Okay; that's it for the metadata. Do we have the right + # bundles? + bundles = {} + for b in trackingBundles: + try: + binfo = ts.getBundleInfo(b) + except KeyError: + logging.warn("Unrecognized bundle %s"%b) + continue + + rp = binfo.getRelativePath() + bfile = self.getBundleFile(rp) + try: + bfile.load() + except OSError: + need.add(rp) + continue + + h_b = glider.formats.getDigest(bfile.get()) + h_expected = binfo.getHash() + if h_b != h_expected: + need.add(rp) + continue + + s = bfile.checkSignatures() + if not s.isValid(): + # Can't actually use it. + continue + + bundles[rp] = bfile + + # Okay. So we have some bundles. See if we have their packages. + packages = {} + for bfile in bundles.values(): + bundle = bfile.get() + for pkginfo in bundle['packages']: + rp = pkginfo['path'] + pfile = self.getPackageFile(rp) + try: + pfile.load() + except OSError: + need.add(rp) + continue + + h_p = glider.formats.getDigest(pfile.get()) + h_expected = glider.formats.parseHash(pkginfo['hash']) + if h_p != h_expected: + need.add(rp) + continue + + s = pfile.checkSignatures() + if not s.isValid(): + # Can't use it. + continue + packages[rp] = pfile + + # Finally, we have some packages. Do we have their underlying + # files? + for pfile in packages.values(): + package = pfile.get() + for f in package['files']: + rp, h = f[:2] + h_expected = glider.formats.parseHash(h) + fn = self.getFilename(rp) + try: + h_got = glider.formats.getFileDigest(fn) + except OSError: + need.add(rp) + continue + if h_got != h_expected: + need.add(rp) + # Okay; these are the files we need. + return need |